flake: update dependencies

- for example, current_clock csr can be connected to tx_init.done

.gitignore

@@ -3,3 +3,11 @@ examples/*.elf
 __pycache__
 
 build
+
+src/libboard_artiq/Cargo.toml
+src/libc/Cargo.toml
+src/libdyld/Cargo.toml
+src/libio/Cargo.toml
+src/libksupport/Cargo.toml
+src/runtime/Cargo.toml
+src/satman/Cargo.toml

README.md

@@ -4,14 +4,13 @@ ARTIQ on Zynq
 How to use
 ----------
 
-1. Install ARTIQ-6 or newer.
-2. Select the latest successful build on Hydra: https://nixbld.m-labs.hk/jobset/artiq/zynq
-3. Search for the job named ``<board>-<variant>-sd`` (for example: ``zc706-nist_clock-sd`` or ``zc706-nist_qc2-sd``).
-4. Download the ``boot.bin`` "binary distribution" and place it at the root of a FAT-formatted SD card.
-5. Optionally, create a ``config.txt`` configuration file at the root of the SD card containing ``key=value`` pairs on each line. Use the ``ip``, ``ip6`` and ``mac`` keys to respectively set the IPv4, IPv6 and MAC address of the board. Configuring an IPv6 address is entirely optional. If these keys are not found, the firmware will use default values that may or may not be compatible with your network.
-6. Insert the SD card into the board and set up the board to boot from the SD card. For the ZC706, this is achieved by placing the large DIP switch SW11 in the 00110 position.
-7. Power up the board. After the firmware starts successfully, it should respond to ping at its IP addresses, and boot messages can be observed from its UART at 115200bps.
-8. Create and use an ARTIQ device database as usual, but set ``"target": "cortexa9"`` in the arguments of the core device.
+1. Install the ARTIQ version that corresponds to the artiq-zynq version you are targeting.
+2. To obtain firmware binaries, select the latest successful build on [Hydra](https://nixbld.m-labs.hk/) for the targeted artiq-zynq version, or use AFWS. If using Hydra, search for the job named ``<board>-<variant>-sd`` (for example: ``zc706-nist_clock-sd`` or ``zc706-nist_qc2-sd``).
+3. Place the ``boot.bin`` file, obtained from Hydra's "binary distribution" download link or from AFWS, at the root of a FAT-formatted SD card.
+4. Optionally, create a ``config.txt`` configuration file at the root of the SD card containing ``key=value`` pairs on each line. Use the ``ip``, ``ip6`` and ``mac`` keys to respectively set the IPv4, IPv6 and MAC address of the board. Configuring an IPv6 address is entirely optional. If these keys are not found, the firmware will use default values that may or may not be compatible with your network.
+5. Insert the SD card into the board and set up the board to boot from the SD card. For the ZC706, this is achieved by placing the large DIP switch SW11 in the 00110 position.
+6. Power up the board. After the firmware starts successfully, it should respond to ping at its IP addresses, and boot messages can be observed from its UART at 115200bps.
+7. Create and use an ARTIQ device database as usual, but set ``"target": "cortexa9"`` in the arguments of the core device.
 
 Configuration
 -------------
@@ -33,7 +32,7 @@ not implemented as it seems not very useful.
 Development instructions
 ------------------------
 
-ARTIQ on Zynq is packaged using the [Nix](https://nixos.org) Flakes system. Install Nix 2.4+ and enable flakes by adding ``experimental-features = nix-command flakes`` to ``nix.conf`` (e.g. ``~/.config/nix/nix.conf``). 
+ARTIQ on Zynq is packaged using the [Nix](https://nixos.org) Flakes system. Install Nix 2.8+ and enable flakes by adding ``experimental-features = nix-command flakes`` to ``nix.conf`` (e.g. ``~/.config/nix/nix.conf``).
 
 Pure build with Nix and execution on a remote JTAG server:
 
@@ -59,12 +58,11 @@ Notes:
 - When calling make, you need to specify both the variant and firmware type.
 - Firmware type must be either ``runtime`` for DRTIO-less or DRTIO master variants, or ``satman`` for DRTIO satellite.
 - If the board is connected to the local machine, use the ``local_run.sh`` script.
-- To update ``zynq-rs``, update the cargo files as per usual for Rust projects, but also keep ``flake.lock`` in sync.
 
 License
 -------
 
-Copyright (C) 2019-2022 M-Labs Limited.
+Copyright (C) 2019-2023 M-Labs Limited.
 
 ARTIQ is free software: you can redistribute it and/or modify
 it under the terms of the GNU Lesser General Public License as published by

examples/device_db.py

@@ -8,7 +8,7 @@ device_db = {
         "arguments": {
             "host": "192.168.1.52",
             "ref_period": 1e-9,
-            "ref_multiplier": 1,
+            "ref_multiplier": 8,
             "target": "cortexa9"
         }
     },
@@ -59,6 +59,14 @@ device_db["ad9914dds1"] = {
     "arguments": {"sysclk": 3e9, "bus_channel": 50, "channel": 1},
 }
 
+for i in range(4):
+    device_db["ttl"+str(i)+"_counter"] = {
+        "type": "local",
+        "module": "artiq.coredevice.edge_counter",
+        "class": "EdgeCounter",
+        "arguments": {"channel": 52+i}
+    }
+
 # for ARTIQ test suite
 device_db.update(
     loop_out="ttl0",

flake.lock

@@ -11,11 +11,11 @@
         "src-pythonparser": "src-pythonparser"
       },
       "locked": {
-        "lastModified": 1654087576,
-        "narHash": "sha256-d+LCvCXikhhEjsf4qBZakx3Sda85p+4vVZVwE1YuS90=",
-        "ref": "master",
-        "rev": "68ef0073ea66fe9c7f7c178979ecb8c681c47caa",
-        "revCount": 8068,
+        "lastModified": 1701663011,
+        "narHash": "sha256-cpVntgYbaj990yQdrt2YusaZp6VUG4xsYvMcYnt/W1I=",
+        "ref": "refs/heads/master",
+        "rev": "49267671f99455cd396a845f4655a380df769c0d",
+        "revCount": 8602,
         "type": "git",
         "url": "https://github.com/m-labs/artiq.git"
       },
@@ -26,6 +26,7 @@
     },
     "artiq-comtools": {
       "inputs": {
+        "flake-utils": "flake-utils",
         "nixpkgs": [
           "artiq",
           "nixpkgs"
@@ -36,11 +37,11 @@
         ]
       },
       "locked": {
-        "lastModified": 1654007592,
-        "narHash": "sha256-vaDFhE1ItjqtIcinC/6RAJGbj44pxxMUEeQUa3FtgEE=",
+        "lastModified": 1693473687,
+        "narHash": "sha256-BdLddCWbvoEyakcGwhph9b5dIU1iA0hCQV7KYgU8nos=",
         "owner": "m-labs",
         "repo": "artiq-comtools",
-        "rev": "cb73281154656ee8f74db1866859e31bf42755cd",
+        "rev": "f522ef3dbc65961f17b2d3d41e927409d970fd79",
         "type": "github"
       },
       "original": {
@@ -49,14 +50,32 @@
         "type": "github"
       }
     },
+    "flake-utils": {
+      "inputs": {
+        "systems": "systems"
+      },
+      "locked": {
+        "lastModified": 1692799911,
+        "narHash": "sha256-3eihraek4qL744EvQXsK1Ha6C3CR7nnT8X2qWap4RNk=",
+        "owner": "numtide",
+        "repo": "flake-utils",
+        "rev": "f9e7cf818399d17d347f847525c5a5a8032e4e44",
+        "type": "github"
+      },
+      "original": {
+        "owner": "numtide",
+        "repo": "flake-utils",
+        "type": "github"
+      }
+    },
     "mozilla-overlay": {
       "flake": false,
       "locked": {
-        "lastModified": 1650459918,
-        "narHash": "sha256-sroCK+QJTmoXtcRkwZyKOP9iAYOPID2Bwdxn4GkG16w=",
+        "lastModified": 1695805681,
+        "narHash": "sha256-1ElPLD8eFfnuIk0G52HGGpRtQZ4QPCjChRlEOfkZ5ro=",
         "owner": "mozilla",
         "repo": "nixpkgs-mozilla",
-        "rev": "e1f7540fc0a8b989fb8cf701dc4fd7fc76bcf168",
+        "rev": "6eabade97bc28d707a8b9d82ad13ef143836736e",
         "type": "github"
       },
       "original": {
@@ -68,11 +87,11 @@
     "mozilla-overlay_2": {
       "flake": false,
       "locked": {
-        "lastModified": 1650459918,
-        "narHash": "sha256-sroCK+QJTmoXtcRkwZyKOP9iAYOPID2Bwdxn4GkG16w=",
+        "lastModified": 1695805681,
+        "narHash": "sha256-1ElPLD8eFfnuIk0G52HGGpRtQZ4QPCjChRlEOfkZ5ro=",
         "owner": "mozilla",
         "repo": "nixpkgs-mozilla",
-        "rev": "e1f7540fc0a8b989fb8cf701dc4fd7fc76bcf168",
+        "rev": "6eabade97bc28d707a8b9d82ad13ef143836736e",
         "type": "github"
       },
       "original": {
@@ -84,11 +103,11 @@
     "mozilla-overlay_3": {
       "flake": false,
       "locked": {
-        "lastModified": 1650459918,
-        "narHash": "sha256-sroCK+QJTmoXtcRkwZyKOP9iAYOPID2Bwdxn4GkG16w=",
+        "lastModified": 1695805681,
+        "narHash": "sha256-1ElPLD8eFfnuIk0G52HGGpRtQZ4QPCjChRlEOfkZ5ro=",
         "owner": "mozilla",
         "repo": "nixpkgs-mozilla",
-        "rev": "e1f7540fc0a8b989fb8cf701dc4fd7fc76bcf168",
+        "rev": "6eabade97bc28d707a8b9d82ad13ef143836736e",
         "type": "github"
       },
       "original": {
@@ -99,16 +118,16 @@
     },
     "nixpkgs": {
       "locked": {
-        "lastModified": 1653920503,
-        "narHash": "sha256-BBeCZwZImtjP3oYy4WogkQYy5OxNyfNciVSc1AfZgLQ=",
+        "lastModified": 1701389149,
+        "narHash": "sha256-rU1suTIEd5DGCaAXKW6yHoCfR1mnYjOXQFOaH7M23js=",
         "owner": "NixOS",
         "repo": "nixpkgs",
-        "rev": "a634c8f6c1fbf9b9730e01764999666f3436f10a",
+        "rev": "5de0b32be6e85dc1a9404c75131316e4ffbc634c",
         "type": "github"
       },
       "original": {
         "owner": "NixOS",
-        "ref": "nixos-22.05",
+        "ref": "nixos-23.11",
         "repo": "nixpkgs",
         "type": "github"
       }
@@ -128,11 +147,11 @@
         ]
       },
       "locked": {
-        "lastModified": 1654006751,
-        "narHash": "sha256-OWAnoTCutvTQcYdtdtLQuL6uRtG+7Jz7sbRhcScv8bo=",
+        "lastModified": 1701572254,
+        "narHash": "sha256-ixq8dlpyOytDr+d/OmW8v1Ioy9V2G2ibOlNj8GFDSq4=",
         "owner": "m-labs",
         "repo": "sipyco",
-        "rev": "b3d03a94c751a24769c54a61a0dbe9d6af52dade",
+        "rev": "cceac0df537887135f99aa6b1bdd82853f16b4d6",
         "type": "github"
       },
       "original": {
@@ -144,11 +163,11 @@
     "src-migen": {
       "flake": false,
       "locked": {
-        "lastModified": 1650337393,
-        "narHash": "sha256-rm1SlFmF2ASz0vIy2nDEzGlyRw2oYNeJRr8Kh8Mg2Qc=",
+        "lastModified": 1699335478,
+        "narHash": "sha256-BsubN4Mfdj02QPK6ZCrl+YOaSg7DaLQdSCVP49ztWik=",
         "owner": "m-labs",
         "repo": "migen",
-        "rev": "d4e3f34177c32f09904397179e6ed9c83175e528",
+        "rev": "fd0bf5855a1367eab14b0d6f7f8266178e25d78e",
         "type": "github"
       },
       "original": {
@@ -160,11 +179,11 @@
     "src-misoc": {
       "flake": false,
       "locked": {
-        "lastModified": 1649324486,
-        "narHash": "sha256-Mw/fQS3lHFvCm7L1k63joRkz5uyijQfywcOq+X2+o2s=",
-        "ref": "master",
-        "rev": "f1dc58d2b8c222ba41c25cee4301626625f46e43",
-        "revCount": 2420,
+        "lastModified": 1699352904,
+        "narHash": "sha256-SglyTmXOPv8jJOjwAjJrj/WhAkItQfUbvKfUqrynwRg=",
+        "ref": "refs/heads/master",
+        "rev": "a53859f2167c31ab5225b6c09f30cf05527b94f4",
+        "revCount": 2452,
         "submodules": true,
         "type": "git",
         "url": "https://github.com/m-labs/misoc.git"
@@ -191,6 +210,21 @@
         "type": "github"
       }
     },
+    "systems": {
+      "locked": {
+        "lastModified": 1681028828,
+        "narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=",
+        "owner": "nix-systems",
+        "repo": "default",
+        "rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e",
+        "type": "github"
+      },
+      "original": {
+        "owner": "nix-systems",
+        "repo": "default",
+        "type": "github"
+      }
+    },
     "zynq-rs": {
       "inputs": {
         "mozilla-overlay": "mozilla-overlay_3",
@@ -200,11 +234,11 @@
         ]
       },
       "locked": {
-        "lastModified": 1654002148,
-        "narHash": "sha256-Ztzx7ze8o0gdu+sDpc+PS2M88h/b22rwFq0b12wMqZQ=",
-        "ref": "master",
-        "rev": "6cd32f6ee0e7bb7ce5a279f8be943903a393a476",
-        "revCount": 617,
+        "lastModified": 1701591382,
+        "narHash": "sha256-/zgofoUZQPbBsjQ74SRusmVHibdjZrae24UPXBZKN4E=",
+        "ref": "refs/heads/master",
+        "rev": "a43b8bf64e939c8f2dc7095beabea6e6a79f8378",
+        "revCount": 635,
         "type": "git",
         "url": "https://git.m-labs.hk/m-labs/zynq-rs"
       },

flake.nix

@@ -12,6 +12,7 @@
     zynqpkgs = zynq-rs.packages.x86_64-linux;
     artiqpkgs = artiq.packages.x86_64-linux;
 
+    rust = zynq-rs.rust;
     rustPlatform = zynq-rs.rustPlatform;
 
     fastnumbers = pkgs.python3Packages.buildPythonPackage rec {
@@ -37,63 +38,53 @@
 
     ramda = pkgs.python3Packages.buildPythonPackage {
       pname = "ramda";
-      version = "unstable-2019-02-01";
+      version = "unstable-2020-04-11";
 
       src = pkgs.fetchFromGitHub {
         owner = "peteut";
         repo = "ramda.py";
-        rev = "bd58f8e69d0e9a713d9c1f286a1ac5e5603956b1";
-        sha256 = "0qzd5yp9lbaham8p1wiymdjapzbqsli7lvngv24c3z4ybd9jlq9g";
+        rev = "d315a9717ebd639366bf3fe26bad9e3d08ec3c49";
+        sha256 = "sha256-bmSt/IHDnULsZjsC6edELnNH7LoJSVF4L4XhwBAXRkY=";
       };
 
       nativeBuildInputs = with pkgs.python3Packages; [ pbr ];
       propagatedBuildInputs = with pkgs.python3Packages; [ future fastnumbers ];
 
-      checkInputs = with pkgs.python3Packages; [ pytest pytest-flake8 ];
+      checkInputs = with pkgs.python3Packages; [ pytest ];
       checkPhase = "pytest";
       doCheck = false;
 
       preBuild = ''
-        export PBR_VERSION=0.0.1
+        export PBR_VERSION=0.5.5
       '';
     };
 
     migen-axi = pkgs.python3Packages.buildPythonPackage {
       pname = "migen-axi";
-      version = "unstable-2021-09-15";
+      version = "unstable-2023-01-06";
 
       src = pkgs.fetchFromGitHub {
         owner = "peteut";
         repo = "migen-axi";
-        rev = "9763505ee96acd7572280a2d1233721342dc7c3f";
-        sha256 = "15c7g05n183rka66fl1glzp6h7xjlpy1p6k8biry24dangsmxmvg";
+        rev = "27eaa84a70a3abfe1930c86c36c4de2cd652da35";
+        sha256 = "sha256-3Y9W5ns+1wbVd14iePzgSBzE+LxnGMUDtUw3BccFt80=";
       };
 
-      nativeBuildInputs = with pkgs.python3Packages; [ pbr ];
+      format = "pyproject";
+
       propagatedBuildInputs = with pkgs.python3Packages; [ setuptools click numpy toolz jinja2 ramda artiqpkgs.migen artiqpkgs.misoc ];
 
+      checkInputs = with pkgs.python3Packages; [ pytest-runner pytestCheckHook pytest-timeout ];
+
+      # migen/misoc version checks are broken with pyproject for some reason
       postPatch = ''
-        substituteInPlace requirements.txt \
-          --replace "jinja2==2.11.3" "jinja2"
-        substituteInPlace requirements.txt \
-          --replace "future==0.18.2" "future"
-        substituteInPlace requirements.txt \
-          --replace "ramda==0.5.5" "ramda"
-        substituteInPlace requirements.txt \
-          --replace "colorama==0.4.3" "colorama"
-        substituteInPlace requirements.txt \
-          --replace "toolz==0.10.0" "toolz"
-        substituteInPlace requirements.txt \
-          --replace "pyserial==3.4" "pyserial"
-        substituteInPlace requirements.txt \
-          --replace "markupsafe==1.1.1" "markupsafe"
-      '';
-
-      checkInputs = with pkgs.python3Packages; [ pytest pytest-timeout pytest-flake8 ];
-      checkPhase = "pytest";
-
-      preBuild = ''
-        export PBR_VERSION=0.0.1
+        sed -i "1,4d" pyproject.toml
+        substituteInPlace pyproject.toml \
+          --replace '"migen@git+https://github.com/m-labs/migen",' ""
+        substituteInPlace pyproject.toml \
+          --replace '"misoc@git+https://github.com/m-labs/misoc.git",' ""
+        # pytest-flake8 is broken with recent flake8. Re-enable after fix.
+        substituteInPlace setup.cfg --replace '--flake8' ""
       '';
     };
     binutils = { platform, target, zlib }: pkgs.stdenv.mkDerivation rec {
@@ -128,28 +119,27 @@
       fwtype = if builtins.elem variant sat_variants then "satman" else "runtime";
 
       firmware = rustPlatform.buildRustPackage rec {
-
         name = "firmware";
         src = ./src;
         cargoLock = { 
           lockFile = src/Cargo.lock;
           outputHashes = {
-            "libasync-0.0.0" = "sha256-xuwesRrQiccopPTCkwGqQxld74X74q7EVsKIrE0zirc=";
-          };
+              "tar-no-std-0.1.8" = "sha256-xm17108v4smXOqxdLvHl9CxTCJslmeogjm4Y87IXFuM=";
+            };
         };
 
         nativeBuildInputs = [
           pkgs.gnumake
-          (pkgs.python3.withPackages(ps: (with artiqpkgs; [ ps.jsonschema migen migen-axi misoc artiq ])))
-          artiqpkgs.artiq
+          (pkgs.python3.withPackages(ps: [ ps.jsonschema artiqpkgs.migen migen-axi artiqpkgs.misoc artiqpkgs.artiq ]))
           zynqpkgs.cargo-xbuild
           pkgs.llvmPackages_9.llvm
           pkgs.llvmPackages_9.clang-unwrapped
         ];
         buildPhase = ''
-          export XARGO_RUST_SRC="${rustPlatform.rust.rustc}/lib/rustlib/src/rust/library"
+          export XARGO_RUST_SRC="${rust}/lib/rustlib/src/rust/library"
           export CLANG_EXTRA_INCLUDE_DIR="${pkgs.llvmPackages_9.clang-unwrapped.lib}/lib/clang/9.0.1/include"
           export CARGO_HOME=$(mktemp -d cargo-home.XXX)
+          export ZYNQ_RS=${zynq-rs}
           make TARGET=${target} GWARGS="${if json == null then "-V ${variant}" else json}" ${fwtype}
         '';
 
@@ -163,12 +153,12 @@
 
         doCheck = false;
         dontFixup = true;
+        auditable = false;
       };
       gateware = pkgs.runCommand "${target}-${variant}-gateware"
         {
           nativeBuildInputs = [ 
-            (pkgs.python3.withPackages(ps: (with artiqpkgs; [ ps.jsonschema migen migen-axi misoc artiq ])))
-            artiqpkgs.artiq
+            (pkgs.python3.withPackages(ps: [ ps.jsonschema artiqpkgs.migen migen-axi artiqpkgs.misoc artiqpkgs.artiq ]))
             artiqpkgs.vivado
           ];
         }
@@ -252,8 +242,7 @@
       name = "gateware-sim";
       
       nativeBuildInputs = [ 
-        (pkgs.python3.withPackages(ps: (with artiqpkgs; [ migen migen-axi artiq ]))) 
-        artiqpkgs.artiq
+        (pkgs.python3.withPackages(ps: [ artiqpkgs.migen migen-axi artiqpkgs.artiq ]))
       ];
 
       phases = [ "buildPhase" ];
@@ -265,6 +254,24 @@
         '';
     };
 
+    fmt-check = pkgs.stdenvNoCC.mkDerivation {
+      name = "fmt-check";
+
+      src = ./src;
+
+      nativeBuildInputs = [ rust pkgs.gnumake ];
+
+      phases = [ "unpackPhase" "buildPhase" ];
+
+      buildPhase =
+        ''
+        export ZYNQ_RS=${zynq-rs}
+        make manifests
+        cargo fmt -- --check
+        touch $out
+        '';
+    };
+
     # for hitl-tests
     zc706-nist_qc2 = (build { target = "zc706"; variant = "nist_qc2"; });
     zc706-hitl-tests = pkgs.stdenv.mkDerivation {
@@ -329,33 +336,40 @@
         '';
     };
   in rec {
-    packages.x86_64-linux = (build { target = "zc706"; variant = "nist_clock"; }) //
+    packages.x86_64-linux =
+      {
+        inherit fastnumbers artiq-netboot ramda migen-axi binutils-arm;
+      } //
+      (build { target = "zc706"; variant = "nist_clock"; }) //
       (build { target = "zc706"; variant = "nist_clock_master"; }) //
+      (build { target = "zc706"; variant = "nist_clock_master_100mhz"; }) //
       (build { target = "zc706"; variant = "nist_clock_satellite"; }) //
       (build { target = "zc706"; variant = "nist_clock_satellite_100mhz"; }) //
       (build { target = "zc706"; variant = "nist_qc2"; }) //
       (build { target = "zc706"; variant = "nist_qc2_master"; }) //
+      (build { target = "zc706"; variant = "nist_qc2_master_100mhz"; }) //
       (build { target = "zc706"; variant = "nist_qc2_satellite"; }) //
       (build { target = "zc706"; variant = "nist_qc2_satellite_100mhz"; }) //
       (build { target = "zc706"; variant = "acpki_nist_clock"; }) //
       (build { target = "zc706"; variant = "acpki_nist_clock_master"; }) //
+      (build { target = "zc706"; variant = "acpki_nist_clock_master_100mhz"; }) //
       (build { target = "zc706"; variant = "acpki_nist_clock_satellite"; }) //
       (build { target = "zc706"; variant = "acpki_nist_clock_satellite_100mhz"; }) //
       (build { target = "zc706"; variant = "acpki_nist_qc2"; }) //
       (build { target = "zc706"; variant = "acpki_nist_qc2_master"; }) //
+      (build { target = "zc706"; variant = "acpki_nist_qc2_master_100mhz"; }) //
       (build { target = "zc706"; variant = "acpki_nist_qc2_satellite"; }) //
       (build { target = "zc706"; variant = "acpki_nist_qc2_satellite_100mhz"; }) //
       (build { target = "kasli_soc"; variant = "demo"; json = ./demo.json; }) //
       (build { target = "kasli_soc"; variant = "master"; json = ./kasli-soc-master.json; }) //
       (build { target = "kasli_soc"; variant = "satellite"; json = ./kasli-soc-satellite.json; });
 
-    hydraJobs = packages.x86_64-linux // { inherit zc706-hitl-tests; inherit gateware-sim; };
+    hydraJobs = packages.x86_64-linux // { inherit zc706-hitl-tests; inherit gateware-sim; inherit fmt-check; };
 
     devShell.x86_64-linux = pkgs.mkShell {
       name = "artiq-zynq-dev-shell";
       buildInputs = with pkgs; [
-        rustPlatform.rust.rustc
-        rustPlatform.rust.cargo
+        rust
         llvmPackages_9.llvm
         llvmPackages_9.clang-unwrapped
         gnumake
@@ -369,8 +383,9 @@
         artiqpkgs.vivado
         binutils-arm
       ];
-      XARGO_RUST_SRC = "${rustPlatform.rust.rustc}/lib/rustlib/src/rust/library";
+      XARGO_RUST_SRC = "${rust}/lib/rustlib/src/rust/library";
       CLANG_EXTRA_INCLUDE_DIR = "${pkgs.llvmPackages_9.clang-unwrapped.lib}/lib/clang/9.0.1/include";
+      ZYNQ_RS = "${zynq-rs}";
       OPENOCD_ZYNQ = "${zynq-rs}/openocd";
       SZL = "${zynqpkgs.szl}";
     };

src/Cargo.lock

@@ -1,5 +1,13 @@
 # This file is automatically @generated by Cargo.
 # It is not intended for manual editing.
+version = 3
+
+[[package]]
+name = "arrayvec"
+version = "0.7.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "96d30a06541fbafbc7f82ed10c06164cfbd2c401138f6addd8404629c4b16711"
+
 [[package]]
 name = "async-recursion"
 version = "0.3.2"
@@ -47,9 +55,9 @@ checksum = "14c189c53d098945499cdfa7ecc63567cf3886b3332b312a5b4585d8d3a6a610"
 
 [[package]]
 name = "cc"
-version = "1.0.73"
+version = "1.0.77"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2fff2a6927b3bb87f9595d67196a70493f627687a71d87a0d692242c33f58c11"
+checksum = "e9f73505338f7d905b19d18738976aae232eb46b8efc15554ffc56deb5d9ebe4"
 
 [[package]]
 name = "cfg-if"
@@ -136,9 +144,9 @@ dependencies = [
 
 [[package]]
 name = "futures"
-version = "0.3.21"
+version = "0.3.25"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f73fe65f54d1e12b726f517d3e2135ca3125a437b6d998caf1962961f7172d9e"
+checksum = "38390104763dc37a5145a53c29c63c1290b5d316d6086ec32c293f6736051bb0"
 dependencies = [
  "futures-channel",
  "futures-core",
@@ -150,9 +158,9 @@ dependencies = [
 
 [[package]]
 name = "futures-channel"
-version = "0.3.21"
+version = "0.3.25"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c3083ce4b914124575708913bca19bfe887522d6e2e6d0952943f5eac4a74010"
+checksum = "52ba265a92256105f45b719605a571ffe2d1f0fea3807304b522c1d778f79eed"
 dependencies = [
  "futures-core",
  "futures-sink",
@@ -160,21 +168,21 @@ dependencies = [
 
 [[package]]
 name = "futures-core"
-version = "0.3.21"
+version = "0.3.25"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0c09fd04b7e4073ac7156a9539b57a484a8ea920f79c7c675d05d289ab6110d3"
+checksum = "04909a7a7e4633ae6c4a9ab280aeb86da1236243a77b694a49eacd659a4bd3ac"
 
 [[package]]
 name = "futures-io"
-version = "0.3.21"
+version = "0.3.25"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "fc4045962a5a5e935ee2fdedaa4e08284547402885ab326734432bed5d12966b"
+checksum = "00f5fb52a06bdcadeb54e8d3671f8888a39697dcb0b81b23b55174030427f4eb"
 
 [[package]]
 name = "futures-macro"
-version = "0.3.21"
+version = "0.3.25"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "33c1e13800337f4d4d7a316bf45a567dbcb6ffe087f16424852d97e97a91f512"
+checksum = "bdfb8ce053d86b91919aad980c220b1fb8401a9394410e1c289ed7e66b61835d"
 dependencies = [
  "proc-macro2",
  "quote",
@@ -183,21 +191,21 @@ dependencies = [
 
 [[package]]
 name = "futures-sink"
-version = "0.3.21"
+version = "0.3.25"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "21163e139fa306126e6eedaf49ecdb4588f939600f0b1e770f4205ee4b7fa868"
+checksum = "39c15cf1a4aa79df40f1bb462fb39676d0ad9e366c2a33b590d7c66f4f81fcf9"
 
 [[package]]
 name = "futures-task"
-version = "0.3.21"
+version = "0.3.25"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "57c66a976bf5909d801bbef33416c41372779507e7a6b3a5e25e4749c58f776a"
+checksum = "2ffb393ac5d9a6eaa9d3fdf37ae2776656b706e200c8e16b1bdb227f5198e6ea"
 
 [[package]]
 name = "futures-util"
-version = "0.3.21"
+version = "0.3.25"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d8b7abd5d659d9b90c8cba917f6ec750a74e2dc23902ef9cd4cc8c8b22e6036a"
+checksum = "197676987abd2f9cadff84926f410af1c183608d36641465df73ae8211dc65d6"
 dependencies = [
  "futures-core",
  "futures-macro",
@@ -216,10 +224,37 @@ dependencies = [
  "libsupport_zynq",
 ]
 
+[[package]]
+name = "ksupport"
+version = "0.1.0"
+dependencies = [
+ "build_zynq",
+ "byteorder",
+ "core_io",
+ "cslice",
+ "dwarf",
+ "dyld",
+ "io",
+ "libasync",
+ "libboard_artiq",
+ "libboard_zynq",
+ "libc",
+ "libconfig",
+ "libcortex_a9",
+ "libm",
+ "libregister",
+ "libsupport_zynq",
+ "log",
+ "log_buffer",
+ "nb 0.1.3",
+ "unwind",
+ "vcell",
+ "void",
+]
+
 [[package]]
 name = "libasync"
 version = "0.0.0"
-source = "git+https://git.m-labs.hk/M-Labs/zynq-rs.git#605c8f73a60cf41a6e2d535c5850933d3154e935"
 dependencies = [
  "embedded-hal",
  "libcortex_a9",
@@ -242,6 +277,7 @@ dependencies = [
  "libconfig",
  "libcortex_a9",
  "libregister",
+ "libsupport_zynq",
  "log",
  "log_buffer",
  "nb 1.0.0",
@@ -251,7 +287,6 @@ dependencies = [
 [[package]]
 name = "libboard_zynq"
 version = "0.0.0"
-source = "git+https://git.m-labs.hk/M-Labs/zynq-rs.git#605c8f73a60cf41a6e2d535c5850933d3154e935"
 dependencies = [
  "bit_field",
  "embedded-hal",
@@ -276,7 +311,6 @@ dependencies = [
 [[package]]
 name = "libconfig"
 version = "0.1.0"
-source = "git+https://git.m-labs.hk/M-Labs/zynq-rs.git#605c8f73a60cf41a6e2d535c5850933d3154e935"
 dependencies = [
  "core_io",
  "fatfs",
@@ -287,7 +321,6 @@ dependencies = [
 [[package]]
 name = "libcortex_a9"
 version = "0.0.0"
-source = "git+https://git.m-labs.hk/M-Labs/zynq-rs.git#605c8f73a60cf41a6e2d535c5850933d3154e935"
 dependencies = [
  "bit_field",
  "libregister",
@@ -296,14 +329,13 @@ dependencies = [
 
 [[package]]
 name = "libm"
-version = "0.2.2"
+version = "0.2.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "33a33a362ce288760ec6a508b94caaec573ae7d3bbbd91b87aa0bad4456839db"
+checksum = "348108ab3fba42ec82ff6e9564fc4ca0247bdccdc68dd8af9764bbc79c3c8ffb"
 
 [[package]]
 name = "libregister"
 version = "0.0.0"
-source = "git+https://git.m-labs.hk/M-Labs/zynq-rs.git#605c8f73a60cf41a6e2d535c5850933d3154e935"
 dependencies = [
  "bit_field",
  "vcell",
@@ -313,7 +345,6 @@ dependencies = [
 [[package]]
 name = "libsupport_zynq"
 version = "0.0.0"
-source = "git+https://git.m-labs.hk/M-Labs/zynq-rs.git#605c8f73a60cf41a6e2d535c5850933d3154e935"
 dependencies = [
  "cc",
  "compiler_builtins",
@@ -332,9 +363,9 @@ checksum = "822add9edb1860698b79522510da17bef885171f75aa395cff099d770c609c24"
 
 [[package]]
 name = "log"
-version = "0.4.14"
+version = "0.4.17"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "51b9bbe6c47d51fc3e1a9b945965946b4c44142ab8792c50835a980d362c2710"
+checksum = "abb12e687cfb44aa40f41fc3978ef76448f9b6038cad6aef4259d3c095a2382e"
 dependencies = [
  "cfg-if 1.0.0",
 ]
@@ -379,18 +410,18 @@ dependencies = [
 
 [[package]]
 name = "num-traits"
-version = "0.2.14"
+version = "0.2.15"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9a64b1ec5cda2586e284722486d802acf1f7dbdc623e2bfc57e65ca1cd099290"
+checksum = "578ede34cf02f8924ab9447f50c28075b4d3e5b269972345e7e0372b38c6cdcd"
 dependencies = [
  "autocfg",
 ]
 
 [[package]]
 name = "pin-project-lite"
-version = "0.2.8"
+version = "0.2.9"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e280fbe77cc62c91527259e9442153f4688736748d24660126286329742b4c6c"
+checksum = "e0a7ae3ac2f1173085d398531c705756c94a4c56843785df85a60c1a0afac116"
 
 [[package]]
 name = "pin-utils"
@@ -400,18 +431,18 @@ checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184"
 
 [[package]]
 name = "proc-macro2"
-version = "1.0.36"
+version = "1.0.43"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c7342d5883fbccae1cc37a2353b09c87c9b0f3afd73f5fb9bba687a1f733b029"
+checksum = "0a2ca2c61bc9f3d74d2886294ab7b9853abd9c1ad903a3ac7815c58989bb7bab"
 dependencies = [
- "unicode-xid",
+ "unicode-ident",
 ]
 
 [[package]]
 name = "quote"
-version = "1.0.15"
+version = "1.0.21"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "864d3e96a899863136fc6e99f3d7cae289dafe43bf2c5ac19b70df7210c0a145"
+checksum = "bbe448f377a7d6961e30f5955f9b8d106c3f5e449d493ee1b125c1d43c2b5179"
 dependencies = [
  "proc-macro2",
 ]
@@ -436,20 +467,20 @@ dependencies = [
  "embedded-hal",
  "futures",
  "io",
+ "ksupport",
  "libasync",
  "libboard_artiq",
  "libboard_zynq",
  "libc",
  "libconfig",
  "libcortex_a9",
- "libm",
  "libregister",
  "libsupport_zynq",
  "log",
  "log_buffer",
- "nb 0.1.3",
  "num-derive",
  "num-traits",
+ "tar-no-std",
  "unwind",
  "vcell",
  "void",
@@ -469,7 +500,11 @@ name = "satman"
 version = "0.0.0"
 dependencies = [
  "build_zynq",
+ "core_io",
+ "cslice",
  "embedded-hal",
+ "io",
+ "ksupport",
  "libasync",
  "libboard_artiq",
  "libboard_zynq",
@@ -501,20 +536,30 @@ dependencies = [
 
 [[package]]
 name = "syn"
-version = "1.0.86"
+version = "1.0.101"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8a65b3f4ffa0092e9887669db0eae07941f023991ab58ea44da8fe8e2d511c6b"
+checksum = "e90cde112c4b9690b8cbe810cba9ddd8bc1d7472e2cae317b69e9438c1cba7d2"
 dependencies = [
  "proc-macro2",
  "quote",
- "unicode-xid",
+ "unicode-ident",
 ]
 
 [[package]]
-name = "unicode-xid"
-version = "0.2.2"
+name = "tar-no-std"
+version = "0.1.8"
+source = "git+https://git.m-labs.hk/M-Labs/tar-no-std?rev=2ab6dc5#2ab6dc58e5249c59c4eb03eaf3a119bcdd678d32"
+dependencies = [
+ "arrayvec",
+ "bitflags",
+ "log",
+]
+
+[[package]]
+name = "unicode-ident"
+version = "1.0.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8ccb82d61f80a663efe1f787a51b16b5a51e3314d6ac365b08639f52387b33f3"
+checksum = "6ceab39d59e4c9499d4e5a8ee0e2735b891bb7308ac83dfb4e80cad195c9f6f3"
 
 [[package]]
 name = "unwind"

src/Cargo.toml

@@ -5,6 +5,7 @@ members = [
   "libdwarf",
   "libio",
   "libunwind",
+  "libksupport",
   "runtime",
   "satman"
 ]

src/Makefile

@@ -7,13 +7,20 @@ runtime: ../build/runtime.bin
 
 satman: ../build/satman.bin
 
-.PHONY: all runtime_target satman_target
+.PHONY: all manifests
+
+manifests = libboard_artiq/Cargo.toml libc/Cargo.toml libdyld/Cargo.toml libio/Cargo.toml libksupport/Cargo.toml runtime/Cargo.toml satman/Cargo.toml
+
+$(manifests): %.toml: %.toml.tpl
+	sed s+@@ZYNQ_RS@@+$(ZYNQ_RS)+g $< > $@
+
+manifests: $(manifests)
 
 ../build/pl.rs ../build/rustc-cfg ../build/mem.rs: gateware/* 
 	mkdir -p ../build
 	python gateware/$(TARGET).py -r ../build/pl.rs -c ../build/rustc-cfg -m ../build/mem.rs $(GWARGS)
 
-../build/firmware/armv7-none-eabihf/release/runtime: ../build/pl.rs ../build/rustc-cfg ../build/mem.rs $(shell find . -type f -print)
+../build/firmware/armv7-none-eabihf/release/runtime: ../build/pl.rs ../build/rustc-cfg ../build/mem.rs $(manifests) $(shell find . -type f -not -name Cargo.toml -print)
 	cd runtime && \
 	XBUILD_SYSROOT_PATH=`pwd`/../../build/sysroot \
 	cargo xbuild --release \
@@ -23,7 +30,7 @@ satman: ../build/satman.bin
 ../build/runtime.bin: ../build/firmware/armv7-none-eabihf/release/runtime
 	llvm-objcopy -O binary ../build/firmware/armv7-none-eabihf/release/runtime ../build/runtime.bin
 
-../build/firmware/armv7-none-eabihf/release/satman: ../build/pl.rs ../build/rustc-cfg ../build/mem.rs $(shell find . -type f -print)
+../build/firmware/armv7-none-eabihf/release/satman: ../build/pl.rs ../build/rustc-cfg ../build/mem.rs $(manifests) $(shell find . -type f -not -name Cargo.toml -print)
 	cd satman && \
 	XBUILD_SYSROOT_PATH=`pwd`/../../build/sysroot \
 	cargo xbuild --release \

src/gateware/config.py

@@ -0,0 +1,16 @@
+from misoc.integration import cpu_interface
+
+def write_csr_file(soc, filename):
+    with open(filename, "w") as f:
+        f.write(cpu_interface.get_csr_rust(
+            soc.get_csr_regions(), soc.get_csr_groups(), soc.get_constants()))
+
+def write_mem_file(soc, filename):
+    with open(filename, "w") as f:
+        f.write(cpu_interface.get_mem_rust(
+            soc.get_memory_regions(), soc.get_memory_groups(), None))
+
+def write_rustc_cfg_file(soc, filename):
+    with open(filename, "w") as f:
+        f.write(cpu_interface.get_rust_cfg(
+            soc.get_csr_regions(), soc.get_constants()))

src/gateware/kasli_soc.py

@@ -10,13 +10,13 @@ from migen.genlib.cdc import MultiReg
 from migen_axi.integration.soc_core import SoCCore
 from migen_axi.platforms import kasli_soc
 from misoc.interconnect.csr import *
-from misoc.integration import cpu_interface
+from misoc.cores import virtual_leds
 
 from artiq.coredevice import jsondesc
 from artiq.gateware import rtio, eem_7series
+from artiq.gateware.rtio.xilinx_clocking import fix_serdes_timing_path
 from artiq.gateware.rtio.phy import ttl_simple
-from artiq.gateware.rtio.xilinx_clocking import RTIOClockMultiplier
-from artiq.gateware.drtio.transceiver import gtx_7series
+from artiq.gateware.drtio.transceiver import gtx_7series, eem_serdes
 from artiq.gateware.drtio.siphaser import SiPhaser7Series
 from artiq.gateware.drtio.rx_synchronizer import XilinxRXSynchronizer
 from artiq.gateware.drtio import *
@@ -25,56 +25,8 @@ import dma
 import analyzer
 import acpki
 import drtio_aux_controller
-
-class RTIOCRG(Module, AutoCSR):
-    def __init__(self, platform):
-        self.pll_reset = CSRStorage(reset=1)
-        self.pll_locked = CSRStatus()
-        self.clock_domains.cd_rtio = ClockDomain()
-        self.clock_domains.cd_rtiox4 = ClockDomain(reset_less=True)
-
-        clk_synth = platform.request("cdr_clk_clean_fabric")
-        clk_synth_se = Signal()
-        platform.add_period_constraint(clk_synth.p, 8.0)
-        self.specials += [
-            Instance("IBUFGDS",
-                p_DIFF_TERM="TRUE", p_IBUF_LOW_PWR="FALSE",
-                i_I=clk_synth.p, i_IB=clk_synth.n, o_O=clk_synth_se),
-        ]
-
-        pll_locked = Signal()
-        rtio_clk = Signal()
-        rtiox4_clk = Signal()
-        fb_clk = Signal()
-        self.specials += [
-            Instance("PLLE2_ADV",
-                     p_STARTUP_WAIT="FALSE", o_LOCKED=pll_locked,
-                     p_BANDWIDTH="HIGH",
-                     p_REF_JITTER1=0.001,
-                     p_CLKIN1_PERIOD=8.0, p_CLKIN2_PERIOD=8.0,
-                     i_CLKIN2=clk_synth_se,
-                     # Warning: CLKINSEL=0 means CLKIN2 is selected
-                     i_CLKINSEL=0,
-
-                     # VCO @ 1.5GHz when using 125MHz input
-                     p_CLKFBOUT_MULT=12, p_DIVCLK_DIVIDE=1,
-                     i_CLKFBIN=fb_clk,
-                     i_RST=self.pll_reset.storage,
-
-                     o_CLKFBOUT=fb_clk,
-
-                     p_CLKOUT0_DIVIDE=3, p_CLKOUT0_PHASE=0.0,
-                     o_CLKOUT0=rtiox4_clk,
-
-                     p_CLKOUT1_DIVIDE=12, p_CLKOUT1_PHASE=0.0,
-                     o_CLKOUT1=rtio_clk),
-            Instance("BUFG", i_I=rtio_clk, o_O=self.cd_rtio.clk),
-            Instance("BUFG", i_I=rtiox4_clk, o_O=self.cd_rtiox4.clk),
-
-            AsyncResetSynchronizer(self.cd_rtio, ~pll_locked),
-            MultiReg(pll_locked, self.pll_locked.status)
-        ]
-
+import zynq_clocking
+from config import write_csr_file, write_mem_file, write_rustc_cfg_file
 
 eem_iostandard_dict = {
      0: "LVDS_25",
@@ -109,35 +61,93 @@ class SMAClkinForward(Module):
         ]
 
 
+class GTPBootstrapClock(Module):
+    def __init__(self, platform, freq=125e6):
+        self.clock_domains.cd_bootstrap = ClockDomain(reset_less=True)
+        self.cd_bootstrap.clk.attr.add("keep")
+
+        bootstrap_125 = platform.request("clk125_gtp")
+        bootstrap_se = Signal()
+        clk_out = Signal()
+        platform.add_period_constraint(bootstrap_125.p, 8.0)
+        self.specials += [
+            Instance("IBUFDS_GTE2",
+                i_CEB=0,
+                i_I=bootstrap_125.p, i_IB=bootstrap_125.n, 
+                o_O=bootstrap_se,
+                p_CLKCM_CFG="TRUE",
+                p_CLKRCV_TRST="TRUE",
+                p_CLKSWING_CFG=3),
+            Instance("BUFG", i_I=bootstrap_se, o_O=clk_out)
+        ]
+        if freq == 125e6:
+            self.comb += self.cd_bootstrap.clk.eq(clk_out)
+        elif freq == 100e6:
+            pll_fb = Signal()
+            pll_out = Signal()
+            self.specials += [
+                Instance("PLLE2_BASE",
+                    p_CLKIN1_PERIOD=8.0,
+                    i_CLKIN1=clk_out,
+                    i_CLKFBIN=pll_fb,
+                    o_CLKFBOUT=pll_fb,
+
+                    # VCO @ 1GHz
+                    p_CLKFBOUT_MULT=8, p_DIVCLK_DIVIDE=1,
+
+                    # 100MHz for bootstrap
+                    p_CLKOUT1_DIVIDE=10, p_CLKOUT1_PHASE=0.0, o_CLKOUT1=pll_out,
+                ),
+                Instance("BUFG", i_I=pll_out, o_O=self.cd_bootstrap.clk)
+            ]
+        else:
+            raise ValueError("Bootstrap frequency must be 100 or 125MHz")
+
+
 class GenericStandalone(SoCCore):
     def __init__(self, description, acpki=False):
         self.acpki = acpki
-        self.rustc_cfg = dict()
+        clk_freq = description["rtio_frequency"]
 
         platform = kasli_soc.Platform()
         platform.toolchain.bitstream_commands.extend([
             "set_property BITSTREAM.GENERAL.COMPRESS True [current_design]",
         ])
-        ident = self.__class__.__name__
+        ident = description["variant"]
         if self.acpki:
             ident = "acpki_" + ident
-        SoCCore.__init__(self, platform=platform, csr_data_width=32, ident=ident)
+        SoCCore.__init__(self, platform=platform, csr_data_width=32, ident=ident, ps_cd_sys=False)
+
+        self.config["HW_REV"] = description["hw_rev"]
         
-        platform.add_platform_command("create_clock -name clk_fpga_0 -period 8 [get_pins \"PS7/FCLKCLK[0]\"]")
-        platform.add_platform_command("set_input_jitter clk_fpga_0 0.24")
 
         self.submodules += SMAClkinForward(self.platform)
 
-        self.rustc_cfg["has_si5324"] = None
-        self.rustc_cfg["si5324_soft_reset"] = None
+        self.config["HAS_SI5324"] = None
+        self.config["SI5324_SOFT_RESET"] = None
 
+        clk_synth = platform.request("cdr_clk_clean_fabric")
+        clk_synth_se = Signal()
+        clk_synth_se_buf = Signal()
+        platform.add_period_constraint(clk_synth.p, 8.0)
+
+        self.specials += [ 
+            Instance("IBUFGDS",
+                p_DIFF_TERM="TRUE", p_IBUF_LOW_PWR="FALSE",
+                i_I=clk_synth.p, i_IB=clk_synth.n, o_O=clk_synth_se
+            ),
+            Instance("BUFG", i_I=clk_synth_se, o_O=clk_synth_se_buf),
+        ]
+        fix_serdes_timing_path(platform)
+        self.submodules.bootstrap = GTPBootstrapClock(self.platform, clk_freq)
+        self.config["CLOCK_FREQUENCY"] = int(clk_freq)
+
+        self.submodules.sys_crg = zynq_clocking.SYSCRG(self.platform, self.ps7, clk_synth_se_buf)
+        platform.add_false_path_constraints(
+            self.bootstrap.cd_bootstrap.clk, self.sys_crg.cd_sys.clk)
+        self.csr_devices.append("sys_crg")
         self.crg = self.ps7 # HACK for eem_7series to find the clock
-        self.submodules.rtio_crg = RTIOCRG(self.platform)
-        self.csr_devices.append("rtio_crg")
-        self.platform.add_period_constraint(self.rtio_crg.cd_rtio.clk, 8.)
-        self.platform.add_false_path_constraints(
-            self.ps7.cd_sys.clk,
-            self.rtio_crg.cd_rtio.clk)
+        self.crg.cd_sys = self.sys_crg.cd_sys
 
         self.rtio_channels = []
         has_grabber = any(peripheral["type"] == "grabber" for peripheral in description["peripherals"])
@@ -153,19 +163,21 @@ class GenericStandalone(SoCCore):
         self.config["RTIO_LOG_CHANNEL"] = len(self.rtio_channels)
         self.rtio_channels.append(rtio.LogChannel())
 
-        self.submodules.rtio_tsc = rtio.TSC("async", glbl_fine_ts_width=3)
-        self.submodules.rtio_core = rtio.Core(self.rtio_tsc, self.rtio_channels)
+        self.submodules.rtio_tsc = rtio.TSC(glbl_fine_ts_width=3)
+        self.submodules.rtio_core = rtio.Core(
+            self.rtio_tsc, self.rtio_channels, lane_count=description["sed_lanes"]
+        )
         self.csr_devices.append("rtio_core")
 
         if self.acpki:
-            self.rustc_cfg["ki_impl"] = "acp"
+            self.config["KI_IMPL"] = "acp"
             self.submodules.rtio = acpki.KernelInitiator(self.rtio_tsc,
                                                          bus=self.ps7.s_axi_acp,
                                                          user=self.ps7.s_axi_acp_user,
                                                          evento=self.ps7.event.o)
             self.csr_devices.append("rtio")
         else:
-            self.rustc_cfg["ki_impl"] = "csr"
+            self.config["KI_IMPL"] = "csr"
             self.submodules.rtio = rtio.KernelInitiator(self.rtio_tsc, now64=True)
             self.csr_devices.append("rtio")
 
@@ -185,52 +197,73 @@ class GenericStandalone(SoCCore):
         self.csr_devices.append("rtio_analyzer")
 
         if has_grabber:
-            self.rustc_cfg["has_grabber"] = None
+            self.config["HAS_GRABBER"] = None
             self.add_csr_group("grabber", self.grabber_csr_group)
             for grabber in self.grabber_csr_group:
                 self.platform.add_false_path_constraints(
-                    self.rtio_crg.cd_rtio.clk, getattr(self, grabber).deserializer.cd_cl.clk)
+                    self.sys_crg.cd_sys.clk, getattr(self, grabber).deserializer.cd_cl.clk)
 
 
 class GenericMaster(SoCCore):
     def __init__(self, description, acpki=False):
-        sys_clk_freq = 125e6
-        rtio_clk_freq = description["rtio_frequency"]
+        clk_freq = description["rtio_frequency"]
 
+        has_drtio_over_eem = any(peripheral["type"] == "shuttler" for peripheral in description["peripherals"])
         self.acpki = acpki
-        self.rustc_cfg = dict()
 
         platform = kasli_soc.Platform()
         platform.toolchain.bitstream_commands.extend([
             "set_property BITSTREAM.GENERAL.COMPRESS True [current_design]",
         ])
-        ident = self.__class__.__name__
+        ident = description["variant"]
         if self.acpki:
             ident = "acpki_" + ident
-        SoCCore.__init__(self, platform=platform, csr_data_width=32, ident=ident)
+        SoCCore.__init__(self, platform=platform, csr_data_width=32, ident=ident, ps_cd_sys=False)
 
-        platform.add_platform_command("create_clock -name clk_fpga_0 -period 8 [get_pins \"PS7/FCLKCLK[0]\"]")
-        platform.add_platform_command("set_input_jitter clk_fpga_0 0.24")
+        self.config["HW_REV"] = description["hw_rev"]
 
         self.submodules += SMAClkinForward(self.platform)
 
         data_pads = [platform.request("sfp", i) for i in range(4)]
 
-        self.submodules.drtio_transceiver = gtx_7series.GTX(
+        self.submodules.gt_drtio = gtx_7series.GTX(
             clock_pads=platform.request("clk_gtp"),
             pads=data_pads,
-            sys_clk_freq=sys_clk_freq)
-        self.csr_devices.append("drtio_transceiver")
+            clk_freq=clk_freq)
+        self.csr_devices.append("gt_drtio")
+        self.config["CLOCK_FREQUENCY"] = int(clk_freq)
 
+        txout_buf = Signal()
+        gtx0 = self.gt_drtio.gtxs[0]
+        self.specials += Instance("BUFG", i_I=gtx0.txoutclk, o_O=txout_buf)
+
+        ext_async_rst = Signal()
+
+        self.submodules.bootstrap = GTPBootstrapClock(self.platform, clk_freq)
+        self.submodules.sys_crg = zynq_clocking.SYSCRG(
+            self.platform,
+            self.ps7,
+            txout_buf,
+            clk_sw=self.gt_drtio.stable_clkin.storage,
+            clk_sw_status=gtx0.tx_init.done,
+            ext_async_rst=ext_async_rst)
+        self.csr_devices.append("sys_crg")
         self.crg = self.ps7 # HACK for eem_7series to find the clock
-        self.submodules.rtio_crg = RTIOClockMultiplier(rtio_clk_freq)
-        self.csr_devices.append("rtio_crg")
+        self.crg.cd_sys = self.sys_crg.cd_sys
+        platform.add_false_path_constraints(
+            self.bootstrap.cd_bootstrap.clk, self.sys_crg.cd_sys.clk)
+        fix_serdes_timing_path(platform)
 
-        self.rustc_cfg["has_si5324"] = None
-        self.rustc_cfg["si5324_soft_reset"] = None
+        self.comb += ext_async_rst.eq(self.sys_crg.clk_sw_fsm.o_clk_sw & ~gtx0.tx_init.done)
+        self.specials += MultiReg(self.sys_crg.clk_sw_fsm.o_clk_sw & self.sys_crg.mmcm_locked, self.gt_drtio.clk_path_ready, odomain="bootstrap")
+
+        self.config["HAS_SI5324"] = None
+        self.config["SI5324_SOFT_RESET"] = None
 
         self.rtio_channels = []
         has_grabber = any(peripheral["type"] == "grabber" for peripheral in description["peripherals"])
+        if has_drtio_over_eem:
+            self.eem_drtio_channels = []
         if has_grabber:
             self.grabber_csr_group = []
         eem_7series.add_peripherals(self, description["peripherals"], iostandard=eem_iostandard)
@@ -243,23 +276,23 @@ class GenericMaster(SoCCore):
         self.config["RTIO_LOG_CHANNEL"] = len(self.rtio_channels)
         self.rtio_channels.append(rtio.LogChannel())
 
-        self.submodules.rtio_tsc = rtio.TSC("async", glbl_fine_ts_width=3)
+        self.submodules.rtio_tsc = rtio.TSC(glbl_fine_ts_width=3)
 
-        drtio_csr_group = []
-        drtioaux_csr_group = []
-        drtioaux_memory_group = []
+        self.drtio_csr_group = []
+        self.drtioaux_csr_group = []
+        self.drtioaux_memory_group = []
         self.drtio_cri = []
-        for i in range(len(self.drtio_transceiver.channels)):
+        for i in range(len(self.gt_drtio.channels)):
             core_name = "drtio" + str(i)
             coreaux_name = "drtioaux" + str(i)
             memory_name = "drtioaux" + str(i) + "_mem"
-            drtio_csr_group.append(core_name)
-            drtioaux_csr_group.append(coreaux_name)
-            drtioaux_memory_group.append(memory_name)
+            self.drtio_csr_group.append(core_name)
+            self.drtioaux_csr_group.append(coreaux_name)
+            self.drtioaux_memory_group.append(memory_name)
 
             cdr = ClockDomainsRenamer({"rtio_rx": "rtio_rx" + str(i)})
 
-            core = cdr(DRTIOMaster(self.rtio_tsc, self.drtio_transceiver.channels[i]))
+            core = cdr(DRTIOMaster(self.rtio_tsc, self.gt_drtio.channels[i]))
             setattr(self.submodules, core_name, core)
             self.drtio_cri.append(core.cri)
             self.csr_devices.append(core_name)
@@ -272,24 +305,27 @@ class GenericMaster(SoCCore):
             memory_address = self.axi2csr.register_port(coreaux.get_tx_port(), size)
             self.axi2csr.register_port(coreaux.get_rx_port(), size)
             self.add_memory_region(memory_name, self.mem_map["csr"] + memory_address, size * 2)
-        self.rustc_cfg["has_drtio"] = None
-        self.rustc_cfg["has_drtio_routing"] = None
-        self.add_csr_group("drtio", drtio_csr_group)
-        self.add_csr_group("drtioaux", drtioaux_csr_group)
-        self.add_memory_group("drtioaux_mem", drtioaux_memory_group)
+        self.config["HAS_DRTIO"] = None
+        self.config["HAS_DRTIO_ROUTING"] = None
 
-        self.submodules.rtio_core = rtio.Core(self.rtio_tsc, self.rtio_channels)
+        if has_drtio_over_eem:
+            self.add_eem_drtio(self.eem_drtio_channels)
+        self.add_drtio_cpuif_groups()
+
+        self.submodules.rtio_core = rtio.Core(
+            self.rtio_tsc, self.rtio_channels, lane_count=description["sed_lanes"]
+        )
         self.csr_devices.append("rtio_core")
 
         if self.acpki:
-            self.rustc_cfg["ki_impl"] = "acp"
+            self.config["KI_IMPL"] = "acp"
             self.submodules.rtio = acpki.KernelInitiator(self.rtio_tsc,
                                                          bus=self.ps7.s_axi_acp,
                                                          user=self.ps7.s_axi_acp_user,
                                                          evento=self.ps7.event.o)
             self.csr_devices.append("rtio")
         else:
-            self.rustc_cfg["ki_impl"] = "csr"
+            self.config["KI_IMPL"] = "csr"
             self.submodules.rtio = rtio.KernelInitiator(self.rtio_tsc, now64=True)
             self.csr_devices.append("rtio")
 
@@ -313,42 +349,103 @@ class GenericMaster(SoCCore):
         self.csr_devices.append("rtio_analyzer")
 
         if has_grabber:
-            self.rustc_cfg["has_grabber"] = None
+            self.config["HAS_GRABBER"] = None
             self.add_csr_group("grabber", self.grabber_csr_group)
         
 
+        self.submodules.virtual_leds = virtual_leds.VirtualLeds()
+        self.csr_devices.append("virtual_leds")
+
+        self.comb += [self.virtual_leds.get(i).eq(channel.rx_ready)
+                for i, channel in enumerate(self.gt_drtio.channels)]
+
+    def add_eem_drtio(self, eem_drtio_channels):
+        # Must be called before invoking add_rtio() to construct the CRI
+        # interconnect properly
+        self.submodules.eem_transceiver = eem_serdes.EEMSerdes(self.platform, eem_drtio_channels)
+        self.csr_devices.append("eem_transceiver")
+        self.config["HAS_DRTIO_EEM"] = None
+        self.config["EEM_DRTIO_COUNT"] = len(eem_drtio_channels)
+
+        cdr = ClockDomainsRenamer({"rtio_rx": "sys"})
+        for i in range(len(self.eem_transceiver.channels)):
+            channel = i + len(self.gt_drtio.channels)
+            core_name = "drtio" + str(channel)
+            coreaux_name = "drtioaux" + str(channel)
+            memory_name = "drtioaux" + str(channel) + "_mem"
+            self.drtio_csr_group.append(core_name)
+            self.drtioaux_csr_group.append(coreaux_name)
+            self.drtioaux_memory_group.append(memory_name)
+
+            core = cdr(DRTIOMaster(self.rtio_tsc, self.eem_transceiver.channels[i]))
+            setattr(self.submodules, core_name, core)
+            self.drtio_cri.append(core.cri)
+            self.csr_devices.append(core_name)
+
+            coreaux = cdr(drtio_aux_controller.DRTIOAuxControllerBare(core.link_layer))
+            setattr(self.submodules, coreaux_name, coreaux)
+            self.csr_devices.append(coreaux_name)
+
+            size = coreaux.get_mem_size()
+            memory_address = self.axi2csr.register_port(coreaux.get_tx_port(), size)
+            self.axi2csr.register_port(coreaux.get_rx_port(), size)
+            self.add_memory_region(memory_name, self.mem_map["csr"] + memory_address, size * 2)
+
+    def add_drtio_cpuif_groups(self):
+        self.add_csr_group("drtio", self.drtio_csr_group)
+        self.add_csr_group("drtioaux", self.drtioaux_csr_group)
+        self.add_memory_group("drtioaux_mem", self.drtioaux_memory_group)
+
+
 class GenericSatellite(SoCCore):
     def __init__(self, description, acpki=False):
-        sys_clk_freq = 125e6  
-        rtio_clk_freq = description["rtio_frequency"]
+        clk_freq = description["rtio_frequency"]
 
         self.acpki = acpki
-        self.rustc_cfg = dict()
 
         platform = kasli_soc.Platform()
         platform.toolchain.bitstream_commands.extend([
             "set_property BITSTREAM.GENERAL.COMPRESS True [current_design]",
         ])
-        ident = self.__class__.__name__
+        ident = description["variant"]
         if self.acpki:
             ident = "acpki_" + ident
-        SoCCore.__init__(self, platform=platform, csr_data_width=32, ident=ident)
+        SoCCore.__init__(self, platform=platform, csr_data_width=32, ident=ident, ps_cd_sys=False)
 
-        platform.add_platform_command("create_clock -name clk_fpga_0 -period 8 [get_pins \"PS7/FCLKCLK[0]\"]")
-        platform.add_platform_command("set_input_jitter clk_fpga_0 0.24")
-
-        self.crg = self.ps7 # HACK for eem_7series to find the clock
-        self.submodules.rtio_crg = RTIOClockMultiplier(rtio_clk_freq)
-        self.csr_devices.append("rtio_crg")
-        self.rustc_cfg["has_rtio_crg"] = None
+        self.config["HW_REV"] = description["hw_rev"]
 
         data_pads = [platform.request("sfp", i) for i in range(4)]
         
-        self.submodules.drtio_transceiver = gtx_7series.GTX(
+        self.submodules.gt_drtio = gtx_7series.GTX(
             clock_pads=platform.request("clk_gtp"),  
             pads=data_pads,
-            sys_clk_freq=sys_clk_freq)
-        self.csr_devices.append("drtio_transceiver")
+            clk_freq=clk_freq)
+        self.csr_devices.append("gt_drtio")
+
+        txout_buf = Signal()
+        gtx0 = self.gt_drtio.gtxs[0]
+        self.specials += Instance("BUFG", i_I=gtx0.txoutclk, o_O=txout_buf)
+
+        ext_async_rst = Signal()
+
+        self.submodules.bootstrap = GTPBootstrapClock(self.platform, clk_freq)
+        self.submodules.sys_crg = zynq_clocking.SYSCRG(
+            self.platform, 
+            self.ps7,
+            txout_buf,
+            clk_sw=self.gt_drtio.stable_clkin.storage,
+            clk_sw_status=gtx0.tx_init.done,
+            ext_async_rst=ext_async_rst)
+        platform.add_false_path_constraints(
+            self.bootstrap.cd_bootstrap.clk, self.sys_crg.cd_sys.clk)
+        self.csr_devices.append("sys_crg")
+        self.crg = self.ps7 # HACK for eem_7series to find the clock
+        self.crg.cd_sys = self.sys_crg.cd_sys
+
+        fix_serdes_timing_path(platform)
+
+        self.comb += ext_async_rst.eq(self.sys_crg.clk_sw_fsm.o_clk_sw & ~gtx0.tx_init.done)
+        self.specials += MultiReg(self.sys_crg.clk_sw_fsm.o_clk_sw & self.sys_crg.mmcm_locked, self.gt_drtio.clk_path_ready, odomain="bootstrap")
 
         self.rtio_channels = []
         has_grabber = any(peripheral["type"] == "grabber" for peripheral in description["peripherals"])
@@ -364,13 +461,13 @@ class GenericSatellite(SoCCore):
         self.config["RTIO_LOG_CHANNEL"] = len(self.rtio_channels)
         self.rtio_channels.append(rtio.LogChannel())
 
-        self.submodules.rtio_tsc = rtio.TSC("sync", glbl_fine_ts_width=3)
+        self.submodules.rtio_tsc = rtio.TSC(glbl_fine_ts_width=3)
 
         drtioaux_csr_group = []
         drtioaux_memory_group = []
         drtiorep_csr_group = []
         self.drtio_cri = []
-        for i in range(len(self.drtio_transceiver.channels)):
+        for i in range(len(self.gt_drtio.channels)):
             coreaux_name = "drtioaux" + str(i)
             memory_name = "drtioaux" + str(i) + "_mem"
             drtioaux_csr_group.append(coreaux_name)
@@ -381,7 +478,7 @@ class GenericSatellite(SoCCore):
             if i == 0:
                 self.submodules.rx_synchronizer = cdr(XilinxRXSynchronizer())
                 core = cdr(DRTIOSatellite(
-                    self.rtio_tsc, self.drtio_transceiver.channels[i],
+                    self.rtio_tsc, self.gt_drtio.channels[i],
                     self.rx_synchronizer))
                 self.submodules.drtiosat = core
                 self.csr_devices.append("drtiosat")
@@ -390,7 +487,7 @@ class GenericSatellite(SoCCore):
                 drtiorep_csr_group.append(corerep_name)
 
                 core = cdr(DRTIORepeater(
-                    self.rtio_tsc, self.drtio_transceiver.channels[i]))
+                    self.rtio_tsc, self.gt_drtio.channels[i]))
                 setattr(self.submodules, corerep_name, core)
                 self.drtio_cri.append(core.cri)
                 self.csr_devices.append(corerep_name)
@@ -408,34 +505,36 @@ class GenericSatellite(SoCCore):
             # and registered in PS interface
             # manually, because software refers to rx/tx by halves of entire memory block, not names
             self.add_memory_region(memory_name, self.mem_map["csr"] + memory_address, mem_size * 2)
-        self.rustc_cfg["has_drtio"] = None
-        self.rustc_cfg["has_drtio_routing"] = None
+        self.config["HAS_DRTIO"] = None
+        self.config["HAS_DRTIO_ROUTING"] = None
         self.add_csr_group("drtioaux", drtioaux_csr_group)
         self.add_memory_group("drtioaux_mem", drtioaux_memory_group)
         self.add_csr_group("drtiorep", drtiorep_csr_group)
 
         if self.acpki:
-            self.rustc_cfg["ki_impl"] = "acp"
+            self.config["KI_IMPL"] = "acp"
             self.submodules.rtio = acpki.KernelInitiator(self.rtio_tsc,
                                                          bus=self.ps7.s_axi_acp,
                                                          user=self.ps7.s_axi_acp_user,
                                                          evento=self.ps7.event.o)
             self.csr_devices.append("rtio")
         else:
-            self.rustc_cfg["ki_impl"] = "csr"
+            self.config["KI_IMPL"] = "csr"
             self.submodules.rtio = rtio.KernelInitiator(self.rtio_tsc, now64=True)
             self.csr_devices.append("rtio")
 
         self.submodules.rtio_dma = dma.DMA(self.ps7.s_axi_hp0)
         self.csr_devices.append("rtio_dma")
 
-        self.submodules.local_io = SyncRTIO(self.rtio_tsc, self.rtio_channels)
+        self.submodules.local_io = SyncRTIO(
+            self.rtio_tsc, self.rtio_channels, lane_count=description["sed_lanes"]
+        )
         self.comb += self.drtiosat.async_errors.eq(self.local_io.async_errors)
 
         self.submodules.cri_con = rtio.CRIInterconnectShared(
-            [self.drtiosat.cri],
+            [self.drtiosat.cri, self.rtio_dma.cri, self.rtio.cri],
             [self.local_io.cri] + self.drtio_cri,
-            mode="sync", enable_routing=True)
+            enable_routing=True)
         self.csr_devices.append("cri_con")
 
         self.submodules.routing_table = rtio.RoutingTableAccess(self.cri_con)
@@ -444,58 +543,37 @@ class GenericSatellite(SoCCore):
         self.submodules.rtio_moninj = rtio.MonInj(self.rtio_channels)
         self.csr_devices.append("rtio_moninj")
 
-        rtio_clk_period = 1e9/rtio_clk_freq
-        self.rustc_cfg["rtio_frequency"] = str(rtio_clk_freq/1e6)
+        self.submodules.rtio_analyzer = analyzer.Analyzer(self.rtio_tsc, self.local_io.cri,
+                                                          self.ps7.s_axi_hp1)
+        self.csr_devices.append("rtio_analyzer")
+
+        rtio_clk_period = 1e9/clk_freq
+        self.config["RTIO_FREQUENCY"] = str(clk_freq/1e6)
+        self.config["CLOCK_FREQUENCY"] = int(clk_freq)
 
         self.submodules.siphaser = SiPhaser7Series(
             si5324_clkin=platform.request("cdr_clk"),
             rx_synchronizer=self.rx_synchronizer,
             ultrascale=False,
-            rtio_clk_freq=self.drtio_transceiver.rtio_clk_freq)
-        platform.add_false_path_constraints(
-            self.crg.cd_sys.clk, self.siphaser.mmcm_freerun_output)
+            rtio_clk_freq=self.gt_drtio.rtio_clk_freq)
         self.csr_devices.append("siphaser")
-        self.rustc_cfg["has_si5324"] = None
-        self.rustc_cfg["has_siphaser"] = None
-        self.rustc_cfg["si5324_soft_reset"] = None
+        self.config["HAS_SI5324"] = None
+        self.config["SI5324_SOFT_RESET"] = None
 
-        gtx0 = self.drtio_transceiver.gtxs[0]
-        platform.add_period_constraint(gtx0.txoutclk, rtio_clk_period)
-        platform.add_period_constraint(gtx0.rxoutclk, rtio_clk_period)
+        gtx0 = self.gt_drtio.gtxs[0]
         platform.add_false_path_constraints(
-            self.crg.cd_sys.clk,
             gtx0.txoutclk, gtx0.rxoutclk)
-        for gtx in self.drtio_transceiver.gtxs[1:]:
-            platform.add_period_constraint(gtx.rxoutclk, rtio_clk_period)
-            platform.add_false_path_constraints(
-                self.crg.cd_sys.clk, gtx.rxoutclk)
 
         if has_grabber:
-            self.rustc_cfg["has_grabber"] = None
+            self.config["HAS_GRABBER"] = None
             self.add_csr_group("grabber", self.grabber_csr_group)
             # no RTIO CRG here
         
+        self.submodules.virtual_leds = virtual_leds.VirtualLeds()
+        self.csr_devices.append("virtual_leds")
 
-def write_mem_file(soc, filename):
-    with open(filename, "w") as f:
-        f.write(cpu_interface.get_mem_rust(
-            soc.get_memory_regions(), soc.get_memory_groups(), None))
-
-
-def write_csr_file(soc, filename):
-    with open(filename, "w") as f:
-        f.write(cpu_interface.get_csr_rust(
-            soc.get_csr_regions(), soc.get_csr_groups(), soc.get_constants()))
-
-
-def write_rustc_cfg_file(soc, filename):
-    with open(filename, "w") as f:
-        for k, v in sorted(soc.rustc_cfg.items(), key=itemgetter(0)):
-            if v is None:
-                f.write("{}\n".format(k))
-            else:
-                f.write("{}=\"{}\"\n".format(k, v))
-
+        self.comb += [self.virtual_leds.get(i).eq(channel.rx_ready)
+                for i, channel in enumerate(self.gt_drtio.channels)]
 
 def main():
     parser = argparse.ArgumentParser(
@@ -518,14 +596,14 @@ def main():
     if description["target"] != "kasli_soc":
         raise ValueError("Description is for a different target")
 
-    if description["base"] == "standalone":
+    if description["drtio_role"] == "standalone":
         cls = GenericStandalone
-    elif description["base"] == "master":
+    elif description["drtio_role"] == "master":
         cls = GenericMaster
-    elif description["base"] == "satellite":
+    elif description["drtio_role"] == "satellite":
         cls = GenericSatellite
     else:
-        raise ValueError("Invalid base")
+        raise ValueError("Invalid DRTIO role")
 
     soc = cls(description, acpki=args.acpki)
     soc.finalize()

src/gateware/test_dma.py

@@ -168,7 +168,7 @@ class FullStackTB(Module):
         bus = axi.Interface(ws*8)
         self.memory = AXIMemorySim(bus, sequence)
         self.submodules.dut = dma.DMA(bus)
-        self.submodules.tsc = rtio.TSC("async")
+        self.submodules.tsc = rtio.TSC()
         self.submodules.rtio = rtio.Core(self.tsc, rtio_channels)
         self.comb += self.dut.cri.connect(self.rtio.cri)
 
@@ -229,7 +229,7 @@ class TestDMA(unittest.TestCase):
             do_dma(tb.dut, 0), monitor(),
             (None for _ in range(70)),
             tb.memory.ar(), tb.memory.r()
-        ]}, {"sys": 8, "rsys": 8, "rtio": 8, "rio": 8, "rio_phy": 8})
+        ]}, {"sys": 8, "rsys": 8, "rio": 8, "rio_phy": 8})
 
         correct_changes = [(timestamp + 11, channel)
                            for channel, timestamp, _, _ in test_writes_full_stack]

src/gateware/zc706.py

@@ -10,12 +10,11 @@ from migen.genlib.cdc import MultiReg
 from migen_axi.integration.soc_core import SoCCore
 from migen_axi.platforms import zc706
 from misoc.interconnect.csr import *
-from misoc.integration import cpu_interface
 from misoc.cores import gpio
 
 from artiq.gateware import rtio, nist_clock, nist_qc2
-from artiq.gateware.rtio.phy import ttl_simple, ttl_serdes_7series, dds, spi2
-from artiq.gateware.rtio.xilinx_clocking import RTIOClockMultiplier, fix_serdes_timing_path
+from artiq.gateware.rtio.phy import ttl_simple, ttl_serdes_7series, dds, spi2, edge_counter
+from artiq.gateware.rtio.xilinx_clocking import fix_serdes_timing_path
 from artiq.gateware.drtio.transceiver import gtx_7series
 from artiq.gateware.drtio.siphaser import SiPhaser7Series
 from artiq.gateware.drtio.rx_synchronizer import XilinxRXSynchronizer
@@ -25,51 +24,8 @@ import dma
 import analyzer
 import acpki
 import drtio_aux_controller
-
-
-class RTIOCRG(Module, AutoCSR):
-    def __init__(self, platform, rtio_internal_clk):
-        self.clock_sel = CSRStorage()
-        self.pll_reset = CSRStorage(reset=1)
-        self.pll_locked = CSRStatus()
-        self.clock_domains.cd_rtio = ClockDomain()
-        self.clock_domains.cd_rtiox4 = ClockDomain(reset_less=True)
-
-        rtio_external_clk = Signal()
-        user_sma_clock = platform.request("user_sma_clock")
-        platform.add_period_constraint(user_sma_clock.p, 8.0)
-        self.specials += Instance("IBUFDS",
-                                  i_I=user_sma_clock.p, i_IB=user_sma_clock.n,
-                                  o_O=rtio_external_clk)
-
-        pll_locked = Signal()
-        rtio_clk = Signal()
-        rtiox4_clk = Signal()
-        self.specials += [
-            Instance("PLLE2_ADV",
-                     p_STARTUP_WAIT="FALSE", o_LOCKED=pll_locked,
-
-                     p_REF_JITTER1=0.01,
-                     p_CLKIN1_PERIOD=8.0, p_CLKIN2_PERIOD=8.0,
-                     i_CLKIN1=rtio_internal_clk, i_CLKIN2=rtio_external_clk,
-                     # Warning: CLKINSEL=0 means CLKIN2 is selected
-                     i_CLKINSEL=~self.clock_sel.storage,
-
-                     # VCO @ 1GHz when using 125MHz input
-                     p_CLKFBOUT_MULT=8, p_DIVCLK_DIVIDE=1,
-                     i_CLKFBIN=self.cd_rtio.clk,
-                     i_RST=self.pll_reset.storage,
-
-                     o_CLKFBOUT=rtio_clk,
-
-                     p_CLKOUT0_DIVIDE=2, p_CLKOUT0_PHASE=0.0,
-                     o_CLKOUT0=rtiox4_clk),
-            Instance("BUFG", i_I=rtio_clk, o_O=self.cd_rtio.clk),
-            Instance("BUFG", i_I=rtiox4_clk, o_O=self.cd_rtiox4.clk),
-            AsyncResetSynchronizer(self.cd_rtio, ~pll_locked),
-            MultiReg(pll_locked, self.pll_locked.status)
-        ]
-
+import zynq_clocking
+from config import write_csr_file, write_mem_file, write_rustc_cfg_file
 
 class SMAClkinForward(Module):
     def __init__(self, platform):
@@ -84,6 +40,37 @@ class SMAClkinForward(Module):
         ]
 
 
+class CLK200BootstrapClock(Module):
+    def __init__(self, platform, freq=125e6):
+        self.clock_domains.cd_bootstrap = ClockDomain(reset_less=True)
+        self.cd_bootstrap.clk.attr.add("keep")
+
+        clk200 = platform.request("clk200")
+        clk200_se = Signal()
+
+        pll_fb = Signal()
+        pll_clkout = Signal()
+        assert freq in [125e6, 100e6]
+        divide = int(1e9/freq)
+        self.specials += [
+            Instance("IBUFDS",
+                i_I=clk200.p, i_IB=clk200.n, o_O=clk200_se),
+            Instance("PLLE2_BASE",
+                p_CLKIN1_PERIOD=5.0,
+                i_CLKIN1=clk200_se,
+                i_CLKFBIN=pll_fb,
+                o_CLKFBOUT=pll_fb,
+
+                # VCO @ 1GHz
+                p_CLKFBOUT_MULT=5, p_DIVCLK_DIVIDE=1,
+
+                # 125MHz/100MHz for bootstrap
+                p_CLKOUT1_DIVIDE=divide, p_CLKOUT1_PHASE=0.0, o_CLKOUT1=pll_clkout,
+            ),
+            Instance("BUFG", i_I=pll_clkout, o_O=self.cd_bootstrap.clk)
+        ]
+
+
 # The NIST backplanes require setting VADJ to 3.3V by reprogramming the power supply.
 # This also changes the I/O standard for some on-board LEDs.
 leds_fmc33 = [
@@ -135,14 +122,10 @@ def prepare_zc706_platform(platform):
     platform.toolchain.bitstream_commands.extend([
         "set_property BITSTREAM.GENERAL.COMPRESS True [current_design]",
     ])
-    platform.add_platform_command("create_clock -name clk_fpga_0 -period 8 [get_pins \"PS7/FCLKCLK[0]\"]")
-    platform.add_platform_command("set_input_jitter clk_fpga_0 0.24")
-
 
 class ZC706(SoCCore):
     def __init__(self, acpki=False):
         self.acpki = acpki
-        self.rustc_cfg = dict()
 
         platform = zc706.Platform()
         prepare_zc706_platform(platform)
@@ -150,30 +133,49 @@ class ZC706(SoCCore):
         ident = self.__class__.__name__
         if self.acpki:
             ident = "acpki_" + ident
-        SoCCore.__init__(self, platform=platform, csr_data_width=32, ident=ident)
+        SoCCore.__init__(self, platform=platform, csr_data_width=32, ident=ident, ps_cd_sys=False)
 
-        self.submodules.rtio_crg = RTIOCRG(self.platform, self.ps7.cd_sys.clk)
-        self.csr_devices.append("rtio_crg")
-        self.rustc_cfg["has_rtio_crg_clock_sel"] = None
-        self.platform.add_period_constraint(self.rtio_crg.cd_rtio.clk, 8.)
-        self.platform.add_false_path_constraints(
-            self.ps7.cd_sys.clk,
-            self.rtio_crg.cd_rtio.clk)
+        platform.add_extension(si5324_fmc33)
+        self.comb += platform.request("si5324_33").rst_n.eq(1)
+
+        cdr_clk = Signal()
+        cdr_clk_buf = Signal()
+        si5324_out = platform.request("si5324_clkout")
+        platform.add_period_constraint(si5324_out.p, 8.0)
+        self.specials += [
+            Instance("IBUFDS_GTE2",
+                i_CEB=0,
+                i_I=si5324_out.p, i_IB=si5324_out.n,
+                o_O=cdr_clk,
+                p_CLKCM_CFG="TRUE",
+                p_CLKRCV_TRST="TRUE",
+                p_CLKSWING_CFG=3),
+            Instance("BUFG", i_I=cdr_clk, o_O=cdr_clk_buf)
+        ]
+        self.config["HAS_SI5324"] = None
+        self.config["SI5324_AS_SYNTHESIZER"] = None
+        self.config["SI5324_SOFT_RESET"] = None
+        
+        self.submodules.bootstrap = CLK200BootstrapClock(platform)
+        self.submodules.sys_crg = zynq_clocking.SYSCRG(self.platform, self.ps7, cdr_clk_buf)
+        platform.add_false_path_constraints(
+            self.bootstrap.cd_bootstrap.clk, self.sys_crg.cd_sys.clk)
+        self.csr_devices.append("sys_crg")
 
     def add_rtio(self, rtio_channels):
-        self.submodules.rtio_tsc = rtio.TSC("async", glbl_fine_ts_width=3)
+        self.submodules.rtio_tsc = rtio.TSC(glbl_fine_ts_width=3)
         self.submodules.rtio_core = rtio.Core(self.rtio_tsc, rtio_channels)
         self.csr_devices.append("rtio_core")
 
         if self.acpki:
-            self.rustc_cfg["ki_impl"] = "acp"
+            self.config["KI_IMPL"] = "acp"
             self.submodules.rtio = acpki.KernelInitiator(self.rtio_tsc,
                                                          bus=self.ps7.s_axi_acp,
                                                          user=self.ps7.s_axi_acp_user,
                                                          evento=self.ps7.event.o)
             self.csr_devices.append("rtio")
         else:
-            self.rustc_cfg["ki_impl"] = "csr"
+            self.config["KI_IMPL"] = "csr"
             self.submodules.rtio = rtio.KernelInitiator(self.rtio_tsc, now64=True)
             self.csr_devices.append("rtio")
 
@@ -196,22 +198,18 @@ class ZC706(SoCCore):
 class _MasterBase(SoCCore):
     def __init__(self, acpki=False, drtio100mhz=False):
         self.acpki = acpki
-        self.rustc_cfg = dict()
+
+        clk_freq = 100e6 if drtio100mhz else 125e6
 
         platform = zc706.Platform()
         prepare_zc706_platform(platform)
         ident = self.__class__.__name__
         if self.acpki:
             ident = "acpki_" + ident
-        SoCCore.__init__(self, platform=platform, csr_data_width=32, ident=ident)
+        SoCCore.__init__(self, platform=platform, csr_data_width=32, ident=ident, ps_cd_sys=False)
 
         platform.add_extension(si5324_fmc33)
 
-        self.sys_clk_freq = 125e6
-        rtio_clk_freq = 100e6 if drtio100mhz else self.sys_clk_freq
-
-        platform = self.platform
-
         self.comb += platform.request("sfp_tx_disable_n").eq(1)
         data_pads = [
             platform.request("sfp"),
@@ -221,20 +219,38 @@ class _MasterBase(SoCCore):
         self.submodules += SMAClkinForward(self.platform)
 
         # 1000BASE_BX10 Ethernet compatible, 125MHz RTIO clock
-        self.submodules.drtio_transceiver = gtx_7series.GTX(
+        self.submodules.gt_drtio = gtx_7series.GTX(
             clock_pads=platform.request("si5324_clkout"),
             pads=data_pads,
-            sys_clk_freq=self.sys_clk_freq,
-            rtio_clk_freq=rtio_clk_freq)
-        self.csr_devices.append("drtio_transceiver")
+            clk_freq=clk_freq)
+        self.csr_devices.append("gt_drtio")
 
-        self.submodules.rtio_tsc = rtio.TSC("async", glbl_fine_ts_width=3)
+        self.submodules.rtio_tsc = rtio.TSC(glbl_fine_ts_width=3)
+        ext_async_rst = Signal()
+        txout_buf = Signal()
+        gtx0 = self.gt_drtio.gtxs[0]
+        self.specials += Instance("BUFG", i_I=gtx0.txoutclk, o_O=txout_buf)
+        self.submodules.bootstrap = CLK200BootstrapClock(platform, clk_freq)
+        self.submodules.sys_crg = zynq_clocking.SYSCRG(
+            self.platform, 
+            self.ps7, 
+            txout_buf,
+            clk_sw=self.gt_drtio.stable_clkin.storage,
+            clk_sw_status=gtx0.tx_init.done,
+            ext_async_rst=ext_async_rst,
+            freq=clk_freq)
+        platform.add_false_path_constraints(
+            self.bootstrap.cd_bootstrap.clk, self.sys_crg.cd_sys.clk)
+        self.csr_devices.append("sys_crg")
+
+        self.comb += ext_async_rst.eq(self.sys_crg.clk_sw_fsm.o_clk_sw & ~gtx0.tx_init.done)
+        self.specials += MultiReg(self.sys_crg.clk_sw_fsm.o_clk_sw & self.sys_crg.mmcm_locked, self.gt_drtio.clk_path_ready, odomain="bootstrap")
 
         drtio_csr_group = []
         drtioaux_csr_group = []
         drtioaux_memory_group = []
         self.drtio_cri = []
-        for i in range(len(self.drtio_transceiver.channels)):
+        for i in range(len(self.gt_drtio.channels)):
             core_name = "drtio" + str(i)
             coreaux_name = "drtioaux" + str(i)
             memory_name = "drtioaux" + str(i) + "_mem"
@@ -245,7 +261,7 @@ class _MasterBase(SoCCore):
             cdr = ClockDomainsRenamer({"rtio_rx": "rtio_rx" + str(i)})
 
             core = cdr(DRTIOMaster(
-                self.rtio_tsc, self.drtio_transceiver.channels[i]))
+                self.rtio_tsc, self.gt_drtio.channels[i]))
             setattr(self.submodules, core_name, core)
             self.drtio_cri.append(core.cri)
             self.csr_devices.append(core_name)
@@ -258,53 +274,45 @@ class _MasterBase(SoCCore):
             memory_address = self.axi2csr.register_port(coreaux.get_tx_port(), mem_size)
             self.axi2csr.register_port(coreaux.get_rx_port(), mem_size)
             self.add_memory_region(memory_name, self.mem_map["csr"] + memory_address, mem_size * 2)
-        self.rustc_cfg["has_drtio"] = None
-        self.rustc_cfg["has_drtio_routing"] = None
+        self.config["HAS_DRTIO"] = None
+        self.config["HAS_DRTIO_ROUTING"] = None
         self.add_csr_group("drtio", drtio_csr_group)
         self.add_csr_group("drtioaux", drtioaux_csr_group)
         self.add_memory_group("drtioaux_mem", drtioaux_memory_group)
 
-        self.rustc_cfg["rtio_frequency"] = str(self.drtio_transceiver.rtio_clk_freq/1e6)
+        self.config["RTIO_FREQUENCY"] = str(self.gt_drtio.rtio_clk_freq/1e6)
 
         self.submodules.si5324_rst_n = gpio.GPIOOut(platform.request("si5324_33").rst_n)
         self.csr_devices.append("si5324_rst_n")
-        self.rustc_cfg["has_si5324"] = None
-        self.rustc_cfg["si5324_as_synthesizer"] = None
+        self.config["HAS_SI5324"] = None
+        self.config["SI5324_AS_SYNTHESIZER"] = None
 
-        rtio_clk_period = 1e9/self.drtio_transceiver.rtio_clk_freq
         # Constrain TX & RX timing for the first transceiver channel
         # (First channel acts as master for phase alignment for all channels' TX)
-        gtx0 = self.drtio_transceiver.gtxs[0]
-        platform.add_period_constraint(gtx0.txoutclk, rtio_clk_period)
-        platform.add_period_constraint(gtx0.rxoutclk, rtio_clk_period)
         platform.add_false_path_constraints(
-            self.ps7.cd_sys.clk,
             gtx0.txoutclk, gtx0.rxoutclk)
         # Constrain RX timing for the each transceiver channel
         # (Each channel performs single-lane phase alignment for RX)
-        for gtx in self.drtio_transceiver.gtxs[1:]:
-            platform.add_period_constraint(gtx.rxoutclk, rtio_clk_period)
+        for gtx in self.gt_drtio.gtxs[1:]:
             platform.add_false_path_constraints(
-                self.ps7.cd_sys.clk, gtx0.txoutclk, gtx.rxoutclk)
+                gtx0.txoutclk, gtx.rxoutclk)
 
-        self.submodules.rtio_crg = RTIOClockMultiplier(self.sys_clk_freq)
-        self.csr_devices.append("rtio_crg")
-        fix_serdes_timing_path(self.platform)
+        fix_serdes_timing_path(platform)
 
     def add_rtio(self, rtio_channels):
-        self.submodules.rtio_tsc = rtio.TSC("async", glbl_fine_ts_width=3)
+        self.submodules.rtio_tsc = rtio.TSC(glbl_fine_ts_width=3)
         self.submodules.rtio_core = rtio.Core(self.rtio_tsc, rtio_channels)
         self.csr_devices.append("rtio_core")
 
         if self.acpki:
-            self.rustc_cfg["ki_impl"] = "acp"
+            self.config["KI_IMPL"] = "acp"
             self.submodules.rtio = acpki.KernelInitiator(self.rtio_tsc,
                                                          bus=self.ps7.s_axi_acp,
                                                          user=self.ps7.s_axi_acp_user,
                                                          evento=self.ps7.event.o)
             self.csr_devices.append("rtio")
         else:
-            self.rustc_cfg["ki_impl"] = "csr"
+            self.config["KI_IMPL"] = "csr"
             self.submodules.rtio = rtio.KernelInitiator(self.rtio_tsc, now64=True)
             self.csr_devices.append("rtio")
 
@@ -314,7 +322,7 @@ class _MasterBase(SoCCore):
         self.submodules.cri_con = rtio.CRIInterconnectShared(
             [self.rtio.cri, self.rtio_dma.cri],
             [self.rtio_core.cri] + self.drtio_cri,
-            mode="sync", enable_routing=True)
+            enable_routing=True)
         self.csr_devices.append("cri_con")
 
         self.submodules.rtio_moninj = rtio.MonInj(rtio_channels)
@@ -331,21 +339,18 @@ class _MasterBase(SoCCore):
 class _SatelliteBase(SoCCore):
     def __init__(self, acpki=False, drtio100mhz=False):
         self.acpki = acpki
-        self.rustc_cfg = dict()
+
+        clk_freq = 100e6 if drtio100mhz else 125e6
 
         platform = zc706.Platform()
         prepare_zc706_platform(platform)
         ident = self.__class__.__name__
         if self.acpki:
             ident = "acpki_" + ident
-        SoCCore.__init__(self, platform=platform, csr_data_width=32, ident=ident)
+        SoCCore.__init__(self, platform=platform, csr_data_width=32, ident=ident, ps_cd_sys=False)
 
         platform.add_extension(si5324_fmc33)
 
-        self.sys_clk_freq = 125e6
-        rtio_clk_freq = 100e6 if drtio100mhz else self.sys_clk_freq
-        platform = self.platform
-
         # SFP
         self.comb += platform.request("sfp_tx_disable_n").eq(0)
         data_pads = [
@@ -353,21 +358,44 @@ class _SatelliteBase(SoCCore):
             platform.request("user_sma_mgt")
         ]
 
-        self.submodules.rtio_tsc = rtio.TSC("sync", glbl_fine_ts_width=3)
+        self.submodules.rtio_tsc = rtio.TSC(glbl_fine_ts_width=3)
 
         # 1000BASE_BX10 Ethernet compatible, 125MHz RTIO clock
-        self.submodules.drtio_transceiver = gtx_7series.GTX(
+        self.submodules.gt_drtio = gtx_7series.GTX(
             clock_pads=platform.request("si5324_clkout"),
             pads=data_pads,
-            sys_clk_freq=self.sys_clk_freq,
-            rtio_clk_freq=rtio_clk_freq)
-        self.csr_devices.append("drtio_transceiver")
+            clk_freq=clk_freq)
+        self.csr_devices.append("gt_drtio")
+
+        ext_async_rst = Signal()
+        txout_buf = Signal()
+        txout_buf.attr.add("keep")
+        gtx0 = self.gt_drtio.gtxs[0]
+        self.specials += Instance(
+            "BUFG", 
+            i_I=gtx0.txoutclk, 
+            o_O=txout_buf)
+        self.submodules.bootstrap = CLK200BootstrapClock(platform, clk_freq)
+        self.submodules.sys_crg = zynq_clocking.SYSCRG(
+            self.platform, 
+            self.ps7, 
+            txout_buf,
+            clk_sw=self.gt_drtio.stable_clkin.storage,
+            clk_sw_status=gtx0.tx_init.done,
+            ext_async_rst=ext_async_rst,
+            freq=clk_freq)
+        platform.add_false_path_constraints(
+            self.bootstrap.cd_bootstrap.clk, self.sys_crg.cd_sys.clk)
+        self.csr_devices.append("sys_crg")
+
+        self.comb += ext_async_rst.eq(self.sys_crg.clk_sw_fsm.o_clk_sw & ~gtx0.tx_init.done)
+        self.specials += MultiReg(self.sys_crg.clk_sw_fsm.o_clk_sw & self.sys_crg.mmcm_locked, self.gt_drtio.clk_path_ready, odomain="bootstrap")
 
         drtioaux_csr_group = []
         drtioaux_memory_group = []
         drtiorep_csr_group = []
         self.drtio_cri = []
-        for i in range(len(self.drtio_transceiver.channels)):
+        for i in range(len(self.gt_drtio.channels)):
             coreaux_name = "drtioaux" + str(i)
             memory_name = "drtioaux" + str(i) + "_mem"
             drtioaux_csr_group.append(coreaux_name)
@@ -379,7 +407,7 @@ class _SatelliteBase(SoCCore):
             if i == 0:
                 self.submodules.rx_synchronizer = cdr(XilinxRXSynchronizer())
                 core = cdr(DRTIOSatellite(
-                    self.rtio_tsc, self.drtio_transceiver.channels[0], self.rx_synchronizer))
+                    self.rtio_tsc, self.gt_drtio.channels[0], self.rx_synchronizer))
                 self.submodules.drtiosat = core
                 self.csr_devices.append("drtiosat")
             # Repeaters
@@ -387,7 +415,7 @@ class _SatelliteBase(SoCCore):
                 corerep_name = "drtiorep" + str(i-1)
                 drtiorep_csr_group.append(corerep_name)
                 core = cdr(DRTIORepeater(
-                    self.rtio_tsc, self.drtio_transceiver.channels[i]))
+                    self.rtio_tsc, self.gt_drtio.channels[i]))
                 setattr(self.submodules, corerep_name, core)
                 self.drtio_cri.append(core.cri)
                 self.csr_devices.append(corerep_name)
@@ -405,72 +433,70 @@ class _SatelliteBase(SoCCore):
             # and registered in PS interface
             # manually, because software refers to rx/tx by halves of entire memory block, not names
             self.add_memory_region(memory_name, self.mem_map["csr"] + memory_address, mem_size * 2)
-        self.rustc_cfg["has_drtio"] = None
-        self.rustc_cfg["has_drtio_routing"] = None
+        self.config["HAS_DRTIO"] = None
+        self.config["HAS_DRTIO_ROUTING"] = None
         self.add_csr_group("drtioaux", drtioaux_csr_group)
         self.add_csr_group("drtiorep", drtiorep_csr_group)
         self.add_memory_group("drtioaux_mem", drtioaux_memory_group)
 
-        self.rustc_cfg["rtio_frequency"] = str(self.drtio_transceiver.rtio_clk_freq/1e6)
+        self.config["RTIO_FREQUENCY"] = str(self.gt_drtio.rtio_clk_freq/1e6)
 
         # Si5324 Phaser
         self.submodules.siphaser = SiPhaser7Series(
             si5324_clkin=platform.request("si5324_clkin"),
             rx_synchronizer=self.rx_synchronizer,
             ultrascale=False,
-            rtio_clk_freq=self.drtio_transceiver.rtio_clk_freq)
+            rtio_clk_freq=self.gt_drtio.rtio_clk_freq)
         platform.add_false_path_constraints(
-            self.ps7.cd_sys.clk, self.siphaser.mmcm_freerun_output)
+            self.sys_crg.cd_sys.clk, self.siphaser.mmcm_freerun_output)
         self.csr_devices.append("siphaser")
         self.submodules.si5324_rst_n = gpio.GPIOOut(platform.request("si5324_33").rst_n)
         self.csr_devices.append("si5324_rst_n")
-        self.rustc_cfg["has_si5324"] = None
-        self.rustc_cfg["has_siphaser"] = None
+        self.config["HAS_SI5324"] = None
 
-        rtio_clk_period = 1e9/self.drtio_transceiver.rtio_clk_freq
+        rtio_clk_period = 1e9/self.gt_drtio.rtio_clk_freq
         # Constrain TX & RX timing for the first transceiver channel
         # (First channel acts as master for phase alignment for all channels' TX)
-        gtx0 = self.drtio_transceiver.gtxs[0]
-        platform.add_period_constraint(gtx0.txoutclk, rtio_clk_period)
-        platform.add_period_constraint(gtx0.rxoutclk, rtio_clk_period)
         platform.add_false_path_constraints(
-            self.ps7.cd_sys.clk,
             gtx0.txoutclk, gtx0.rxoutclk)
         # Constrain RX timing for the each transceiver channel
         # (Each channel performs single-lane phase alignment for RX)
-        for gtx in self.drtio_transceiver.gtxs[1:]:
-            platform.add_period_constraint(gtx.rxoutclk, rtio_clk_period)
+        for gtx in self.gt_drtio.gtxs[1:]:
             platform.add_false_path_constraints(
-                self.ps7.cd_sys.clk, gtx.rxoutclk)
+                self.sys_crg.cd_sys.clk, gtx.rxoutclk)
 
-        self.submodules.rtio_crg = RTIOClockMultiplier(self.sys_clk_freq)
-        self.csr_devices.append("rtio_crg")
-        self.rustc_cfg["has_rtio_crg"] = None
-        fix_serdes_timing_path(self.platform)
+        fix_serdes_timing_path(platform)
 
     def add_rtio(self, rtio_channels):
         self.submodules.rtio_moninj = rtio.MonInj(rtio_channels)
         self.csr_devices.append("rtio_moninj")
 
         if self.acpki:
-            self.rustc_cfg["ki_impl"] = "acp"
+            self.config["KI_IMPL"] = "acp"
             self.submodules.rtio = acpki.KernelInitiator(self.rtio_tsc,
                                                          bus=self.ps7.s_axi_acp,
                                                          user=self.ps7.s_axi_acp_user,
                                                          evento=self.ps7.event.o)
             self.csr_devices.append("rtio")
         else:
-            self.rustc_cfg["ki_impl"] = "csr"
+            self.config["KI_IMPL"] = "csr"
             self.submodules.rtio = rtio.KernelInitiator(self.rtio_tsc, now64=True)
             self.csr_devices.append("rtio")
 
+        self.submodules.rtio_dma = dma.DMA(self.ps7.s_axi_hp0)
+        self.csr_devices.append("rtio_dma")
+
         self.submodules.local_io = SyncRTIO(self.rtio_tsc, rtio_channels)
         self.submodules.cri_con = rtio.CRIInterconnectShared(
-            [self.drtiosat.cri],
+            [self.drtiosat.cri, self.rtio_dma.cri, self.rtio.cri],
             [self.local_io.cri] + self.drtio_cri,
-            mode="sync", enable_routing=True)
+            enable_routing=True)
         self.csr_devices.append("cri_con")
 
+        self.submodules.rtio_analyzer = analyzer.Analyzer(self.rtio_tsc, self.local_io.cri,
+                                                          self.ps7.s_axi_hp1)
+        self.csr_devices.append("rtio_analyzer")
+
         self.submodules.routing_table = rtio.RoutingTableAccess(self.cri_con)
         self.csr_devices.append("routing_table")
 
@@ -562,12 +588,16 @@ class _NIST_QC2_RTIO:
         platform.add_extension(pmod1_33)
 
         rtio_channels = []
+        edge_counter_phy = []
 
         # All TTL channels are In+Out capable
         for i in range(40):
             phy = ttl_serdes_7series.InOut_8X(platform.request("ttl", i))
             self.submodules += phy
             rtio_channels.append(rtio.Channel.from_phy(phy, ififo_depth=512))
+            # first four TTLs will also have edge counters
+            if i < 4:
+                edge_counter_phy.append(phy)
 
         # no SMA GPIO, replaced with PMOD1_0
         phy = ttl_serdes_7series.InOut_8X(platform.request("pmod1_33", 0))
@@ -607,6 +637,11 @@ class _NIST_QC2_RTIO:
             self.submodules += phy
             rtio_channels.append(rtio.Channel.from_phy(phy, ififo_depth=4))
         
+        for phy in edge_counter_phy:
+            counter = edge_counter.SimpleEdgeCounter(phy.input_state)
+            self.submodules += counter
+            rtio_channels.append(rtio.Channel.from_phy(counter))
+
         self.config["RTIO_LOG_CHANNEL"] = len(rtio_channels)
         rtio_channels.append(rtio.LogChannel())
 
@@ -616,6 +651,7 @@ class _NIST_QC2_RTIO:
 class NIST_CLOCK(ZC706, _NIST_CLOCK_RTIO):
     def __init__(self, acpki, drtio100mhz):
         ZC706.__init__(self, acpki)
+        self.submodules += SMAClkinForward(self.platform)
         _NIST_CLOCK_RTIO.__init__(self)
 
 class NIST_CLOCK_Master(_MasterBase, _NIST_CLOCK_RTIO):
@@ -631,6 +667,7 @@ class NIST_CLOCK_Satellite(_SatelliteBase, _NIST_CLOCK_RTIO):
 class NIST_QC2(ZC706, _NIST_QC2_RTIO):
     def __init__(self, acpki, drtio100mhz):
         ZC706.__init__(self, acpki)
+        self.submodules += SMAClkinForward(self.platform)
         _NIST_QC2_RTIO.__init__(self)
 
 class NIST_QC2_Master(_MasterBase, _NIST_QC2_RTIO):
@@ -646,27 +683,6 @@ class NIST_QC2_Satellite(_SatelliteBase, _NIST_QC2_RTIO):
 VARIANTS = {cls.__name__.lower(): cls for cls in [NIST_CLOCK, NIST_CLOCK_Master, NIST_CLOCK_Satellite,
                                                   NIST_QC2, NIST_QC2_Master, NIST_QC2_Satellite]}
 
-
-def write_csr_file(soc, filename):
-    with open(filename, "w") as f:
-        f.write(cpu_interface.get_csr_rust(
-            soc.get_csr_regions(), soc.get_csr_groups(), soc.get_constants()))
-
-def write_mem_file(soc, filename):
-    with open(filename, "w") as f:
-        f.write(cpu_interface.get_mem_rust(
-            soc.get_memory_regions(), soc.get_memory_groups(), None))
-
-
-def write_rustc_cfg_file(soc, filename):
-    with open(filename, "w") as f:
-        for k, v in sorted(soc.rustc_cfg.items(), key=itemgetter(0)):
-            if v is None:
-                f.write("{}\n".format(k))
-            else:
-                f.write("{}=\"{}\"\n".format(k, v))
-
-
 def main():
     parser = argparse.ArgumentParser(
         description="ARTIQ port to the ZC706 Zynq development kit")

src/gateware/zynq_clocking.py

@@ -0,0 +1,154 @@
+from migen import *
+from migen.genlib.cdc import MultiReg
+from migen.genlib.resetsync import AsyncResetSynchronizer
+from misoc.interconnect.csr import *
+
+
+class ClockSwitchFSM(Module):
+    def __init__(self):
+        self.i_clk_sw = Signal()
+
+        self.o_clk_sw = Signal()
+        self.o_reset = Signal()
+
+        ###
+
+        i_switch = Signal()
+        o_switch = Signal()
+        reset = Signal()
+
+        # at 125MHz bootstrap cd, will get around 0.5ms
+        delay_counter = Signal(16, reset=0xFFFF)
+
+        # register to prevent glitches
+        self.sync.bootstrap += [
+            self.o_clk_sw.eq(o_switch),
+            self.o_reset.eq(reset),
+        ]
+
+        self.o_clk_sw.attr.add("no_retiming")
+        self.o_reset.attr.add("no_retiming")
+        self.i_clk_sw.attr.add("no_retiming")
+        i_switch.attr.add("no_retiming")
+
+        self.specials += MultiReg(self.i_clk_sw, i_switch, "bootstrap")
+
+        fsm = ClockDomainsRenamer("bootstrap")(FSM(reset_state="START"))
+
+        self.submodules += fsm
+
+        fsm.act("START",
+            If(i_switch & ~o_switch,
+                NextState("RESET_START"))
+        )
+        
+        fsm.act("RESET_START",
+            reset.eq(1),
+            If(delay_counter == 0,
+                NextValue(delay_counter, 0xFFFF),
+                NextState("CLOCK_SWITCH")
+            ).Else(
+                NextValue(delay_counter, delay_counter-1),
+            )
+        )
+
+        fsm.act("CLOCK_SWITCH",
+            reset.eq(1),
+            NextValue(o_switch, 1),
+            NextValue(delay_counter, delay_counter-1),
+            If(delay_counter == 0,
+                NextState("END"))
+        )
+        fsm.act("END",
+            NextValue(o_switch, 1),
+            reset.eq(0))
+
+
+class SYSCRG(Module, AutoCSR):
+    def __init__(self, platform, ps7, main_clk, clk_sw=None, clk_sw_status=None, freq=125e6, ext_async_rst=None, ):
+        # assumes bootstrap clock is same freq as main and sys output
+        self.clock_domains.cd_sys = ClockDomain()
+        self.clock_domains.cd_sys4x = ClockDomain(reset_less=True)
+        self.clock_domains.cd_sys5x = ClockDomain(reset_less=True)
+        self.clock_domains.cd_clk200 = ClockDomain()
+
+        self.current_clock = CSRStatus()
+
+        self.cd_sys.clk.attr.add("keep")
+
+        bootstrap_clk = ClockSignal("bootstrap")
+
+        period = 1e9/freq
+
+        self.submodules.clk_sw_fsm = ClockSwitchFSM()
+
+        if clk_sw is None:
+            self.clock_switch = CSRStorage()
+            self.comb += self.clk_sw_fsm.i_clk_sw.eq(self.clock_switch.storage)
+        else:
+            self.comb += self.clk_sw_fsm.i_clk_sw.eq(clk_sw)
+
+        self.mmcm_locked = Signal()
+        mmcm_sys = Signal()
+        mmcm_sys4x = Signal()
+        mmcm_sys5x = Signal()
+        mmcm_clk208 = Signal()
+        mmcm_fb_clk = Signal()
+        self.specials += [
+            Instance("MMCME2_ADV",
+                p_STARTUP_WAIT="FALSE", o_LOCKED=self.mmcm_locked,
+                p_BANDWIDTH="HIGH",
+                p_REF_JITTER1=0.001,
+                p_CLKIN1_PERIOD=period, i_CLKIN1=main_clk,
+                p_CLKIN2_PERIOD=period, i_CLKIN2=bootstrap_clk,
+                i_CLKINSEL=self.clk_sw_fsm.o_clk_sw,
+
+                # VCO @ 1.25GHz
+                p_CLKFBOUT_MULT_F=10, p_DIVCLK_DIVIDE=1,
+                i_CLKFBIN=mmcm_fb_clk,
+                i_RST=self.clk_sw_fsm.o_reset,
+
+                o_CLKFBOUT=mmcm_fb_clk,
+
+                p_CLKOUT0_DIVIDE_F=2.5, p_CLKOUT0_PHASE=0.0, o_CLKOUT0=mmcm_sys4x,
+
+                # 125MHz
+                p_CLKOUT1_DIVIDE=10, p_CLKOUT1_PHASE=0.0, o_CLKOUT1=mmcm_sys,
+
+                # 625MHz
+                p_CLKOUT2_DIVIDE=2, p_CLKOUT2_PHASE=0.0, o_CLKOUT2=mmcm_sys5x,
+
+                # 208MHz
+                p_CLKOUT3_DIVIDE=6, p_CLKOUT3_PHASE=0.0, o_CLKOUT3=mmcm_clk208,
+            ),
+            Instance("BUFG", i_I=mmcm_sys5x, o_O=self.cd_sys5x.clk),
+            Instance("BUFG", i_I=mmcm_sys, o_O=self.cd_sys.clk),
+            Instance("BUFG", i_I=mmcm_sys4x, o_O=self.cd_sys4x.clk),
+            Instance("BUFG", i_I=mmcm_clk208, o_O=self.cd_clk200.clk),
+        ]
+
+        if ext_async_rst is not None:
+            self.specials += [
+                AsyncResetSynchronizer(self.cd_sys, ~self.mmcm_locked | ext_async_rst),
+                AsyncResetSynchronizer(self.cd_clk200, ~self.mmcm_locked | ext_async_rst),
+            ]
+        else:
+            self.specials += [
+                AsyncResetSynchronizer(self.cd_sys, ~self.mmcm_locked),
+                AsyncResetSynchronizer(self.cd_clk200, ~self.mmcm_locked),
+            ]
+
+        reset_counter = Signal(4, reset=15)
+        ic_reset = Signal(reset=1)
+        self.sync.clk200 += \
+            If(reset_counter != 0,
+                reset_counter.eq(reset_counter - 1)
+            ).Else(
+                ic_reset.eq(0)
+            )
+        self.specials += Instance("IDELAYCTRL", i_REFCLK=ClockSignal("clk200"), i_RST=ic_reset)
+
+        if clk_sw_status is None:
+            self.comb += self.current_clock.status.eq(self.clk_sw_fsm.o_clk_sw)
+        else:
+            self.comb += self.current_clock.status.eq(clk_sw_status)

src/libboard_artiq/Cargo.toml.tpl

@@ -24,8 +24,9 @@ nb = "1.0"
 void = { version = "1", default-features = false }
 
 io = { path = "../libio", features = ["byteorder"] }
-libboard_zynq = { git = "https://git.m-labs.hk/M-Labs/zynq-rs.git"}
-libregister = { git = "https://git.m-labs.hk/M-Labs/zynq-rs.git" }
-libconfig = { git = "https://git.m-labs.hk/M-Labs/zynq-rs.git", features = ["fat_lfn"] }
-libcortex_a9 = { git = "https://git.m-labs.hk/M-Labs/zynq-rs.git" }
-libasync = { git = "https://git.m-labs.hk/M-Labs/zynq-rs.git" }
+libboard_zynq = { path = "@@ZYNQ_RS@@/libboard_zynq" }
+libsupport_zynq = { path = "@@ZYNQ_RS@@/libsupport_zynq", default-features = false, features = ["alloc_core"] }
+libregister = { path = "@@ZYNQ_RS@@/libregister" }
+libconfig = { path = "@@ZYNQ_RS@@/libconfig", features = ["fat_lfn"] }
+libcortex_a9 = { path = "@@ZYNQ_RS@@/libcortex_a9" }
+libasync = { path = "@@ZYNQ_RS@@/libasync" }

src/libboard_artiq/src/drtio_eem.rs

@@ -0,0 +1,228 @@
+use embedded_hal::prelude::_embedded_hal_blocking_delay_DelayUs;
+use libboard_zynq::timer::GlobalTimer;
+use libconfig::Config;
+use libsupport_zynq::alloc::format;
+use log::{debug, error, info};
+
+use crate::pl;
+
+struct SerdesConfig {
+    pub delay: [u8; 4],
+}
+
+impl SerdesConfig {
+    pub fn as_bytes(&self) -> &[u8] {
+        unsafe {
+            core::slice::from_raw_parts(
+                (self as *const SerdesConfig) as *const u8,
+                core::mem::size_of::<SerdesConfig>(),
+            )
+        }
+    }
+}
+
+fn select_lane(lane_no: u8) {
+    unsafe {
+        pl::csr::eem_transceiver::lane_sel_write(lane_no);
+    }
+}
+
+fn apply_delay(tap: u8, timer: &mut GlobalTimer) {
+    unsafe {
+        pl::csr::eem_transceiver::dly_cnt_in_write(tap);
+        pl::csr::eem_transceiver::dly_ld_write(1);
+        timer.delay_us(1);
+        assert!(tap as u8 == pl::csr::eem_transceiver::dly_cnt_out_read());
+    }
+}
+
+fn apply_config(config: &SerdesConfig, timer: &mut GlobalTimer) {
+    for lane_no in 0..4 {
+        select_lane(lane_no as u8);
+        apply_delay(config.delay[lane_no], timer);
+    }
+}
+
+unsafe fn assign_delay(timer: &mut GlobalTimer) -> SerdesConfig {
+    // Select an appropriate delay for lane 0
+    select_lane(0);
+
+    //
+
+    let mut best_dly = None;
+
+    loop {
+        let mut prev = None;
+        for curr_dly in 0..32 {
+            //let read_align = read_align_fn(curr_dly, timer);
+            let curr_low_rate = read_align(curr_dly, timer);
+
+            if let Some(prev_low_rate) = prev {
+                // This is potentially a crossover position
+                if prev_low_rate <= curr_low_rate && curr_low_rate >= 0.5 {
+                    let prev_dev = 0.5 - prev_low_rate;
+                    let curr_dev = curr_low_rate - 0.5;
+                    let selected_idx = if prev_dev < curr_dev { curr_dly - 1 } else { curr_dly };
+
+                    // The setup setup/hold calibration timing (even with
+                    // tolerance) might be invalid in other lanes due to skew.
+                    // 5 taps is very conservative, generally it is 1 or 2
+                    if selected_idx < 5 {
+                        prev = None;
+                        continue;
+                    } else {
+                        best_dly = Some(selected_idx);
+                        break;
+                    }
+                }
+            }
+
+            // Only rising slope from <= 0.5 can result in a rising low rate
+            // crossover at 50%.
+            if curr_low_rate <= 0.5 {
+                prev = Some(curr_low_rate);
+            }
+        }
+
+        if best_dly.is_none() {
+            error!("setup/hold timing calibration failed, retry in 1s...");
+            timer.delay_us(1_000_000);
+        } else {
+            break;
+        }
+    }
+
+    let best_dly = best_dly.unwrap();
+
+    apply_delay(best_dly, timer);
+    let mut delay_list = [best_dly; 4];
+
+    // Assign delay for other lanes
+    for lane_no in 1..=3 {
+        select_lane(lane_no as u8);
+
+        let mut min_deviation = 0.5;
+        let mut min_idx = 0;
+        for dly_delta in -3..=3 {
+            let index = (best_dly as isize + dly_delta) as u8;
+            let low_rate = read_align(index, timer);
+            // abs() from f32 is not available in core library
+            let deviation = if low_rate < 0.5 { 0.5 - low_rate } else { low_rate - 0.5 };
+
+            if deviation < min_deviation {
+                min_deviation = deviation;
+                min_idx = index;
+            }
+        }
+
+        apply_delay(min_idx, timer);
+        delay_list[lane_no] = min_idx;
+    }
+
+    debug!("setup/hold timing calibration: {:?}", delay_list);
+
+    SerdesConfig { delay: delay_list }
+}
+
+fn read_align(dly: u8, timer: &mut GlobalTimer) -> f32 {
+    unsafe {
+        apply_delay(dly, timer);
+        pl::csr::eem_transceiver::counter_reset_write(1);
+
+        pl::csr::eem_transceiver::counter_enable_write(1);
+        timer.delay_us(2000);
+        pl::csr::eem_transceiver::counter_enable_write(0);
+
+        let (high, low) = (
+            pl::csr::eem_transceiver::counter_high_count_read(),
+            pl::csr::eem_transceiver::counter_low_count_read(),
+        );
+        if pl::csr::eem_transceiver::counter_overflow_read() == 1 {
+            panic!("Unexpected phase detector counter overflow");
+        }
+
+        low as f32 / (low + high) as f32
+    }
+}
+
+unsafe fn align_comma(timer: &mut GlobalTimer) {
+    loop {
+        for slip in 1..=10 {
+            // The soft transceiver has 2 8b10b decoders, which receives lane
+            // 0/1 and lane 2/3 respectively. The decoder are time-multiplexed
+            // to decode exactly 1 lane each sysclk cycle.
+            //
+            // The decoder decodes lane 0/2 data on odd sysclk cycles, buffer
+            // on even cycles, and vice versa for lane 1/3. Data/Clock latency
+            // could change timing. The extend bit flips the decoding timing,
+            // so lane 0/2 data are decoded on even cycles, and lane 1/3 data
+            // are decoded on odd cycles.
+            //
+            // This is needed because transmitting/receiving a 8b10b character
+            // takes 2 sysclk cycles. Adjusting bitslip only via ISERDES
+            // limits the range to 1 cycle. The wordslip bit extends the range
+            // to 2 sysclk cycles.
+            pl::csr::eem_transceiver::wordslip_write((slip > 5) as u8);
+
+            // Apply a double bitslip since the ISERDES is 2x oversampled.
+            // Bitslip is used for comma alignment purposes once setup/hold
+            // timing is met.
+            pl::csr::eem_transceiver::bitslip_write(1);
+            pl::csr::eem_transceiver::bitslip_write(1);
+            timer.delay_us(1);
+
+            pl::csr::eem_transceiver::comma_align_reset_write(1);
+            timer.delay_us(100);
+
+            if pl::csr::eem_transceiver::comma_read() == 1 {
+                debug!("comma alignment completed after {} bitslips", slip);
+                return;
+            }
+        }
+
+        error!("comma alignment failed, retrying in 1s...");
+        timer.delay_us(1_000_000);
+    }
+}
+
+pub fn init(timer: &mut GlobalTimer, cfg: &Config) {
+    for trx_no in 0..pl::csr::CONFIG_EEM_DRTIO_COUNT {
+        unsafe {
+            pl::csr::eem_transceiver::transceiver_sel_write(trx_no as u8);
+        }
+
+        let key = format!("eem_drtio_delay{}", trx_no);
+
+        let cfg_read = cfg.read(&key);
+        match cfg_read {
+            Ok(record) => {
+                info!("loading calibrated timing values from sd card");
+                unsafe {
+                    apply_config(&*(record.as_ptr() as *const SerdesConfig), timer);
+                }
+            }
+            Err(_) => {
+                info!("calibrating...");
+                let config = unsafe { assign_delay(timer) };
+
+                match cfg.write(&key, config.as_bytes().to_vec()) {
+                    Ok(()) => {
+                        info!("storing calibration timing values into sd card");
+                    }
+                    Err(e) => {
+                        error!(
+                            "calibration successful but calibration timing values cannot be stored into sd card. \
+                             Error:{}",
+                            e
+                        );
+                    }
+                };
+            }
+        }
+
+        unsafe {
+            align_comma(timer);
+            pl::csr::eem_transceiver::rx_ready_write(1);
+        }
+    }
+}

src/libboard_artiq/src/drtio_routing.rs

@@ -1,9 +1,10 @@
-use libconfig::Config;
-#[cfg(has_drtio_routing)]
-use crate::pl::csr;
 use core::fmt;
 
-use log::{warn, info};
+use libconfig::Config;
+use log::{info, warn};
+
+#[cfg(has_drtio_routing)]
+use crate::pl::csr;
 
 #[cfg(has_drtio_routing)]
 pub const DEST_COUNT: usize = 256;
@@ -18,7 +19,7 @@ impl RoutingTable {
     // default routing table is for star topology with no repeaters
     pub fn default_master(default_n_links: usize) -> RoutingTable {
         let mut ret = RoutingTable([[INVALID_HOP; MAX_HOPS]; DEST_COUNT]);
-        let n_entries = default_n_links + 1;  // include local RTIO
+        let n_entries = default_n_links + 1; // include local RTIO
         for i in 0..n_entries {
             ret.0[i][0] = i as u8;
         }
@@ -58,10 +59,10 @@ impl fmt::Display for RoutingTable {
 pub fn config_routing_table(default_n_links: usize, cfg: &Config) -> RoutingTable {
     let mut ret = RoutingTable::default_master(default_n_links);
     if let Ok(data) = cfg.read("routing_table") {
-        if data.len() == DEST_COUNT*MAX_HOPS {
+        if data.len() == DEST_COUNT * MAX_HOPS {
             for i in 0..DEST_COUNT {
                 for j in 0..MAX_HOPS {
-                    ret.0[i][j] = data[i*MAX_HOPS+j];
+                    ret.0[i][j] = data[i * MAX_HOPS + j];
                 }
             }
         } else {

src/libboard_artiq/src/drtioaux.rs

@@ -1,14 +1,12 @@
+use core_io::{Error as IoError, ErrorKind as IoErrorKind};
 use crc;
-
-use core_io::{ErrorKind as IoErrorKind, Error as IoError};
-use io::{proto::ProtoRead, proto::ProtoWrite, Cursor};
-use libboard_zynq::{timer::GlobalTimer, time::Milliseconds};
+use io::{proto::{ProtoRead, ProtoWrite},
+         Cursor};
+use libboard_zynq::{time::Milliseconds, timer::GlobalTimer};
 use libcortex_a9::asm::dmb;
-use crate::mem::mem::DRTIOAUX_MEM;
-use crate::pl::csr::DRTIOAUX;
-use crate::drtioaux_proto::Error as ProtocolError;
 
 pub use crate::drtioaux_proto::Packet;
+use crate::{drtioaux_proto::Error as ProtocolError, mem::mem::DRTIOAUX_MEM, pl::csr::DRTIOAUX};
 
 #[derive(Debug)]
 pub enum Error {
@@ -21,7 +19,7 @@ pub enum Error {
 
     RoutingError,
 
-    Protocol(ProtocolError)
+    Protocol(ProtocolError),
 }
 
 impl From<ProtocolError> for Error {
@@ -63,7 +61,7 @@ pub fn copy_work_buffer(src: *mut u32, dst: *mut u32, len: isize) {
     // and AXI burst reads/writes are not implemented yet in gateware
     // thus the need for a work buffer for transmitting and copying it over
     unsafe {
-        for i in 0..(len/4) {
+        for i in 0..(len / 4) {
             *dst.offset(i) = *src.offset(i);
             //data memory barrier to prevent bursts
             dmb();
@@ -72,8 +70,7 @@ pub fn copy_work_buffer(src: *mut u32, dst: *mut u32, len: isize) {
 }
 
 fn receive<F, T>(linkno: u8, f: F) -> Result<Option<T>, Error>
-    where F: FnOnce(&[u8]) -> Result<T, Error>
-{
+where F: FnOnce(&[u8]) -> Result<T, Error> {
     let linkidx = linkno as usize;
     unsafe {
         if (DRTIOAUX[linkidx].aux_rx_present_read)() == 1 {
@@ -93,12 +90,12 @@ fn receive<F, T>(linkno: u8, f: F) -> Result<Option<T>, Error>
 
 pub fn recv(linkno: u8) -> Result<Option<Packet>, Error> {
     if has_rx_error(linkno) {
-        return Err(Error::GatewareError)
+        return Err(Error::GatewareError);
     }
 
     receive(linkno, |buffer| {
         if buffer.len() < 8 {
-            return Err(IoError::new(IoErrorKind::UnexpectedEof, "Unexpected end").into())
+            return Err(IoError::new(IoErrorKind::UnexpectedEof, "Unexpected end").into());
         }
 
         let mut reader = Cursor::new(buffer);
@@ -107,7 +104,7 @@ pub fn recv(linkno: u8) -> Result<Option<Packet>, Error> {
         let checksum = crc::crc32::checksum_ieee(&reader.get_ref()[0..checksum_at]);
         reader.set_position(checksum_at);
         if reader.read_u32()? != checksum {
-            return Err(Error::CorruptedPacket)
+            return Err(Error::CorruptedPacket);
         }
         reader.set_position(0);
 
@@ -115,9 +112,7 @@ pub fn recv(linkno: u8) -> Result<Option<Packet>, Error> {
     })
 }
 
-pub fn recv_timeout(linkno: u8, timeout_ms: Option<u64>,
-    timer: GlobalTimer) -> Result<Packet, Error> 
-{
+pub fn recv_timeout(linkno: u8, timeout_ms: Option<u64>, timer: GlobalTimer) -> Result<Packet, Error> {
     let timeout_ms = Milliseconds(timeout_ms.unwrap_or(10));
     let limit = timer.get_time() + timeout_ms;
     while timer.get_time() < limit {
@@ -130,8 +125,7 @@ pub fn recv_timeout(linkno: u8, timeout_ms: Option<u64>,
 }
 
 fn transmit<F>(linkno: u8, f: F) -> Result<(), Error>
-    where F: FnOnce(&mut [u8]) -> Result<usize, Error>
-{
+where F: FnOnce(&mut [u8]) -> Result<usize, Error> {
     let linkno = linkno as usize;
     unsafe {
         while (DRTIOAUX[linkno].aux_tx_read)() != 0 {}

src/libboard_artiq/src/drtioaux_async.rs

@@ -1,18 +1,16 @@
+use core_io::{Error as IoError, ErrorKind as IoErrorKind};
 use crc;
-
-use core_io::{ErrorKind as IoErrorKind, Error as IoError};
-use void::Void;
+use io::{proto::{ProtoRead, ProtoWrite},
+         Cursor};
+use libasync::{block_async, task};
+use libboard_zynq::{time::Milliseconds, timer::GlobalTimer};
 use nb;
-
-use libboard_zynq::{timer::GlobalTimer, time::Milliseconds};
-use libasync::{task, block_async};
-
-use io::{proto::ProtoRead, proto::ProtoWrite, Cursor};
-use crate::mem::mem::DRTIOAUX_MEM;
-use crate::pl::csr::DRTIOAUX;
-use crate::drtioaux::{Error, has_rx_error, copy_work_buffer};
+use void::Void;
 
 pub use crate::drtioaux_proto::Packet;
+use crate::{drtioaux::{copy_work_buffer, has_rx_error, Error},
+            mem::mem::DRTIOAUX_MEM,
+            pl::csr::DRTIOAUX};
 
 pub async fn reset(linkno: u8) {
     let linkno = linkno as usize;
@@ -29,16 +27,14 @@ fn tx_ready(linkno: usize) -> nb::Result<(), Void> {
     unsafe {
         if (DRTIOAUX[linkno].aux_tx_read)() != 0 {
             Err(nb::Error::WouldBlock)
-        }
-        else {
+        } else {
             Ok(())
         }
     }
 }
 
 async fn receive<F, T>(linkno: u8, f: F) -> Result<Option<T>, Error>
-    where F: FnOnce(&[u8]) -> Result<T, Error>
-{
+where F: FnOnce(&[u8]) -> Result<T, Error> {
     let linkidx = linkno as usize;
     unsafe {
         if (DRTIOAUX[linkidx].aux_rx_present_read)() == 1 {
@@ -58,12 +54,12 @@ async fn receive<F, T>(linkno: u8, f: F) -> Result<Option<T>, Error>
 
 pub async fn recv(linkno: u8) -> Result<Option<Packet>, Error> {
     if has_rx_error(linkno) {
-        return Err(Error::GatewareError)
+        return Err(Error::GatewareError);
     }
 
     receive(linkno, |buffer| {
         if buffer.len() < 8 {
-            return Err(IoError::new(IoErrorKind::UnexpectedEof, "Unexpected end").into())
+            return Err(IoError::new(IoErrorKind::UnexpectedEof, "Unexpected end").into());
         }
 
         let mut reader = Cursor::new(buffer);
@@ -72,17 +68,16 @@ pub async fn recv(linkno: u8) -> Result<Option<Packet>, Error> {
         let checksum = crc::crc32::checksum_ieee(&reader.get_ref()[0..checksum_at]);
         reader.set_position(checksum_at);
         if reader.read_u32()? != checksum {
-            return Err(Error::CorruptedPacket)
+            return Err(Error::CorruptedPacket);
         }
         reader.set_position(0);
 
         Ok(Packet::read_from(&mut reader)?)
-    }).await
+    })
+    .await
 }
 
-pub async fn recv_timeout(linkno: u8, timeout_ms: Option<u64>,
-    timer: GlobalTimer) -> Result<Packet, Error> 
-{
+pub async fn recv_timeout(linkno: u8, timeout_ms: Option<u64>, timer: GlobalTimer) -> Result<Packet, Error> {
     let timeout_ms = Milliseconds(timeout_ms.unwrap_or(10));
     let limit = timer.get_time() + timeout_ms;
     let mut would_block = false;
@@ -93,7 +88,9 @@ pub async fn recv_timeout(linkno: u8, timeout_ms: Option<u64>,
             task::r#yield().await;
         }
         match recv(linkno).await? {
-            None => { would_block = true; },
+            None => {
+                would_block = true;
+            }
             Some(packet) => return Ok(packet),
         }
     }
@@ -101,8 +98,7 @@ pub async fn recv_timeout(linkno: u8, timeout_ms: Option<u64>,
 }
 
 async fn transmit<F>(linkno: u8, f: F) -> Result<(), Error>
-    where F: FnOnce(&mut [u8]) -> Result<usize, Error>
-{
+where F: FnOnce(&mut [u8]) -> Result<usize, Error> {
     let linkno = linkno as usize;
     unsafe {
         let _ = block_async!(tx_ready(linkno)).await;
@@ -134,5 +130,6 @@ pub async fn send(linkno: u8, packet: &Packet) -> Result<(), Error> {
         writer.write_u32(checksum)?;
 
         Ok(writer.position())
-    }).await
+    })
+    .await
 }

src/libboard_artiq/src/drtioaux_proto.rs

@@ -1,11 +1,16 @@
-use core_io::{Write, Read, Error as IoError};
+use core_io::{Error as IoError, Read, Write};
+use io::proto::{ProtoRead, ProtoWrite};
 
-use io::proto::{ProtoWrite, ProtoRead};
+// maximum size of arbitrary payloads
+// used by satellite -> master analyzer, subkernel exceptions
+pub const SAT_PAYLOAD_MAX_SIZE: usize  = /*max size*/512 - /*CRC*/4 - /*packet ID*/1 - /*last*/1 - /*length*/2;
+// used by DDMA, subkernel program data (need to provide extra ID and destination)
+pub const MASTER_PAYLOAD_MAX_SIZE: usize = SAT_PAYLOAD_MAX_SIZE - /*destination*/1 - /*ID*/4;
 
 #[derive(Debug)]
 pub enum Error {
     UnknownPacket(u8),
-    Io(IoError)
+    Io(IoError),
 }
 
 impl From<IoError> for Error {
@@ -14,6 +19,46 @@ impl From<IoError> for Error {
     }
 }
 
+#[derive(PartialEq, Clone, Copy, Debug)]
+#[repr(u8)]
+pub enum PayloadStatus {
+    Middle = 0,
+    First = 1,
+    Last = 2,
+    FirstAndLast = 3,
+}
+
+impl From<u8> for PayloadStatus {
+    fn from(value: u8) -> PayloadStatus {
+        match value {
+            0 => PayloadStatus::Middle,
+            1 => PayloadStatus::First,
+            2 => PayloadStatus::Last,
+            3 => PayloadStatus::FirstAndLast,
+            _ => unreachable!(),
+        }
+    }
+}
+
+impl PayloadStatus {
+    pub fn is_first(self) -> bool {
+        self == PayloadStatus::First || self == PayloadStatus::FirstAndLast
+    }
+
+    pub fn is_last(self) -> bool {
+        self == PayloadStatus::Last || self == PayloadStatus::FirstAndLast
+    }
+
+    pub fn from_status(first: bool, last: bool) -> PayloadStatus {
+        match (first, last) {
+            (true, true) => PayloadStatus::FirstAndLast,
+            (true, false) => PayloadStatus::First,
+            (false, true) => PayloadStatus::Last,
+            (false, false) => PayloadStatus::Middle,
+        }
+    }
+}
+
 #[derive(PartialEq, Debug)]
 pub enum Packet {
     EchoRequest,
@@ -22,45 +67,219 @@ pub enum Packet {
     ResetAck,
     TSCAck,
 
-    DestinationStatusRequest { destination: u8 },
+    DestinationStatusRequest {
+        destination: u8,
+    },
     DestinationDownReply,
     DestinationOkReply,
-    DestinationSequenceErrorReply { channel: u16 },
-    DestinationCollisionReply { channel: u16 },
-    DestinationBusyReply { channel: u16 },
+    DestinationSequenceErrorReply {
+        channel: u16,
+    },
+    DestinationCollisionReply {
+        channel: u16,
+    },
+    DestinationBusyReply {
+        channel: u16,
+    },
 
-    RoutingSetPath { destination: u8, hops: [u8; 32] },
-    RoutingSetRank { rank: u8 },
+    RoutingSetPath {
+        destination: u8,
+        hops: [u8; 32],
+    },
+    RoutingSetRank {
+        rank: u8,
+    },
     RoutingAck,
 
-    MonitorRequest { destination: u8, channel: u16, probe: u8 },
-    MonitorReply { value: u64 },
-    InjectionRequest { destination: u8, channel: u16, overrd: u8, value: u8 },
-    InjectionStatusRequest { destination: u8, channel: u16, overrd: u8 },
-    InjectionStatusReply { value: u8 },
+    MonitorRequest {
+        destination: u8,
+        channel: u16,
+        probe: u8,
+    },
+    MonitorReply {
+        value: u64,
+    },
+    InjectionRequest {
+        destination: u8,
+        channel: u16,
+        overrd: u8,
+        value: u8,
+    },
+    InjectionStatusRequest {
+        destination: u8,
+        channel: u16,
+        overrd: u8,
+    },
+    InjectionStatusReply {
+        value: u8,
+    },
 
-    I2cStartRequest { destination: u8, busno: u8 },
-    I2cRestartRequest { destination: u8, busno: u8 },
-    I2cStopRequest { destination: u8, busno: u8 },
-    I2cWriteRequest { destination: u8, busno: u8, data: u8 },
-    I2cWriteReply { succeeded: bool, ack: bool },
-    I2cReadRequest { destination: u8, busno: u8, ack: bool },
-    I2cReadReply { succeeded: bool, data: u8 },
-    I2cBasicReply { succeeded: bool },
-    I2cSwitchSelectRequest { destination: u8, busno: u8, address: u8, mask: u8 },
+    I2cStartRequest {
+        destination: u8,
+        busno: u8,
+    },
+    I2cRestartRequest {
+        destination: u8,
+        busno: u8,
+    },
+    I2cStopRequest {
+        destination: u8,
+        busno: u8,
+    },
+    I2cWriteRequest {
+        destination: u8,
+        busno: u8,
+        data: u8,
+    },
+    I2cWriteReply {
+        succeeded: bool,
+        ack: bool,
+    },
+    I2cReadRequest {
+        destination: u8,
+        busno: u8,
+        ack: bool,
+    },
+    I2cReadReply {
+        succeeded: bool,
+        data: u8,
+    },
+    I2cBasicReply {
+        succeeded: bool,
+    },
+    I2cSwitchSelectRequest {
+        destination: u8,
+        busno: u8,
+        address: u8,
+        mask: u8,
+    },
 
-    SpiSetConfigRequest { destination: u8, busno: u8, flags: u8, length: u8, div: u8, cs: u8 },
-    SpiWriteRequest { destination: u8, busno: u8, data: u32 },
-    SpiReadRequest { destination: u8, busno: u8 },
-    SpiReadReply { succeeded: bool, data: u32 },
-    SpiBasicReply { succeeded: bool },
+    SpiSetConfigRequest {
+        destination: u8,
+        busno: u8,
+        flags: u8,
+        length: u8,
+        div: u8,
+        cs: u8,
+    },
+    SpiWriteRequest {
+        destination: u8,
+        busno: u8,
+        data: u32,
+    },
+    SpiReadRequest {
+        destination: u8,
+        busno: u8,
+    },
+    SpiReadReply {
+        succeeded: bool,
+        data: u32,
+    },
+    SpiBasicReply {
+        succeeded: bool,
+    },
 
+    AnalyzerHeaderRequest {
+        destination: u8,
+    },
+    AnalyzerHeader {
+        sent_bytes: u32,
+        total_byte_count: u64,
+        overflow_occurred: bool,
+    },
+    AnalyzerDataRequest {
+        destination: u8,
+    },
+    AnalyzerData {
+        last: bool,
+        length: u16,
+        data: [u8; SAT_PAYLOAD_MAX_SIZE],
+    },
+
+    DmaAddTraceRequest {
+        destination: u8,
+        id: u32,
+        status: PayloadStatus,
+        length: u16,
+        trace: [u8; MASTER_PAYLOAD_MAX_SIZE],
+    },
+    DmaAddTraceReply {
+        succeeded: bool,
+    },
+    DmaRemoveTraceRequest {
+        destination: u8,
+        id: u32,
+    },
+    DmaRemoveTraceReply {
+        succeeded: bool,
+    },
+    DmaPlaybackRequest {
+        destination: u8,
+        id: u32,
+        timestamp: u64,
+    },
+    DmaPlaybackReply {
+        succeeded: bool,
+    },
+    DmaPlaybackStatus {
+        destination: u8,
+        id: u32,
+        error: u8,
+        channel: u32,
+        timestamp: u64,
+    },
+
+    SubkernelAddDataRequest {
+        destination: u8,
+        id: u32,
+        status: PayloadStatus,
+        length: u16,
+        data: [u8; MASTER_PAYLOAD_MAX_SIZE],
+    },
+    SubkernelAddDataReply {
+        succeeded: bool,
+    },
+    SubkernelLoadRunRequest {
+        destination: u8,
+        id: u32,
+        run: bool,
+    },
+    SubkernelLoadRunReply {
+        succeeded: bool,
+    },
+    SubkernelStopRequest {
+        destination: u8,
+    },
+    SubkernelStopReply {
+        succeeded: bool,
+    },
+    SubkernelFinished {
+        id: u32,
+        with_exception: bool,
+    },
+    SubkernelExceptionRequest {
+        destination: u8,
+    },
+    SubkernelException {
+        last: bool,
+        length: u16,
+        data: [u8; SAT_PAYLOAD_MAX_SIZE],
+    },
+    SubkernelMessage {
+        destination: u8,
+        id: u32,
+        status: PayloadStatus,
+        length: u16,
+        data: [u8; MASTER_PAYLOAD_MAX_SIZE],
+    },
+    SubkernelMessageAck {
+        destination: u8,
+    },
 }
 
 impl Packet {
     pub fn read_from<R>(reader: &mut R) -> Result<Self, Error>
-        where R: Read + ?Sized
-    {
+    where R: Read + ?Sized {
         Ok(match reader.read_u8()? {
             0x00 => Packet::EchoRequest,
             0x01 => Packet::EchoReply,
@@ -69,18 +288,18 @@ impl Packet {
             0x04 => Packet::TSCAck,
 
             0x20 => Packet::DestinationStatusRequest {
-                destination: reader.read_u8()?
+                destination: reader.read_u8()?,
             },
             0x21 => Packet::DestinationDownReply,
             0x22 => Packet::DestinationOkReply,
             0x23 => Packet::DestinationSequenceErrorReply {
-                channel: reader.read_u16()?
+                channel: reader.read_u16()?,
             },
             0x24 => Packet::DestinationCollisionReply {
-                channel: reader.read_u16()?
+                channel: reader.read_u16()?,
             },
             0x25 => Packet::DestinationBusyReply {
-                channel: reader.read_u16()?
+                channel: reader.read_u16()?,
             },
 
             0x30 => {
@@ -89,75 +308,75 @@ impl Packet {
                 reader.read_exact(&mut hops)?;
                 Packet::RoutingSetPath {
                     destination: destination,
-                    hops: hops
+                    hops: hops,
                 }
-            },
+            }
             0x31 => Packet::RoutingSetRank {
-                rank: reader.read_u8()?
+                rank: reader.read_u8()?,
             },
             0x32 => Packet::RoutingAck,
 
             0x40 => Packet::MonitorRequest {
                 destination: reader.read_u8()?,
                 channel: reader.read_u16()?,
-                probe: reader.read_u8()?
+                probe: reader.read_u8()?,
             },
             0x41 => Packet::MonitorReply {
-                value: reader.read_u64()?
+                value: reader.read_u64()?,
             },
             0x50 => Packet::InjectionRequest {
                 destination: reader.read_u8()?,
                 channel: reader.read_u16()?,
                 overrd: reader.read_u8()?,
-                value: reader.read_u8()?
+                value: reader.read_u8()?,
             },
             0x51 => Packet::InjectionStatusRequest {
                 destination: reader.read_u8()?,
                 channel: reader.read_u16()?,
-                overrd: reader.read_u8()?
+                overrd: reader.read_u8()?,
             },
             0x52 => Packet::InjectionStatusReply {
-                value: reader.read_u8()?
+                value: reader.read_u8()?,
             },
 
             0x80 => Packet::I2cStartRequest {
                 destination: reader.read_u8()?,
-                busno: reader.read_u8()?
+                busno: reader.read_u8()?,
             },
             0x81 => Packet::I2cRestartRequest {
                 destination: reader.read_u8()?,
-                busno: reader.read_u8()?
+                busno: reader.read_u8()?,
             },
             0x82 => Packet::I2cStopRequest {
                 destination: reader.read_u8()?,
-                busno: reader.read_u8()?
+                busno: reader.read_u8()?,
             },
             0x83 => Packet::I2cWriteRequest {
                 destination: reader.read_u8()?,
                 busno: reader.read_u8()?,
-                data: reader.read_u8()?
+                data: reader.read_u8()?,
             },
             0x84 => Packet::I2cWriteReply {
                 succeeded: reader.read_bool()?,
-                ack: reader.read_bool()?
+                ack: reader.read_bool()?,
             },
             0x85 => Packet::I2cReadRequest {
                 destination: reader.read_u8()?,
                 busno: reader.read_u8()?,
-                ack: reader.read_bool()?
+                ack: reader.read_bool()?,
             },
             0x86 => Packet::I2cReadReply {
                 succeeded: reader.read_bool()?,
-                data: reader.read_u8()?
+                data: reader.read_u8()?,
             },
             0x87 => Packet::I2cBasicReply {
-                succeeded: reader.read_bool()?
+                succeeded: reader.read_bool()?,
             },
             0x88 => Packet::I2cSwitchSelectRequest {
                 destination: reader.read_u8()?,
                 busno: reader.read_u8()?,
                 address: reader.read_u8()?,
-                mask: reader.read_u8()?
+                mask: reader.read_u8()?,
             },
 
             0x90 => Packet::SpiSetConfigRequest {
@@ -166,157 +385,313 @@ impl Packet {
                 flags: reader.read_u8()?,
                 length: reader.read_u8()?,
                 div: reader.read_u8()?,
-                cs: reader.read_u8()?
+                cs: reader.read_u8()?,
             },
             /* 0x91: was Packet::SpiSetXferRequest */
             0x92 => Packet::SpiWriteRequest {
                 destination: reader.read_u8()?,
                 busno: reader.read_u8()?,
-                data: reader.read_u32()?
+                data: reader.read_u32()?,
             },
             0x93 => Packet::SpiReadRequest {
                 destination: reader.read_u8()?,
-                busno: reader.read_u8()?
+                busno: reader.read_u8()?,
             },
             0x94 => Packet::SpiReadReply {
                 succeeded: reader.read_bool()?,
-                data: reader.read_u32()?
+                data: reader.read_u32()?,
             },
             0x95 => Packet::SpiBasicReply {
-                succeeded: reader.read_bool()?
+                succeeded: reader.read_bool()?,
             },
 
-            ty => return Err(Error::UnknownPacket(ty))
+            0xa0 => Packet::AnalyzerHeaderRequest {
+                destination: reader.read_u8()?,
+            },
+            0xa1 => Packet::AnalyzerHeader {
+                sent_bytes: reader.read_u32()?,
+                total_byte_count: reader.read_u64()?,
+                overflow_occurred: reader.read_bool()?,
+            },
+            0xa2 => Packet::AnalyzerDataRequest {
+                destination: reader.read_u8()?,
+            },
+            0xa3 => {
+                let last = reader.read_bool()?;
+                let length = reader.read_u16()?;
+                let mut data: [u8; SAT_PAYLOAD_MAX_SIZE] = [0; SAT_PAYLOAD_MAX_SIZE];
+                reader.read_exact(&mut data[0..length as usize])?;
+                Packet::AnalyzerData {
+                    last: last,
+                    length: length,
+                    data: data,
+                }
+            }
+
+            0xb0 => {
+                let destination = reader.read_u8()?;
+                let id = reader.read_u32()?;
+                let status = PayloadStatus::from(reader.read_u8()?);
+                let length = reader.read_u16()?;
+                let mut trace: [u8; MASTER_PAYLOAD_MAX_SIZE] = [0; MASTER_PAYLOAD_MAX_SIZE];
+                reader.read_exact(&mut trace[0..length as usize])?;
+                Packet::DmaAddTraceRequest {
+                    destination: destination,
+                    id: id,
+                    status: status,
+                    length: length as u16,
+                    trace: trace,
+                }
+            }
+            0xb1 => Packet::DmaAddTraceReply {
+                succeeded: reader.read_bool()?,
+            },
+            0xb2 => Packet::DmaRemoveTraceRequest {
+                destination: reader.read_u8()?,
+                id: reader.read_u32()?,
+            },
+            0xb3 => Packet::DmaRemoveTraceReply {
+                succeeded: reader.read_bool()?,
+            },
+            0xb4 => Packet::DmaPlaybackRequest {
+                destination: reader.read_u8()?,
+                id: reader.read_u32()?,
+                timestamp: reader.read_u64()?,
+            },
+            0xb5 => Packet::DmaPlaybackReply {
+                succeeded: reader.read_bool()?,
+            },
+            0xb6 => Packet::DmaPlaybackStatus {
+                destination: reader.read_u8()?,
+                id: reader.read_u32()?,
+                error: reader.read_u8()?,
+                channel: reader.read_u32()?,
+                timestamp: reader.read_u64()?,
+            },
+
+            0xc0 => {
+                let destination = reader.read_u8()?;
+                let id = reader.read_u32()?;
+                let status = PayloadStatus::from(reader.read_u8()?);
+                let length = reader.read_u16()?;
+                let mut data: [u8; MASTER_PAYLOAD_MAX_SIZE] = [0; MASTER_PAYLOAD_MAX_SIZE];
+                reader.read_exact(&mut data[0..length as usize])?;
+                Packet::SubkernelAddDataRequest {
+                    destination: destination,
+                    id: id,
+                    status: status,
+                    length: length as u16,
+                    data: data,
+                }
+            }
+            0xc1 => Packet::SubkernelAddDataReply {
+                succeeded: reader.read_bool()?,
+            },
+            0xc4 => Packet::SubkernelLoadRunRequest {
+                destination: reader.read_u8()?,
+                id: reader.read_u32()?,
+                run: reader.read_bool()?,
+            },
+            0xc5 => Packet::SubkernelLoadRunReply {
+                succeeded: reader.read_bool()?,
+            },
+            0xc6 => Packet::SubkernelStopRequest {
+                destination: reader.read_u8()?,
+            },
+            0xc7 => Packet::SubkernelStopReply {
+                succeeded: reader.read_bool()?,
+            },
+            0xc8 => Packet::SubkernelFinished {
+                id: reader.read_u32()?,
+                with_exception: reader.read_bool()?,
+            },
+            0xc9 => Packet::SubkernelExceptionRequest {
+                destination: reader.read_u8()?,
+            },
+            0xca => {
+                let last = reader.read_bool()?;
+                let length = reader.read_u16()?;
+                let mut data: [u8; SAT_PAYLOAD_MAX_SIZE] = [0; SAT_PAYLOAD_MAX_SIZE];
+                reader.read_exact(&mut data[0..length as usize])?;
+                Packet::SubkernelException {
+                    last: last,
+                    length: length,
+                    data: data,
+                }
+            }
+            0xcb => {
+                let destination = reader.read_u8()?;
+                let id = reader.read_u32()?;
+                let status = PayloadStatus::from(reader.read_u8()?);
+                let length = reader.read_u16()?;
+                let mut data: [u8; MASTER_PAYLOAD_MAX_SIZE] = [0; MASTER_PAYLOAD_MAX_SIZE];
+                reader.read_exact(&mut data[0..length as usize])?;
+                Packet::SubkernelMessage {
+                    destination: destination,
+                    id: id,
+                    status: status,
+                    length: length as u16,
+                    data: data,
+                }
+            }
+            0xcc => Packet::SubkernelMessageAck {
+                destination: reader.read_u8()?,
+            },
+
+            ty => return Err(Error::UnknownPacket(ty)),
         })
     }
 
     pub fn write_to<W>(&self, writer: &mut W) -> Result<(), IoError>
-        where W: Write + ?Sized
-    {
-
+    where W: Write + ?Sized {
         match *self {
-            Packet::EchoRequest =>
-                writer.write_u8(0x00)?,
-            Packet::EchoReply =>
-                writer.write_u8(0x01)?,
-            Packet::ResetRequest =>
-                writer.write_u8(0x02)?,
-            Packet::ResetAck =>
-                writer.write_u8(0x03)?,
-            Packet::TSCAck =>
-                writer.write_u8(0x04)?,
+            Packet::EchoRequest => writer.write_u8(0x00)?,
+            Packet::EchoReply => writer.write_u8(0x01)?,
+            Packet::ResetRequest => writer.write_u8(0x02)?,
+            Packet::ResetAck => writer.write_u8(0x03)?,
+            Packet::TSCAck => writer.write_u8(0x04)?,
 
             Packet::DestinationStatusRequest { destination } => {
                 writer.write_u8(0x20)?;
                 writer.write_u8(destination)?;
-            },
-            Packet::DestinationDownReply =>
-                writer.write_u8(0x21)?,
-            Packet::DestinationOkReply =>
-                writer.write_u8(0x22)?,
+            }
+            Packet::DestinationDownReply => writer.write_u8(0x21)?,
+            Packet::DestinationOkReply => writer.write_u8(0x22)?,
             Packet::DestinationSequenceErrorReply { channel } => {
                 writer.write_u8(0x23)?;
                 writer.write_u16(channel)?;
-            },
+            }
             Packet::DestinationCollisionReply { channel } => {
                 writer.write_u8(0x24)?;
                 writer.write_u16(channel)?;
-            },
+            }
             Packet::DestinationBusyReply { channel } => {
                 writer.write_u8(0x25)?;
                 writer.write_u16(channel)?;
-            },
+            }
 
             Packet::RoutingSetPath { destination, hops } => {
                 writer.write_u8(0x30)?;
                 writer.write_u8(destination)?;
                 writer.write_all(&hops)?;
-            },
+            }
             Packet::RoutingSetRank { rank } => {
                 writer.write_u8(0x31)?;
                 writer.write_u8(rank)?;
-            },
-            Packet::RoutingAck =>
-                writer.write_u8(0x32)?,
+            }
+            Packet::RoutingAck => writer.write_u8(0x32)?,
 
-            Packet::MonitorRequest { destination, channel, probe } => {
+            Packet::MonitorRequest {
+                destination,
+                channel,
+                probe,
+            } => {
                 writer.write_u8(0x40)?;
                 writer.write_u8(destination)?;
                 writer.write_u16(channel)?;
                 writer.write_u8(probe)?;
-            },
+            }
             Packet::MonitorReply { value } => {
                 writer.write_u8(0x41)?;
                 writer.write_u64(value)?;
-            },
-            Packet::InjectionRequest { destination, channel, overrd, value } => {
+            }
+            Packet::InjectionRequest {
+                destination,
+                channel,
+                overrd,
+                value,
+            } => {
                 writer.write_u8(0x50)?;
                 writer.write_u8(destination)?;
                 writer.write_u16(channel)?;
                 writer.write_u8(overrd)?;
                 writer.write_u8(value)?;
-            },
-            Packet::InjectionStatusRequest { destination, channel, overrd } => {
+            }
+            Packet::InjectionStatusRequest {
+                destination,
+                channel,
+                overrd,
+            } => {
                 writer.write_u8(0x51)?;
                 writer.write_u8(destination)?;
                 writer.write_u16(channel)?;
                 writer.write_u8(overrd)?;
-            },
+            }
             Packet::InjectionStatusReply { value } => {
                 writer.write_u8(0x52)?;
                 writer.write_u8(value)?;
-            },
+            }
 
             Packet::I2cStartRequest { destination, busno } => {
                 writer.write_u8(0x80)?;
                 writer.write_u8(destination)?;
                 writer.write_u8(busno)?;
-            },
+            }
             Packet::I2cRestartRequest { destination, busno } => {
                 writer.write_u8(0x81)?;
                 writer.write_u8(destination)?;
                 writer.write_u8(busno)?;
-            },
+            }
             Packet::I2cStopRequest { destination, busno } => {
                 writer.write_u8(0x82)?;
                 writer.write_u8(destination)?;
                 writer.write_u8(busno)?;
-            },
-            Packet::I2cWriteRequest { destination, busno, data } => {
+            }
+            Packet::I2cWriteRequest {
+                destination,
+                busno,
+                data,
+            } => {
                 writer.write_u8(0x83)?;
                 writer.write_u8(destination)?;
                 writer.write_u8(busno)?;
                 writer.write_u8(data)?;
-            },
+            }
             Packet::I2cWriteReply { succeeded, ack } => {
                 writer.write_u8(0x84)?;
                 writer.write_bool(succeeded)?;
                 writer.write_bool(ack)?;
-            },
-            Packet::I2cReadRequest { destination, busno, ack } => {
+            }
+            Packet::I2cReadRequest {
+                destination,
+                busno,
+                ack,
+            } => {
                 writer.write_u8(0x85)?;
                 writer.write_u8(destination)?;
                 writer.write_u8(busno)?;
                 writer.write_bool(ack)?;
-            },
+            }
             Packet::I2cReadReply { succeeded, data } => {
                 writer.write_u8(0x86)?;
                 writer.write_bool(succeeded)?;
                 writer.write_u8(data)?;
-            },
+            }
             Packet::I2cBasicReply { succeeded } => {
                 writer.write_u8(0x87)?;
                 writer.write_bool(succeeded)?;
-            },
-            Packet::I2cSwitchSelectRequest { destination, busno, address, mask } => {
+            }
+            Packet::I2cSwitchSelectRequest {
+                destination,
+                busno,
+                address,
+                mask,
+            } => {
                 writer.write_u8(0x88)?;
                 writer.write_u8(destination)?;
                 writer.write_u8(busno)?;
                 writer.write_u8(address)?;
                 writer.write_u8(mask)?;
-            },
+            }
 
-            Packet::SpiSetConfigRequest { destination, busno, flags, length, div, cs } => {
+            Packet::SpiSetConfigRequest {
+                destination,
+                busno,
+                flags,
+                length,
+                div,
+                cs,
+            } => {
                 writer.write_u8(0x90)?;
                 writer.write_u8(destination)?;
                 writer.write_u8(busno)?;
@@ -324,30 +699,185 @@ impl Packet {
                 writer.write_u8(length)?;
                 writer.write_u8(div)?;
                 writer.write_u8(cs)?;
-            },
-            Packet::SpiWriteRequest { destination, busno, data } => {
+            }
+            Packet::SpiWriteRequest {
+                destination,
+                busno,
+                data,
+            } => {
                 writer.write_u8(0x92)?;
                 writer.write_u8(destination)?;
                 writer.write_u8(busno)?;
                 writer.write_u32(data)?;
-            },
+            }
             Packet::SpiReadRequest { destination, busno } => {
                 writer.write_u8(0x93)?;
                 writer.write_u8(destination)?;
                 writer.write_u8(busno)?;
-            },
+            }
             Packet::SpiReadReply { succeeded, data } => {
                 writer.write_u8(0x94)?;
                 writer.write_bool(succeeded)?;
                 writer.write_u32(data)?;
-            },
+            }
             Packet::SpiBasicReply { succeeded } => {
                 writer.write_u8(0x95)?;
                 writer.write_bool(succeeded)?;
-            },
+            }
 
+            Packet::AnalyzerHeaderRequest { destination } => {
+                writer.write_u8(0xa0)?;
+                writer.write_u8(destination)?;
+            }
+            Packet::AnalyzerHeader {
+                sent_bytes,
+                total_byte_count,
+                overflow_occurred,
+            } => {
+                writer.write_u8(0xa1)?;
+                writer.write_u32(sent_bytes)?;
+                writer.write_u64(total_byte_count)?;
+                writer.write_bool(overflow_occurred)?;
+            }
+            Packet::AnalyzerDataRequest { destination } => {
+                writer.write_u8(0xa2)?;
+                writer.write_u8(destination)?;
+            }
+            Packet::AnalyzerData { last, length, data } => {
+                writer.write_u8(0xa3)?;
+                writer.write_bool(last)?;
+                writer.write_u16(length)?;
+                writer.write_all(&data[0..length as usize])?;
+            }
+
+            Packet::DmaAddTraceRequest {
+                destination,
+                id,
+                status,
+                trace,
+                length,
+            } => {
+                writer.write_u8(0xb0)?;
+                writer.write_u8(destination)?;
+                writer.write_u32(id)?;
+                writer.write_u8(status as u8)?;
+                // trace may be broken down to fit within drtio aux memory limit
+                // will be reconstructed by satellite
+                writer.write_u16(length)?;
+                writer.write_all(&trace[0..length as usize])?;
+            }
+            Packet::DmaAddTraceReply { succeeded } => {
+                writer.write_u8(0xb1)?;
+                writer.write_bool(succeeded)?;
+            }
+            Packet::DmaRemoveTraceRequest { destination, id } => {
+                writer.write_u8(0xb2)?;
+                writer.write_u8(destination)?;
+                writer.write_u32(id)?;
+            }
+            Packet::DmaRemoveTraceReply { succeeded } => {
+                writer.write_u8(0xb3)?;
+                writer.write_bool(succeeded)?;
+            }
+            Packet::DmaPlaybackRequest {
+                destination,
+                id,
+                timestamp,
+            } => {
+                writer.write_u8(0xb4)?;
+                writer.write_u8(destination)?;
+                writer.write_u32(id)?;
+                writer.write_u64(timestamp)?;
+            }
+            Packet::DmaPlaybackReply { succeeded } => {
+                writer.write_u8(0xb5)?;
+                writer.write_bool(succeeded)?;
+            }
+            Packet::DmaPlaybackStatus {
+                destination,
+                id,
+                error,
+                channel,
+                timestamp,
+            } => {
+                writer.write_u8(0xb6)?;
+                writer.write_u8(destination)?;
+                writer.write_u32(id)?;
+                writer.write_u8(error)?;
+                writer.write_u32(channel)?;
+                writer.write_u64(timestamp)?;
+            }
+
+            Packet::SubkernelAddDataRequest {
+                destination,
+                id,
+                status,
+                data,
+                length,
+            } => {
+                writer.write_u8(0xc0)?;
+                writer.write_u8(destination)?;
+                writer.write_u32(id)?;
+                writer.write_u8(status as u8)?;
+                writer.write_u16(length)?;
+                writer.write_all(&data[0..length as usize])?;
+            }
+            Packet::SubkernelAddDataReply { succeeded } => {
+                writer.write_u8(0xc1)?;
+                writer.write_bool(succeeded)?;
+            }
+            Packet::SubkernelLoadRunRequest { destination, id, run } => {
+                writer.write_u8(0xc4)?;
+                writer.write_u8(destination)?;
+                writer.write_u32(id)?;
+                writer.write_bool(run)?;
+            }
+            Packet::SubkernelLoadRunReply { succeeded } => {
+                writer.write_u8(0xc5)?;
+                writer.write_bool(succeeded)?;
+            }
+            Packet::SubkernelStopRequest { destination } => {
+                writer.write_u8(0xc6)?;
+                writer.write_u8(destination)?;
+            }
+            Packet::SubkernelStopReply { succeeded } => {
+                writer.write_u8(0xc7)?;
+                writer.write_bool(succeeded)?;
+            }
+            Packet::SubkernelFinished { id, with_exception } => {
+                writer.write_u8(0xc8)?;
+                writer.write_u32(id)?;
+                writer.write_bool(with_exception)?;
+            }
+            Packet::SubkernelExceptionRequest { destination } => {
+                writer.write_u8(0xc9)?;
+                writer.write_u8(destination)?;
+            }
+            Packet::SubkernelException { last, length, data } => {
+                writer.write_u8(0xca)?;
+                writer.write_bool(last)?;
+                writer.write_u16(length)?;
+                writer.write_all(&data[0..length as usize])?;
+            }
+            Packet::SubkernelMessage {
+                destination,
+                id,
+                status,
+                data,
+                length,
+            } => {
+                writer.write_u8(0xcb)?;
+                writer.write_u8(destination)?;
+                writer.write_u32(id)?;
+                writer.write_u8(status as u8)?;
+                writer.write_u16(length)?;
+                writer.write_all(&data[0..length as usize])?;
+            }
+            Packet::SubkernelMessageAck { destination } => {
+                writer.write_u8(0xcc)?;
+                writer.write_u8(destination)?;
+            }
         }
         Ok(())
     }
-    
 }

src/libboard_artiq/src/grabber.rs

@@ -0,0 +1,163 @@
+use log::info;
+
+use crate::pl::csr;
+
+#[derive(PartialEq, Clone, Copy)]
+enum State {
+    Reset,
+    ExitReset,
+    Lock,
+    Align,
+    Watch,
+}
+
+#[derive(Clone, Copy)]
+struct Info {
+    state: State,
+    frame_size: (u16, u16),
+}
+
+static mut INFO: [Info; csr::GRABBER_LEN] = [Info {
+    state: State::Reset,
+    frame_size: (0, 0),
+}; csr::GRABBER_LEN];
+
+fn get_pll_reset(g: usize) -> bool {
+    unsafe { (csr::GRABBER[g].pll_reset_read)() != 0 }
+}
+
+fn set_pll_reset(g: usize, reset: bool) {
+    let val = if reset { 1 } else { 0 };
+    unsafe { (csr::GRABBER[g].pll_reset_write)(val) }
+}
+
+fn pll_locked(g: usize) -> bool {
+    unsafe { (csr::GRABBER[g].pll_locked_read)() != 0 }
+}
+
+fn clock_pattern_ok(g: usize) -> bool {
+    unsafe { (csr::GRABBER[g].clk_sampled_read)() == 0b1100011 }
+}
+
+fn clock_pattern_ok_filter(g: usize) -> bool {
+    for _ in 0..128 {
+        if !clock_pattern_ok(g) {
+            return false;
+        }
+    }
+    true
+}
+
+fn phase_shift(g: usize, direction: u8) {
+    unsafe {
+        (csr::GRABBER[g].phase_shift_write)(direction);
+        while (csr::GRABBER[g].phase_shift_done_read)() == 0 {}
+    }
+}
+
+fn clock_align(g: usize) -> bool {
+    while clock_pattern_ok_filter(g) {
+        phase_shift(g, 1);
+    }
+    phase_shift(g, 1);
+
+    let mut count = 0;
+    while !clock_pattern_ok_filter(g) {
+        phase_shift(g, 1);
+        count += 1;
+        if count > 1024 {
+            return false;
+        }
+    }
+
+    let mut window = 1;
+    phase_shift(g, 1);
+    while clock_pattern_ok_filter(g) {
+        phase_shift(g, 1);
+        window += 1;
+    }
+
+    for _ in 0..window / 2 {
+        phase_shift(g, 0);
+    }
+
+    true
+}
+
+fn get_last_pixels(g: usize) -> (u16, u16) {
+    unsafe { ((csr::GRABBER[g].last_x_read)(), (csr::GRABBER[g].last_y_read)()) }
+}
+
+fn get_video_clock(g: usize) -> u32 {
+    let freq_count = unsafe { (csr::GRABBER[g].freq_count_read)() } as u32;
+    2 * freq_count * (csr::CONFIG_CLOCK_FREQUENCY / 1000) / (511 * 1000)
+}
+
+pub fn tick() {
+    for g in 0..csr::GRABBER.len() {
+        let next = match unsafe { INFO[g].state } {
+            State::Reset => {
+                set_pll_reset(g, true);
+                unsafe {
+                    INFO[g].frame_size = (0, 0);
+                }
+                State::ExitReset
+            }
+            State::ExitReset => {
+                if get_pll_reset(g) {
+                    set_pll_reset(g, false);
+                    State::Lock
+                } else {
+                    State::ExitReset
+                }
+            }
+            State::Lock => {
+                if pll_locked(g) {
+                    info!("grabber{} locked: {}MHz", g, get_video_clock(g));
+                    State::Align
+                } else {
+                    State::Lock
+                }
+            }
+            State::Align => {
+                if pll_locked(g) {
+                    if clock_align(g) {
+                        info!("grabber{} alignment success", g);
+                        State::Watch
+                    } else {
+                        info!("grabber{} alignment failure", g);
+                        State::Reset
+                    }
+                } else {
+                    info!("grabber{} lock lost", g);
+                    State::Reset
+                }
+            }
+            State::Watch => {
+                if pll_locked(g) {
+                    if clock_pattern_ok(g) {
+                        let last_xy = get_last_pixels(g);
+                        if last_xy != unsafe { INFO[g].frame_size } {
+                            // x capture is on ~LVAL which is after
+                            // the last increment on DVAL
+                            // y capture is on ~FVAL which coincides with the
+                            // last increment on ~LVAL
+                            info!("grabber{} frame size: {}x{}", g, last_xy.0, last_xy.1 + 1);
+                            unsafe { INFO[g].frame_size = last_xy }
+                        }
+                        State::Watch
+                    } else {
+                        info!("grabber{} alignment lost", g);
+                        State::Reset
+                    }
+                } else {
+                    info!("grabber{} lock lost", g);
+                    State::Reset
+                }
+            }
+        };
+        unsafe {
+            INFO[g].state = next;
+        }
+    }
+}

src/libboard_artiq/src/io_expander.rs

@@ -0,0 +1,175 @@
+use libboard_zynq::i2c;
+use log::info;
+
+use crate::pl::csr;
+
+// Only the bare minimum registers. Bits/IO connections equivalent between IC types.
+struct Registers {
+    // PCA9539 equivalent register names in comments
+    iodira: u8, // Configuration Port 0
+    iodirb: u8, // Configuration Port 1
+    gpioa: u8,  // Output Port 0
+    gpiob: u8,  // Output Port 1
+}
+
+//IO expanders pins
+const IODIR_OUT_SFP_TX_DISABLE: u8 = 0x02;
+const IODIR_OUT_SFP_LED: u8 = 0x40;
+#[cfg(hw_rev = "v1.0")]
+const IODIR_OUT_SFP0_LED: u8 = 0x40;
+#[cfg(hw_rev = "v1.1")]
+const IODIR_OUT_SFP0_LED: u8 = 0x80;
+
+//IO expander port direction
+const IODIR0: [u8; 2] = [
+    0xFF & !IODIR_OUT_SFP_TX_DISABLE & !IODIR_OUT_SFP0_LED,
+    0xFF & !IODIR_OUT_SFP_TX_DISABLE & !IODIR_OUT_SFP_LED,
+];
+
+const IODIR1: [u8; 2] = [
+    0xFF & !IODIR_OUT_SFP_TX_DISABLE & !IODIR_OUT_SFP_LED,
+    0xFF & !IODIR_OUT_SFP_TX_DISABLE & !IODIR_OUT_SFP_LED,
+];
+
+pub struct IoExpander {
+    address: u8,
+    virtual_led_mapping: &'static [(u8, u8, u8)],
+    iodir: [u8; 2],
+    out_current: [u8; 2],
+    out_target: [u8; 2],
+    registers: Registers,
+}
+
+impl IoExpander {
+    pub fn new(i2c: &mut i2c::I2c, index: u8) -> Result<Self, &'static str> {
+        #[cfg(hw_rev = "v1.0")]
+        const VIRTUAL_LED_MAPPING0: [(u8, u8, u8); 2] = [(0, 0, 6), (1, 1, 6)];
+        #[cfg(hw_rev = "v1.1")]
+        const VIRTUAL_LED_MAPPING0: [(u8, u8, u8); 2] = [(0, 0, 7), (1, 1, 6)];
+
+        const VIRTUAL_LED_MAPPING1: [(u8, u8, u8); 2] = [(2, 0, 6), (3, 1, 6)];
+
+        // Both expanders on SHARED I2C bus
+        let mut io_expander = match index {
+            0 => IoExpander {
+                address: 0x40,
+                virtual_led_mapping: &VIRTUAL_LED_MAPPING0,
+                iodir: IODIR0,
+                out_current: [0; 2],
+                out_target: [0; 2],
+                registers: Registers {
+                    iodira: 0x00,
+                    iodirb: 0x01,
+                    gpioa: 0x12,
+                    gpiob: 0x13,
+                },
+            },
+            1 => IoExpander {
+                address: 0x42,
+                virtual_led_mapping: &VIRTUAL_LED_MAPPING1,
+                iodir: IODIR1,
+                out_current: [0; 2],
+                out_target: [0; 2],
+                registers: Registers {
+                    iodira: 0x00,
+                    iodirb: 0x01,
+                    gpioa: 0x12,
+                    gpiob: 0x13,
+                },
+            },
+            _ => return Err("incorrect I/O expander index"),
+        };
+        if !io_expander.check_ack(i2c)? {
+            info!("MCP23017 io expander {} not found. Checking for PCA9539.", index);
+            io_expander.address += 0xa8; // translate to PCA9539 addresses (see schematic)
+            io_expander.registers = Registers {
+                iodira: 0x06,
+                iodirb: 0x07,
+                gpioa: 0x02,
+                gpiob: 0x03,
+            };
+            if !io_expander.check_ack(i2c)? {
+                return Err("Neither MCP23017 nor PCA9539 io expander found.");
+            };
+        }
+        Ok(io_expander)
+    }
+
+    fn select(&self, i2c: &mut i2c::I2c) -> Result<(), &'static str> {
+        i2c.pca954x_select(0x70, None)?;
+        i2c.pca954x_select(0x71, Some(3))?;
+        Ok(())
+    }
+
+    fn write(&self, i2c: &mut i2c::I2c, addr: u8, value: u8) -> Result<(), &'static str> {
+        i2c.start()?;
+        i2c.write(self.address)?;
+        i2c.write(addr)?;
+        i2c.write(value)?;
+        i2c.stop()?;
+        Ok(())
+    }
+
+    fn check_ack(&self, i2c: &mut i2c::I2c) -> Result<bool, &'static str> {
+        // Check for ack from io expander
+        self.select(i2c)?;
+        i2c.start()?;
+        let ack = i2c.write(self.address)?;
+        i2c.stop()?;
+        Ok(ack)
+    }
+
+    fn update_iodir(&self, i2c: &mut i2c::I2c) -> Result<(), &'static str> {
+        self.write(i2c, self.registers.iodira, self.iodir[0])?;
+        self.write(i2c, self.registers.iodirb, self.iodir[1])?;
+        Ok(())
+    }
+
+    pub fn init(&mut self, i2c: &mut i2c::I2c) -> Result<(), &'static str> {
+        self.select(i2c)?;
+
+        self.update_iodir(i2c)?;
+
+        self.out_current[0] = 0x00;
+        self.write(i2c, self.registers.gpioa, 0x00)?;
+        self.out_current[1] = 0x00;
+        self.write(i2c, self.registers.gpiob, 0x00)?;
+        Ok(())
+    }
+
+    pub fn set_oe(&mut self, i2c: &mut i2c::I2c, port: u8, outputs: u8) -> Result<(), &'static str> {
+        self.iodir[port as usize] &= !outputs;
+        self.update_iodir(i2c)?;
+        Ok(())
+    }
+
+    pub fn set(&mut self, port: u8, bit: u8, high: bool) {
+        if high {
+            self.out_target[port as usize] |= 1 << bit;
+        } else {
+            self.out_target[port as usize] &= !(1 << bit);
+        }
+    }
+
+    pub fn service(&mut self, i2c: &mut i2c::I2c) -> Result<(), &'static str> {
+        #[cfg(has_virtual_leds)]
+        for (led, port, bit) in self.virtual_led_mapping.iter() {
+            let level = unsafe { csr::virtual_leds::status_read() >> led & 1 };
+            self.set(*port, *bit, level != 0);
+        }
+
+        if self.out_target != self.out_current {
+            self.select(i2c)?;
+            if self.out_target[0] != self.out_current[0] {
+                self.write(i2c, self.registers.gpioa, self.out_target[0])?;
+                self.out_current[0] = self.out_target[0];
+            }
+            if self.out_target[1] != self.out_current[1] {
+                self.write(i2c, self.registers.gpiob, self.out_target[1])?;
+                self.out_current[1] = self.out_target[1];
+            }
+        }
+
+        Ok(())
+    }
+}

src/libboard_artiq/src/lib.rs

@@ -1,36 +1,42 @@
 #![no_std]
 #![feature(never_type)]
 
-extern crate log;
+extern crate core_io;
 extern crate crc;
 extern crate embedded_hal;
-extern crate core_io;
 extern crate io;
+extern crate libasync;
 extern crate libboard_zynq;
-extern crate libregister;
 extern crate libconfig;
 extern crate libcortex_a9;
-extern crate libasync;
+extern crate libregister;
+extern crate log;
 extern crate log_buffer;
 
-#[path = "../../../build/pl.rs"]
-pub mod pl;
-pub mod drtioaux_proto;
 pub mod drtio_routing;
-pub mod logger;
-#[cfg(has_si5324)]
-pub mod si5324;
 #[cfg(has_drtio)]
 pub mod drtioaux;
 #[cfg(has_drtio)]
 pub mod drtioaux_async;
+pub mod drtioaux_proto;
+#[cfg(all(feature = "target_kasli_soc", has_drtio))]
+pub mod io_expander;
+pub mod logger;
 #[cfg(has_drtio)]
+#[rustfmt::skip]
 #[path = "../../../build/mem.rs"]
 pub mod mem;
+#[rustfmt::skip]
+#[path = "../../../build/pl.rs"]
+pub mod pl;
+#[cfg(has_drtio_eem)]
+pub mod drtio_eem;
+#[cfg(has_grabber)]
+pub mod grabber;
+#[cfg(has_si5324)]
+pub mod si5324;
 
 use core::{cmp, str};
-use libboard_zynq::slcr;
-use libregister::RegisterW;
 
 pub fn identifier_read(buf: &mut [u8]) -> &str {
     unsafe {
@@ -44,26 +50,3 @@ pub fn identifier_read(buf: &mut [u8]) -> &str {
         str::from_utf8_unchecked(&buf[..len as usize])
     }
 }
-
-pub fn init_gateware() {
-    // Set up PS->PL clocks
-    slcr::RegisterBlock::unlocked(|slcr| {
-        // As we are touching the mux, the clock may glitch, so reset the PL.
-        slcr.fpga_rst_ctrl.write(
-            slcr::FpgaRstCtrl::zeroed()
-                .fpga0_out_rst(true)
-                .fpga1_out_rst(true)
-                .fpga2_out_rst(true)
-                .fpga3_out_rst(true)
-        );
-        slcr.fpga0_clk_ctrl.write(
-            slcr::Fpga0ClkCtrl::zeroed()
-                .src_sel(slcr::PllSource::IoPll)
-                .divisor0(8)
-                .divisor1(1)
-        );
-        slcr.fpga_rst_ctrl.write(
-            slcr::FpgaRstCtrl::zeroed()
-        );
-    });
-}

src/libboard_artiq/src/logger.rs

@@ -1,13 +1,13 @@
-use core::cell::Cell;
-use core::fmt::Write;
-use log::{Log, LevelFilter};
-use log_buffer::LogBuffer;
-use libcortex_a9::mutex::{Mutex, MutexGuard};
+use core::{cell::Cell, fmt::Write};
+
 use libboard_zynq::{println, timer::GlobalTimer};
+use libcortex_a9::mutex::{Mutex, MutexGuard};
+use log::{LevelFilter, Log};
+use log_buffer::LogBuffer;
 
 pub struct LogBufferRef<'a> {
-    buffer:        MutexGuard<'a, LogBuffer<&'static mut [u8]>>,
-    old_log_level: LevelFilter
+    buffer: MutexGuard<'a, LogBuffer<&'static mut [u8]>>,
+    old_log_level: LevelFilter,
 }
 
 impl<'a> LogBufferRef<'a> {
@@ -37,7 +37,7 @@ impl<'a> Drop for LogBufferRef<'a> {
 }
 
 pub struct BufferLogger {
-    buffer:      Mutex<LogBuffer<&'static mut [u8]>>,
+    buffer: Mutex<LogBuffer<&'static mut [u8]>>,
     uart_filter: Cell<LevelFilter>,
     buffer_filter: Cell<LevelFilter>,
 }
@@ -56,8 +56,7 @@ impl BufferLogger {
     pub fn register(self) {
         unsafe {
             LOGGER = Some(self);
-            log::set_logger(LOGGER.as_ref().unwrap())
-                .expect("global logger can only be initialized once");
+            log::set_logger(LOGGER.as_ref().unwrap()).expect("global logger can only be initialized once");
         }
     }
 
@@ -66,9 +65,7 @@ impl BufferLogger {
     }
 
     pub fn buffer<'a>(&'a self) -> Option<LogBufferRef<'a>> {
-        self.buffer
-            .try_lock()
-            .map(LogBufferRef::new)
+        self.buffer.try_lock().map(LogBufferRef::new)
     }
 
     pub fn uart_log_level(&self) -> LevelFilter {
@@ -99,25 +96,36 @@ impl Log for BufferLogger {
 
     fn log(&self, record: &log::Record) {
         if self.enabled(record.metadata()) {
-            let timestamp = unsafe {
-                GlobalTimer::get()
-            }.get_us().0;
-            let seconds   = timestamp / 1_000_000;
-            let micros    = timestamp % 1_000_000;
+            let timestamp = unsafe { GlobalTimer::get() }.get_us().0;
+            let seconds = timestamp / 1_000_000;
+            let micros = timestamp % 1_000_000;
 
             if record.level() <= self.buffer_log_level() {
                 let mut buffer = self.buffer.lock();
-                writeln!(buffer, "[{:6}.{:06}s] {:>5}({}): {}", seconds, micros,
-                         record.level(), record.target(), record.args()).unwrap();
+                writeln!(
+                    buffer,
+                    "[{:6}.{:06}s] {:>5}({}): {}",
+                    seconds,
+                    micros,
+                    record.level(),
+                    record.target(),
+                    record.args()
+                )
+                .unwrap();
             }
 
             if record.level() <= self.uart_log_level() {
-                println!("[{:6}.{:06}s] {:>5}({}): {}", seconds, micros,
-                         record.level(), record.target(), record.args());
+                println!(
+                    "[{:6}.{:06}s] {:>5}({}): {}",
+                    seconds,
+                    micros,
+                    record.level(),
+                    record.target(),
+                    record.args()
+                );
             }
         }
     }
 
-    fn flush(&self) {
-    }
+    fn flush(&self) {}
 }

src/libboard_artiq/src/si5324.rs

@@ -1,7 +1,9 @@
 use core::result;
-use log::info;
-use libboard_zynq::{i2c::I2c, timer::GlobalTimer, time::Milliseconds};
+
 use embedded_hal::blocking::delay::DelayUs;
+use libboard_zynq::{i2c::I2c, time::Milliseconds, timer::GlobalTimer};
+use log::info;
+
 #[cfg(not(si5324_soft_reset))]
 use crate::pl::csr;
 
@@ -11,9 +13,13 @@ const ADDRESS: u8 = 0x68;
 
 #[cfg(not(si5324_soft_reset))]
 fn hard_reset(timer: &mut GlobalTimer) {
-    unsafe { csr::si5324_rst_n::out_write(0); }
+    unsafe {
+        csr::si5324_rst_n::out_write(0);
+    }
     timer.delay_us(1_000);
-    unsafe { csr::si5324_rst_n::out_write(1); }
+    unsafe {
+        csr::si5324_rst_n::out_write(1);
+    }
     timer.delay_us(10_000);
 }
 
@@ -29,7 +35,7 @@ pub struct FrequencySettings {
     pub n31: u32,
     pub n32: u32,
     pub bwsel: u8,
-    pub crystal_ref: bool
+    pub crystal_as_ckin2: bool,
 }
 
 pub enum Input {
@@ -39,52 +45,52 @@ pub enum Input {
 
 fn map_frequency_settings(settings: &FrequencySettings) -> Result<FrequencySettings> {
     if settings.nc1_ls != 0 && (settings.nc1_ls % 2) == 1 {
-        return Err("NC1_LS must be 0 or even")
+        return Err("NC1_LS must be 0 or even");
     }
     if settings.nc1_ls > (1 << 20) {
-        return Err("NC1_LS is too high")
+        return Err("NC1_LS is too high");
     }
     if (settings.n2_ls % 2) == 1 {
-        return Err("N2_LS must be even")
+        return Err("N2_LS must be even");
     }
     if settings.n2_ls > (1 << 20) {
-        return Err("N2_LS is too high")
+        return Err("N2_LS is too high");
     }
     if settings.n31 > (1 << 19) {
-        return Err("N31 is too high")
+        return Err("N31 is too high");
     }
     if settings.n32 > (1 << 19) {
-        return Err("N32 is too high")
+        return Err("N32 is too high");
     }
     let r = FrequencySettings {
         n1_hs: match settings.n1_hs {
-            4  => 0b000,
-            5  => 0b001,
-            6  => 0b010,
-            7  => 0b011,
-            8  => 0b100,
-            9  => 0b101,
+            4 => 0b000,
+            5 => 0b001,
+            6 => 0b010,
+            7 => 0b011,
+            8 => 0b100,
+            9 => 0b101,
             10 => 0b110,
             11 => 0b111,
-            _  => return Err("N1_HS has an invalid value")
+            _ => return Err("N1_HS has an invalid value"),
         },
         nc1_ls: settings.nc1_ls - 1,
         n2_hs: match settings.n2_hs {
-            4  => 0b000,
-            5  => 0b001,
-            6  => 0b010,
-            7  => 0b011,
-            8  => 0b100,
-            9  => 0b101,
+            4 => 0b000,
+            5 => 0b001,
+            6 => 0b010,
+            7 => 0b011,
+            8 => 0b100,
+            9 => 0b101,
             10 => 0b110,
             11 => 0b111,
-            _  => return Err("N2_HS has an invalid value")
+            _ => return Err("N2_HS has an invalid value"),
         },
         n2_ls: settings.n2_ls - 1,
         n31: settings.n31 - 1,
         n32: settings.n32 - 1,
         bwsel: settings.bwsel,
-        crystal_ref: settings.crystal_ref
+        crystal_as_ckin2: settings.crystal_as_ckin2,
     };
     Ok(r)
 }
@@ -92,13 +98,13 @@ fn map_frequency_settings(settings: &FrequencySettings) -> Result<FrequencySetti
 fn write(i2c: &mut I2c, reg: u8, val: u8) -> Result<()> {
     i2c.start().unwrap();
     if !i2c.write(ADDRESS << 1).unwrap() {
-        return Err("Si5324 failed to ack write address")
+        return Err("Si5324 failed to ack write address");
     }
     if !i2c.write(reg).unwrap() {
-        return Err("Si5324 failed to ack register")
+        return Err("Si5324 failed to ack register");
     }
     if !i2c.write(val).unwrap() {
-        return Err("Si5324 failed to ack value")
+        return Err("Si5324 failed to ack value");
     }
     i2c.stop().unwrap();
     Ok(())
@@ -108,10 +114,10 @@ fn write(i2c: &mut I2c, reg: u8, val: u8) -> Result<()> {
 fn write_no_ack_value(i2c: &mut I2c, reg: u8, val: u8) -> Result<()> {
     i2c.start().unwrap();
     if !i2c.write(ADDRESS << 1).unwrap() {
-        return Err("Si5324 failed to ack write address")
+        return Err("Si5324 failed to ack write address");
     }
     if !i2c.write(reg).unwrap() {
-        return Err("Si5324 failed to ack register")
+        return Err("Si5324 failed to ack register");
     }
     i2c.write(val).unwrap();
     i2c.stop().unwrap();
@@ -121,22 +127,22 @@ fn write_no_ack_value(i2c: &mut I2c, reg: u8, val: u8) -> Result<()> {
 fn read(i2c: &mut I2c, reg: u8) -> Result<u8> {
     i2c.start().unwrap();
     if !i2c.write(ADDRESS << 1).unwrap() {
-        return Err("Si5324 failed to ack write address")
+        return Err("Si5324 failed to ack write address");
     }
     if !i2c.write(reg).unwrap() {
-        return Err("Si5324 failed to ack register")
+        return Err("Si5324 failed to ack register");
     }
     i2c.restart().unwrap();
     if !i2c.write((ADDRESS << 1) | 1).unwrap() {
-        return Err("Si5324 failed to ack read address")
+        return Err("Si5324 failed to ack read address");
     }
     let val = i2c.read(false).unwrap();
     i2c.stop().unwrap();
     Ok(val)
 }
 
-fn rmw<F>(i2c: &mut I2c, reg: u8, f: F) -> Result<()> where
-        F: Fn(u8) -> u8 {
+fn rmw<F>(i2c: &mut I2c, reg: u8, f: F) -> Result<()>
+where F: Fn(u8) -> u8 {
     let value = read(i2c, reg)?;
     write(i2c, reg, f(value))?;
     Ok(())
@@ -155,18 +161,18 @@ fn soft_reset(i2c: &mut I2c, timer: &mut GlobalTimer) -> Result<()> {
 }
 
 fn has_xtal(i2c: &mut I2c) -> Result<bool> {
-    Ok((read(i2c, 129)? & 0x01) == 0)  // LOSX_INT=0
+    Ok((read(i2c, 129)? & 0x01) == 0) // LOSX_INT=0
 }
 
 fn has_ckin(i2c: &mut I2c, input: Input) -> Result<bool> {
     match input {
-        Input::Ckin1 => Ok((read(i2c, 129)? & 0x02) == 0),  // LOS1_INT=0
-        Input::Ckin2 => Ok((read(i2c, 129)? & 0x04) == 0),  // LOS2_INT=0
+        Input::Ckin1 => Ok((read(i2c, 129)? & 0x02) == 0), // LOS1_INT=0
+        Input::Ckin2 => Ok((read(i2c, 129)? & 0x04) == 0), // LOS2_INT=0
     }
 }
 
 fn locked(i2c: &mut I2c) -> Result<bool> {
-    Ok((read(i2c, 130)? & 0x01) == 0)  // LOL_INT=0
+    Ok((read(i2c, 130)? & 0x01) == 0) // LOL_INT=0
 }
 
 fn monitor_lock(i2c: &mut I2c, timer: &mut GlobalTimer) -> Result<()> {
@@ -211,11 +217,11 @@ pub fn bypass(i2c: &mut I2c, input: Input, timer: &mut GlobalTimer) -> Result<()
         Input::Ckin2 => 0b01,
     };
     init(i2c, timer)?;
-    rmw(i2c, 21,  |v| v & 0xfe)?;                        // CKSEL_PIN=0
-    rmw(i2c, 3,   |v| (v & 0x3f) | (cksel_reg << 6))?;   // CKSEL_REG
-    rmw(i2c, 4,   |v| (v & 0x3f) | (0b00 << 6))?;        // AUTOSEL_REG=b00
-    rmw(i2c, 6,   |v| (v & 0xc0) | 0b111111)?;           // SFOUT2_REG=b111 SFOUT1_REG=b111
-    rmw(i2c, 0,   |v| (v & 0xfd) | 0x02)?;               // BYPASS_REG=1
+    rmw(i2c, 21, |v| v & 0xfe)?; // CKSEL_PIN=0
+    rmw(i2c, 3, |v| (v & 0x3f) | (cksel_reg << 6))?; // CKSEL_REG
+    rmw(i2c, 4, |v| (v & 0x3f) | (0b00 << 6))?; // AUTOSEL_REG=b00
+    rmw(i2c, 6, |v| (v & 0xc0) | 0b111111)?; // SFOUT2_REG=b111 SFOUT1_REG=b111
+    rmw(i2c, 0, |v| (v & 0xfd) | 0x02)?; // BYPASS_REG=1
     Ok(())
 }
 
@@ -227,32 +233,32 @@ pub fn setup(i2c: &mut I2c, settings: &FrequencySettings, input: Input, timer: &
     };
 
     init(i2c, timer)?;
-    if settings.crystal_ref {
-        rmw(i2c, 0,   |v| v | 0x40)?;                     // FREE_RUN=1
+    if settings.crystal_as_ckin2 {
+        rmw(i2c, 0, |v| v | 0x40)?; // FREE_RUN=1
     }
-    rmw(i2c, 2,   |v| (v & 0x0f) | (s.bwsel << 4))?;
-    rmw(i2c, 21,  |v| v & 0xfe)?;                        // CKSEL_PIN=0
-    rmw(i2c, 3,   |v| (v & 0x2f) | (cksel_reg << 6) | 0x10)?;  // CKSEL_REG, SQ_ICAL=1
-    rmw(i2c, 4,   |v| (v & 0x3f) | (0b00 << 6))?;        // AUTOSEL_REG=b00
-    rmw(i2c, 6,   |v| (v & 0xc0) | 0b111111)?;           // SFOUT2_REG=b111 SFOUT1_REG=b111
-    write(i2c, 25,  (s.n1_hs  << 5 ) as u8)?;
-    write(i2c, 31,  (s.nc1_ls >> 16) as u8)?;
-    write(i2c, 32,  (s.nc1_ls >> 8 ) as u8)?;
-    write(i2c, 33,  (s.nc1_ls)       as u8)?;
-    write(i2c, 34,  (s.nc1_ls >> 16) as u8)?;            // write to NC2_LS as well
-    write(i2c, 35,  (s.nc1_ls >> 8 ) as u8)?;
-    write(i2c, 36,  (s.nc1_ls)       as u8)?;
-    write(i2c, 40,  (s.n2_hs  << 5 ) as u8 | (s.n2_ls  >> 16) as u8)?;
-    write(i2c, 41,  (s.n2_ls  >> 8 ) as u8)?;
-    write(i2c, 42,  (s.n2_ls)        as u8)?;
-    write(i2c, 43,  (s.n31    >> 16) as u8)?;
-    write(i2c, 44,  (s.n31    >> 8)  as u8)?;
-    write(i2c, 45,  (s.n31)          as u8)?;
-    write(i2c, 46,  (s.n32    >> 16) as u8)?;
-    write(i2c, 47,  (s.n32    >> 8)  as u8)?;
-    write(i2c, 48,  (s.n32)          as u8)?;
-    rmw(i2c, 137, |v| v | 0x01)?;                       // FASTLOCK=1
-    rmw(i2c, 136, |v| v | 0x40)?;                       // ICAL=1
+    rmw(i2c, 2, |v| (v & 0x0f) | (s.bwsel << 4))?;
+    rmw(i2c, 21, |v| v & 0xfe)?; // CKSEL_PIN=0
+    rmw(i2c, 3, |v| (v & 0x2f) | (cksel_reg << 6) | 0x10)?; // CKSEL_REG, SQ_ICAL=1
+    rmw(i2c, 4, |v| (v & 0x3f) | (0b00 << 6))?; // AUTOSEL_REG=b00
+    rmw(i2c, 6, |v| (v & 0xc0) | 0b111111)?; // SFOUT2_REG=b111 SFOUT1_REG=b111
+    write(i2c, 25, (s.n1_hs << 5) as u8)?;
+    write(i2c, 31, (s.nc1_ls >> 16) as u8)?;
+    write(i2c, 32, (s.nc1_ls >> 8) as u8)?;
+    write(i2c, 33, (s.nc1_ls) as u8)?;
+    write(i2c, 34, (s.nc1_ls >> 16) as u8)?; // write to NC2_LS as well
+    write(i2c, 35, (s.nc1_ls >> 8) as u8)?;
+    write(i2c, 36, (s.nc1_ls) as u8)?;
+    write(i2c, 40, (s.n2_hs << 5) as u8 | (s.n2_ls >> 16) as u8)?;
+    write(i2c, 41, (s.n2_ls >> 8) as u8)?;
+    write(i2c, 42, (s.n2_ls) as u8)?;
+    write(i2c, 43, (s.n31 >> 16) as u8)?;
+    write(i2c, 44, (s.n31 >> 8) as u8)?;
+    write(i2c, 45, (s.n31) as u8)?;
+    write(i2c, 46, (s.n32 >> 16) as u8)?;
+    write(i2c, 47, (s.n32 >> 8) as u8)?;
+    write(i2c, 48, (s.n32) as u8)?;
+    rmw(i2c, 137, |v| v | 0x01)?; // FASTLOCK=1
+    rmw(i2c, 136, |v| v | 0x40)?; // ICAL=1
 
     if !has_xtal(i2c)? {
         return Err("Si5324 misses XA/XB signal");
@@ -270,7 +276,7 @@ pub fn select_input(i2c: &mut I2c, input: Input, timer: &mut GlobalTimer) -> Res
         Input::Ckin1 => 0b00,
         Input::Ckin2 => 0b01,
     };
-    rmw(i2c, 3,   |v| (v & 0x3f) | (cksel_reg << 6))?;
+    rmw(i2c, 3, |v| (v & 0x3f) | (cksel_reg << 6))?;
     if !has_ckin(i2c, input)? {
         return Err("Si5324 misses clock input signal");
     }
@@ -285,12 +291,12 @@ pub mod siphaser {
 
     pub fn select_recovered_clock(i2c: &mut I2c, rc: bool, timer: &mut GlobalTimer) -> Result<()> {
         let val = read(i2c, 3)?;
-        write(i2c, 3,   (val & 0xdf) | (1 << 5))?;  // DHOLD=1
+        write(i2c, 3, (val & 0xdf) | (1 << 5))?; // DHOLD=1
         unsafe {
             csr::siphaser::switch_clocks_write(if rc { 1 } else { 0 });
         }
         let val = read(i2c, 3)?;
-        write(i2c, 3,   (val & 0xdf) | (0 << 5))?;  // DHOLD=0
+        write(i2c, 3, (val & 0xdf) | (0 << 5))?; // DHOLD=0
         monitor_lock(i2c, timer)?;
         Ok(())
     }
@@ -309,9 +315,7 @@ pub mod siphaser {
             csr::siphaser::error_write(1);
         }
         timer.delay_us(5_000);
-        unsafe {
-            csr::siphaser::error_read() != 0
-        }
+        unsafe { csr::siphaser::error_read() != 0 }
     }
 
     fn find_edge(target: bool, timer: &mut GlobalTimer) -> Result<u32> {
@@ -340,11 +344,16 @@ pub mod siphaser {
         }
         let width = find_edge(true, timer)? + jitter_margin;
         // width is 360 degrees (one full rotation of the phase between s/h limits) minus jitter
-        info!("calibration successful, lead: {}, width: {} ({}deg)", lead, width, width*360/(56*8));
+        info!(
+            "calibration successful, lead: {}, width: {} ({}deg)",
+            lead,
+            width,
+            width * 360 / (56 * 8)
+        );
 
         // Apply reverse phase shift for half the width to get into the
         // middle of the working region.
-        for _ in 0..width/2 {
+        for _ in 0..width / 2 {
             phase_shift(0, timer);
         }
 

src/libbuild_zynq/lib.rs

@@ -1,8 +1,7 @@
-use std::env;
-use std::fs::File;
-use std::io::{BufRead, BufReader, Write};
-use std::path::PathBuf;
-
+use std::{env,
+          fs::File,
+          io::{BufRead, BufReader, Write},
+          path::PathBuf};
 
 pub fn add_linker_script() {
     // Put the linker script somewhere the linker can find it

src/libc/Cargo.toml.tpl

@@ -6,7 +6,7 @@ edition = "2018"
 build = "build.rs"
 
 [dependencies]
-libboard_zynq = { git = "https://git.m-labs.hk/M-Labs/zynq-rs.git" }
+libboard_zynq = { path = "@@ZYNQ_RS@@/libboard_zynq" }
 
 [build-dependencies]
 cc = { version = "1.0.1" }

src/libc/build.rs

@@ -4,8 +4,7 @@ fn main() {
 }
 
 mod libc {
-    use std::path::Path;
-    use std::env;
+    use std::{env, path::Path};
 
     pub fn compile() {
         let cfg = &mut cc::Build::new();
@@ -32,9 +31,7 @@ mod libc {
         cfg.flag("-U_FORTIFY_SOURCE");
         cfg.define("_FORTIFY_SOURCE", Some("0"));
 
-        let sources = vec![
-            "printf.c"
-        ];
+        let sources = vec!["printf.c"];
 
         let root = Path::new("./");
         for src in sources {

src/libdwarf/src/eh.rs

@@ -11,10 +11,12 @@
 #![allow(non_upper_case_globals)]
 #![allow(unused)]
 
-use crate::DwarfReader;
 use core::mem;
+
 use cslice::CSlice;
 
+use crate::DwarfReader;
+
 pub const DW_EH_PE_omit: u8 = 0xFF;
 pub const DW_EH_PE_absptr: u8 = 0x00;
 
@@ -160,15 +162,11 @@ pub unsafe fn find_eh_action(
                         if ar_filter == 0 {
                             saw_cleanup = true;
                         } else if ar_filter > 0 {
-                            let catch_type = get_ttype_entry(
-                                ar_filter as usize,
-                                ttype_encoding,
-                                ttype_base,
-                                ttype_table,
-                            )?;
+                            let catch_type =
+                                get_ttype_entry(ar_filter as usize, ttype_encoding, ttype_base, ttype_table)?;
                             match catch_type {
                                 Some(clause_ptr) if *(clause_ptr as *const u32) == id => {
-                                    return Ok(EHAction::Catch(lpad))
+                                    return Ok(EHAction::Catch(lpad));
                                 }
                                 None => return Ok(EHAction::Catch(lpad)),
                                 _ => {}
@@ -251,19 +249,11 @@ fn get_base(encoding: u8, context: &EHContext<'_>) -> Result<usize, ()> {
     }
 }
 
-unsafe fn read_encoded_pointer(
-    reader: &mut DwarfReader,
-    context: &EHContext<'_>,
-    encoding: u8,
-) -> Result<usize, ()> {
+unsafe fn read_encoded_pointer(reader: &mut DwarfReader, context: &EHContext<'_>, encoding: u8) -> Result<usize, ()> {
     read_encoded_pointer_with_base(reader, encoding, get_base(encoding, context)?)
 }
 
-unsafe fn read_encoded_pointer_with_base(
-    reader: &mut DwarfReader,
-    encoding: u8,
-    base: usize,
-) -> Result<usize, ()> {
+unsafe fn read_encoded_pointer_with_base(reader: &mut DwarfReader, encoding: u8, base: usize) -> Result<usize, ()> {
     if encoding == DW_EH_PE_omit {
         return Err(());
     }

src/libdyld/Cargo.toml.tpl

@@ -8,4 +8,4 @@ name = "dyld"
 
 [dependencies]
 log = "0.4"
-libcortex_a9 = { git = "https://git.m-labs.hk/M-Labs/zynq-rs.git" }
+libcortex_a9 = { path = "@@ZYNQ_RS@@/libcortex_a9" }

src/libdyld/src/elf.rs

@@ -1451,8 +1451,7 @@ pub const R_AARCH64_TLSDESC_CALL: usize = 569;
 pub const R_AARCH64_TLSLE_LDST128_TPREL_LO12: usize = 570;
 pub const R_AARCH64_TLSLE_LDST128_TPREL_LO12_NC: usize = 571;
 pub const R_AARCH64_TLSLD_LDST128_DTPREL_LO12: usize = 572;
-pub const R_AARCH64_TLSLD_LDST128_DTPREL_LO12_NC: usize =
-    573;
+pub const R_AARCH64_TLSLD_LDST128_DTPREL_LO12_NC: usize = 573;
 pub const R_AARCH64_COPY: usize = 1024;
 pub const R_AARCH64_GLOB_DAT: usize = 1025;
 pub const R_AARCH64_JUMP_SLOT: usize = 1026;
@@ -2267,7 +2266,9 @@ pub struct Elf32_Ehdr {
     pub e_shstrndx: Elf32_Half,
 }
 impl Clone for Elf32_Ehdr {
-    fn clone(&self) -> Self { *self }
+    fn clone(&self) -> Self {
+        *self
+    }
 }
 #[repr(C)]
 #[derive(Debug, Copy)]
@@ -2288,7 +2289,9 @@ pub struct Elf64_Ehdr {
     pub e_shstrndx: Elf64_Half,
 }
 impl Clone for Elf64_Ehdr {
-    fn clone(&self) -> Self { *self }
+    fn clone(&self) -> Self {
+        *self
+    }
 }
 #[repr(C)]
 #[derive(Debug, Copy)]
@@ -2305,7 +2308,9 @@ pub struct Elf32_Shdr {
     pub sh_entsize: Elf32_Word,
 }
 impl Clone for Elf32_Shdr {
-    fn clone(&self) -> Self { *self }
+    fn clone(&self) -> Self {
+        *self
+    }
 }
 #[repr(C)]
 #[derive(Debug, Copy)]
@@ -2322,7 +2327,9 @@ pub struct Elf64_Shdr {
     pub sh_entsize: Elf64_Xword,
 }
 impl Clone for Elf64_Shdr {
-    fn clone(&self) -> Self { *self }
+    fn clone(&self) -> Self {
+        *self
+    }
 }
 #[repr(C)]
 #[derive(Debug, Copy)]
@@ -2335,7 +2342,9 @@ pub struct Elf32_Sym {
     pub st_shndx: Elf32_Section,
 }
 impl Clone for Elf32_Sym {
-    fn clone(&self) -> Self { *self }
+    fn clone(&self) -> Self {
+        *self
+    }
 }
 #[repr(C)]
 #[derive(Debug, Copy)]
@@ -2348,7 +2357,9 @@ pub struct Elf64_Sym {
     pub st_size: Elf64_Xword,
 }
 impl Clone for Elf64_Sym {
-    fn clone(&self) -> Self { *self }
+    fn clone(&self) -> Self {
+        *self
+    }
 }
 #[repr(C)]
 #[derive(Debug, Copy)]
@@ -2357,7 +2368,9 @@ pub struct Elf32_Syminfo {
     pub si_flags: Elf32_Half,
 }
 impl Clone for Elf32_Syminfo {
-    fn clone(&self) -> Self { *self }
+    fn clone(&self) -> Self {
+        *self
+    }
 }
 #[repr(C)]
 #[derive(Debug, Copy)]
@@ -2366,7 +2379,9 @@ pub struct Elf64_Syminfo {
     pub si_flags: Elf64_Half,
 }
 impl Clone for Elf64_Syminfo {
-    fn clone(&self) -> Self { *self }
+    fn clone(&self) -> Self {
+        *self
+    }
 }
 #[repr(C)]
 #[derive(Debug, Copy)]
@@ -2375,7 +2390,9 @@ pub struct Elf32_Rel {
     pub r_info: Elf32_Word,
 }
 impl Clone for Elf32_Rel {
-    fn clone(&self) -> Self { *self }
+    fn clone(&self) -> Self {
+        *self
+    }
 }
 #[repr(C)]
 #[derive(Debug, Copy)]
@@ -2384,7 +2401,9 @@ pub struct Elf64_Rel {
     pub r_info: Elf64_Xword,
 }
 impl Clone for Elf64_Rel {
-    fn clone(&self) -> Self { *self }
+    fn clone(&self) -> Self {
+        *self
+    }
 }
 #[repr(C)]
 #[derive(Debug, Copy)]
@@ -2394,7 +2413,9 @@ pub struct Elf32_Rela {
     pub r_addend: Elf32_Sword,
 }
 impl Clone for Elf32_Rela {
-    fn clone(&self) -> Self { *self }
+    fn clone(&self) -> Self {
+        *self
+    }
 }
 #[repr(C)]
 #[derive(Debug, Copy)]
@@ -2404,7 +2425,9 @@ pub struct Elf64_Rela {
     pub r_addend: Elf64_Sxword,
 }
 impl Clone for Elf64_Rela {
-    fn clone(&self) -> Self { *self }
+    fn clone(&self) -> Self {
+        *self
+    }
 }
 #[repr(C)]
 #[derive(Debug, Copy)]
@@ -2419,7 +2442,9 @@ pub struct Elf32_Phdr {
     pub p_align: Elf32_Word,
 }
 impl Clone for Elf32_Phdr {
-    fn clone(&self) -> Self { *self }
+    fn clone(&self) -> Self {
+        *self
+    }
 }
 #[repr(C)]
 #[derive(Debug, Copy)]
@@ -2434,7 +2459,9 @@ pub struct Elf64_Phdr {
     pub p_align: Elf64_Xword,
 }
 impl Clone for Elf64_Phdr {
-    fn clone(&self) -> Self { *self }
+    fn clone(&self) -> Self {
+        *self
+    }
 }
 #[repr(C)]
 #[derive(Copy)]
@@ -2449,10 +2476,14 @@ pub union Elf32_Dyn__bindgen_ty_1 {
     pub d_ptr: Elf32_Addr,
 }
 impl Clone for Elf32_Dyn__bindgen_ty_1 {
-    fn clone(&self) -> Self { *self }
+    fn clone(&self) -> Self {
+        *self
+    }
 }
 impl Clone for Elf32_Dyn {
-    fn clone(&self) -> Self { *self }
+    fn clone(&self) -> Self {
+        *self
+    }
 }
 #[repr(C)]
 #[derive(Copy)]
@@ -2467,10 +2498,14 @@ pub union Elf64_Dyn__bindgen_ty_1 {
     pub d_ptr: Elf64_Addr,
 }
 impl Clone for Elf64_Dyn__bindgen_ty_1 {
-    fn clone(&self) -> Self { *self }
+    fn clone(&self) -> Self {
+        *self
+    }
 }
 impl Clone for Elf64_Dyn {
-    fn clone(&self) -> Self { *self }
+    fn clone(&self) -> Self {
+        *self
+    }
 }
 #[repr(C)]
 #[derive(Debug, Copy)]
@@ -2484,7 +2519,9 @@ pub struct Elf32_Verdef {
     pub vd_next: Elf32_Word,
 }
 impl Clone for Elf32_Verdef {
-    fn clone(&self) -> Self { *self }
+    fn clone(&self) -> Self {
+        *self
+    }
 }
 #[repr(C)]
 #[derive(Debug, Copy)]
@@ -2498,7 +2535,9 @@ pub struct Elf64_Verdef {
     pub vd_next: Elf64_Word,
 }
 impl Clone for Elf64_Verdef {
-    fn clone(&self) -> Self { *self }
+    fn clone(&self) -> Self {
+        *self
+    }
 }
 #[repr(C)]
 #[derive(Debug, Copy)]
@@ -2507,7 +2546,9 @@ pub struct Elf32_Verdaux {
     pub vda_next: Elf32_Word,
 }
 impl Clone for Elf32_Verdaux {
-    fn clone(&self) -> Self { *self }
+    fn clone(&self) -> Self {
+        *self
+    }
 }
 #[repr(C)]
 #[derive(Debug, Copy)]
@@ -2516,7 +2557,9 @@ pub struct Elf64_Verdaux {
     pub vda_next: Elf64_Word,
 }
 impl Clone for Elf64_Verdaux {
-    fn clone(&self) -> Self { *self }
+    fn clone(&self) -> Self {
+        *self
+    }
 }
 #[repr(C)]
 #[derive(Debug, Copy)]
@@ -2528,7 +2571,9 @@ pub struct Elf32_Verneed {
     pub vn_next: Elf32_Word,
 }
 impl Clone for Elf32_Verneed {
-    fn clone(&self) -> Self { *self }
+    fn clone(&self) -> Self {
+        *self
+    }
 }
 #[repr(C)]
 #[derive(Debug, Copy)]
@@ -2540,7 +2585,9 @@ pub struct Elf64_Verneed {
     pub vn_next: Elf64_Word,
 }
 impl Clone for Elf64_Verneed {
-    fn clone(&self) -> Self { *self }
+    fn clone(&self) -> Self {
+        *self
+    }
 }
 #[repr(C)]
 #[derive(Debug, Copy)]
@@ -2552,7 +2599,9 @@ pub struct Elf32_Vernaux {
     pub vna_next: Elf32_Word,
 }
 impl Clone for Elf32_Vernaux {
-    fn clone(&self) -> Self { *self }
+    fn clone(&self) -> Self {
+        *self
+    }
 }
 #[repr(C)]
 #[derive(Debug, Copy)]
@@ -2564,7 +2613,9 @@ pub struct Elf64_Vernaux {
     pub vna_next: Elf64_Word,
 }
 impl Clone for Elf64_Vernaux {
-    fn clone(&self) -> Self { *self }
+    fn clone(&self) -> Self {
+        *self
+    }
 }
 #[repr(C)]
 #[derive(Copy)]
@@ -2578,10 +2629,14 @@ pub union Elf32_auxv_t__bindgen_ty_1 {
     pub a_val: u32,
 }
 impl Clone for Elf32_auxv_t__bindgen_ty_1 {
-    fn clone(&self) -> Self { *self }
+    fn clone(&self) -> Self {
+        *self
+    }
 }
 impl Clone for Elf32_auxv_t {
-    fn clone(&self) -> Self { *self }
+    fn clone(&self) -> Self {
+        *self
+    }
 }
 #[repr(C)]
 #[derive(Copy)]
@@ -2595,10 +2650,14 @@ pub union Elf64_auxv_t__bindgen_ty_1 {
     pub a_val: u64,
 }
 impl Clone for Elf64_auxv_t__bindgen_ty_1 {
-    fn clone(&self) -> Self { *self }
+    fn clone(&self) -> Self {
+        *self
+    }
 }
 impl Clone for Elf64_auxv_t {
-    fn clone(&self) -> Self { *self }
+    fn clone(&self) -> Self {
+        *self
+    }
 }
 #[repr(C)]
 #[derive(Debug, Copy)]
@@ -2608,7 +2667,9 @@ pub struct Elf32_Nhdr {
     pub n_type: Elf32_Word,
 }
 impl Clone for Elf32_Nhdr {
-    fn clone(&self) -> Self { *self }
+    fn clone(&self) -> Self {
+        *self
+    }
 }
 #[repr(C)]
 #[derive(Debug, Copy)]
@@ -2618,7 +2679,9 @@ pub struct Elf64_Nhdr {
     pub n_type: Elf64_Word,
 }
 impl Clone for Elf64_Nhdr {
-    fn clone(&self) -> Self { *self }
+    fn clone(&self) -> Self {
+        *self
+    }
 }
 #[repr(C)]
 #[derive(Debug, Copy)]
@@ -2630,7 +2693,9 @@ pub struct Elf32_Move {
     pub m_stride: Elf32_Half,
 }
 impl Clone for Elf32_Move {
-    fn clone(&self) -> Self { *self }
+    fn clone(&self) -> Self {
+        *self
+    }
 }
 #[repr(C)]
 #[derive(Debug, Copy)]
@@ -2642,7 +2707,9 @@ pub struct Elf64_Move {
     pub m_stride: Elf64_Half,
 }
 impl Clone for Elf64_Move {
-    fn clone(&self) -> Self { *self }
+    fn clone(&self) -> Self {
+        *self
+    }
 }
 #[repr(C)]
 #[derive(Copy)]
@@ -2657,7 +2724,9 @@ pub struct Elf32_gptab__bindgen_ty_1 {
     pub gt_unused: Elf32_Word,
 }
 impl Clone for Elf32_gptab__bindgen_ty_1 {
-    fn clone(&self) -> Self { *self }
+    fn clone(&self) -> Self {
+        *self
+    }
 }
 #[repr(C)]
 #[derive(Debug, Copy)]
@@ -2666,10 +2735,14 @@ pub struct Elf32_gptab__bindgen_ty_2 {
     pub gt_bytes: Elf32_Word,
 }
 impl Clone for Elf32_gptab__bindgen_ty_2 {
-    fn clone(&self) -> Self { *self }
+    fn clone(&self) -> Self {
+        *self
+    }
 }
 impl Clone for Elf32_gptab {
-    fn clone(&self) -> Self { *self }
+    fn clone(&self) -> Self {
+        *self
+    }
 }
 #[repr(C)]
 #[derive(Debug, Copy)]
@@ -2679,7 +2752,9 @@ pub struct Elf32_RegInfo {
     pub ri_gp_value: Elf32_Sword,
 }
 impl Clone for Elf32_RegInfo {
-    fn clone(&self) -> Self { *self }
+    fn clone(&self) -> Self {
+        *self
+    }
 }
 #[repr(C)]
 #[derive(Debug, Copy)]
@@ -2690,7 +2765,9 @@ pub struct Elf_Options {
     pub info: Elf32_Word,
 }
 impl Clone for Elf_Options {
-    fn clone(&self) -> Self { *self }
+    fn clone(&self) -> Self {
+        *self
+    }
 }
 #[repr(C)]
 #[derive(Debug, Copy)]
@@ -2699,7 +2776,9 @@ pub struct Elf_Options_Hw {
     pub hwp_flags2: Elf32_Word,
 }
 impl Clone for Elf_Options_Hw {
-    fn clone(&self) -> Self { *self }
+    fn clone(&self) -> Self {
+        *self
+    }
 }
 #[repr(C)]
 #[derive(Debug, Copy)]
@@ -2711,7 +2790,9 @@ pub struct Elf32_Lib {
     pub l_flags: Elf32_Word,
 }
 impl Clone for Elf32_Lib {
-    fn clone(&self) -> Self { *self }
+    fn clone(&self) -> Self {
+        *self
+    }
 }
 #[repr(C)]
 #[derive(Debug, Copy)]
@@ -2723,17 +2804,31 @@ pub struct Elf64_Lib {
     pub l_flags: Elf64_Word,
 }
 impl Clone for Elf64_Lib {
-    fn clone(&self) -> Self { *self }
+    fn clone(&self) -> Self {
+        *self
+    }
 }
 pub type Elf32_Conflict = Elf32_Addr;
 #[repr(C)]
 #[derive(Clone, Copy)]
 pub struct EXIDX_Entry(u32, u32);
 
-pub fn ELF32_R_SYM(info: Elf32_Word) -> Elf32_Word { info >> 8 }
-pub fn ELF32_R_TYPE(info: Elf32_Word) -> u8 { info as u8 }
-pub fn ELF32_R_INFO(sym: Elf32_Word, ty: u8) -> Elf32_Word { sym << 8 | ty as Elf32_Word }
+pub fn ELF32_R_SYM(info: Elf32_Word) -> Elf32_Word {
+    info >> 8
+}
+pub fn ELF32_R_TYPE(info: Elf32_Word) -> u8 {
+    info as u8
+}
+pub fn ELF32_R_INFO(sym: Elf32_Word, ty: u8) -> Elf32_Word {
+    sym << 8 | ty as Elf32_Word
+}
 
-pub fn ELF32_ST_BIND(info: u8) -> u8 { info >> 4 }
-pub fn ELF32_ST_TYPE(info: u8) -> u8 { info & 0xf }
-pub fn ELF32_ST_INFO(bind: u8, ty: u8) -> u8 { (bind << 4) | (ty & 0xf) }
+pub fn ELF32_ST_BIND(info: u8) -> u8 {
+    info >> 4
+}
+pub fn ELF32_ST_TYPE(info: u8) -> u8 {
+    info & 0xf
+}
+pub fn ELF32_ST_INFO(bind: u8, ty: u8) -> u8 {
+    (bind << 4) | (ty & 0xf)
+}

src/libdyld/src/file.rs

@@ -1,8 +1,8 @@
-use core::{mem, ptr, ops::{Deref, Range}};
-use super::{
-    Arch,
-    elf::*,
-};
+use core::{mem,
+           ops::{Deref, Range},
+           ptr};
+
+use super::{elf::*, Arch};
 
 fn read_unaligned<T: Copy>(data: &[u8], offset: usize) -> Option<T> {
     if data.len() < offset + mem::size_of::<T>() {
@@ -31,14 +31,40 @@ impl<'a> File<'a> {
 
     pub fn arch(&self) -> Option<Arch> {
         const IDENT_OPENRISC: [u8; EI_NIDENT] = [
-            ELFMAG0,    ELFMAG1,     ELFMAG2,    ELFMAG3,
-            ELFCLASS32, ELFDATA2MSB, EV_CURRENT, ELFOSABI_NONE,
-            /* ABI version */ 0, /* padding */ 0, 0, 0, 0, 0, 0, 0
+            ELFMAG0,
+            ELFMAG1,
+            ELFMAG2,
+            ELFMAG3,
+            ELFCLASS32,
+            ELFDATA2MSB,
+            EV_CURRENT,
+            ELFOSABI_NONE,
+            /* ABI version */ 0,
+            /* padding */ 0,
+            0,
+            0,
+            0,
+            0,
+            0,
+            0,
         ];
         const IDENT_ARM: [u8; EI_NIDENT] = [
-            ELFMAG0,    ELFMAG1,     ELFMAG2,    ELFMAG3,
-            ELFCLASS32, ELFDATA2LSB, EV_CURRENT, ELFOSABI_NONE,
-            /* ABI version */ 0, /* padding */ 0, 0, 0, 0, 0, 0, 0
+            ELFMAG0,
+            ELFMAG1,
+            ELFMAG2,
+            ELFMAG3,
+            ELFCLASS32,
+            ELFDATA2LSB,
+            EV_CURRENT,
+            ELFOSABI_NONE,
+            /* ABI version */ 0,
+            /* padding */ 0,
+            0,
+            0,
+            0,
+            0,
+            0,
+            0,
         ];
 
         match (self.ehdr.e_ident, self.ehdr.e_machine) {
@@ -48,16 +74,14 @@ impl<'a> File<'a> {
         }
     }
 
-    pub fn program_headers<'b>(&'b self) -> impl Iterator<Item = Option<Elf32_Phdr>> + 'b
-    {
+    pub fn program_headers<'b>(&'b self) -> impl Iterator<Item = Option<Elf32_Phdr>> + 'b {
         (0..self.ehdr.e_phnum).map(move |i| {
             let phdr_off = self.ehdr.e_phoff as usize + mem::size_of::<Elf32_Phdr>() * i as usize;
             self.read_unaligned::<Elf32_Phdr>(phdr_off)
         })
     }
 
-    pub fn section_headers<'b>(&'b self) -> impl Iterator<Item = Option<Elf32_Shdr>> + 'b
-    {
+    pub fn section_headers<'b>(&'b self) -> impl Iterator<Item = Option<Elf32_Shdr>> + 'b {
         (0..self.ehdr.e_shnum).map(move |i| {
             let shdr_off = self.ehdr.e_shoff as usize + mem::size_of::<Elf32_Shdr>() * i as usize;
             self.read_unaligned::<Elf32_Shdr>(shdr_off)

src/libdyld/src/image.rs

@@ -1,13 +1,9 @@
-use core::{
-    ops::{Deref, DerefMut, Range},
-    mem,
-    slice,
-};
 use alloc::alloc::{alloc_zeroed, dealloc, Layout, LayoutError};
-use super::{
-    elf::*,
-    Error,
-};
+use core::{mem,
+           ops::{Deref, DerefMut, Range},
+           slice};
+
+use super::{elf::*, Error};
 
 pub struct DynamicSection {
     pub strtab: Range<usize>,
@@ -34,17 +30,12 @@ impl Image {
             slice::from_raw_parts_mut(ptr, size)
         };
 
-        Ok(Image {
-            layout,
-            data,
-        })
+        Ok(Image { layout, data })
     }
 
     /// assumes that self.data is properly aligned
     pub(crate) fn get_ref<T>(&self, offset: usize) -> Option<&T>
-    where
-        T: Copy,
-    {
+    where T: Copy {
         if self.data.len() < offset + mem::size_of::<T>() {
             None
         } else if (self.data.as_ptr() as usize + offset) & (mem::align_of::<T>() - 1) != 0 {
@@ -66,55 +57,53 @@ impl Image {
         unsafe { slice::from_raw_parts(ptr, len) }
     }
 
-    fn dyn_headers<'a>(&'a self, range: Range<usize>) ->
-        impl Iterator<Item = &'a Elf32_Dyn> + 'a
-    {
+    fn dyn_headers<'a>(&'a self, range: Range<usize>) -> impl Iterator<Item = &'a Elf32_Dyn> + 'a {
         range
             .step_by(mem::size_of::<Elf32_Dyn>())
-            .filter_map(move |offset| {
-                self.get_ref::<Elf32_Dyn>(offset)
-            })
+            .filter_map(move |offset| self.get_ref::<Elf32_Dyn>(offset))
             .take_while(|d| unsafe { d.d_un.d_val } as i32 != DT_NULL)
     }
 
     pub fn dyn_section(&self, range: Range<usize>) -> Result<DynamicSection, Error> {
         let (mut strtab_off, mut strtab_sz) = (0, 0);
-        let (mut rel_off,    mut rel_sz)    = (0, 0);
-        let (mut rela_off,   mut rela_sz)   = (0, 0);
+        let (mut rel_off, mut rel_sz) = (0, 0);
+        let (mut rela_off, mut rela_sz) = (0, 0);
         let (mut pltrel_off, mut pltrel_sz) = (0, 0);
-        let (mut hash_off,   mut hash_sz)   = (0, 0);
+        let (mut hash_off, mut hash_sz) = (0, 0);
         let mut symtab_off = 0;
-        let mut sym_ent    = 0;
-        let mut rel_ent    = 0;
-        let mut rela_ent   = 0;
-        let mut nbucket    = 0;
-        let mut nchain     = 0;
+        let mut sym_ent = 0;
+        let mut rel_ent = 0;
+        let mut rela_ent = 0;
+        let mut nbucket = 0;
+        let mut nchain = 0;
 
         for dyn_header in self.dyn_headers(range) {
             let val = unsafe { dyn_header.d_un.d_val } as usize;
             match dyn_header.d_tag {
-                DT_NULL     => break,
-                DT_STRTAB   => strtab_off = val,
-                DT_STRSZ    => strtab_sz  = val,
-                DT_SYMTAB   => symtab_off = val,
-                DT_SYMENT   => sym_ent    = val,
-                DT_REL      => rel_off    = val,
-                DT_RELSZ    => rel_sz     = val,
-                DT_RELENT   => rel_ent    = val,
-                DT_RELA     => rela_off   = val,
-                DT_RELASZ   => rela_sz    = val,
-                DT_RELAENT  => rela_ent   = val,
-                DT_JMPREL   => pltrel_off = val,
-                DT_PLTRELSZ => pltrel_sz  = val,
-                DT_HASH     => {
-                    nbucket  = *self.get_ref::<Elf32_Word>(val + 0)
+                DT_NULL => break,
+                DT_STRTAB => strtab_off = val,
+                DT_STRSZ => strtab_sz = val,
+                DT_SYMTAB => symtab_off = val,
+                DT_SYMENT => sym_ent = val,
+                DT_REL => rel_off = val,
+                DT_RELSZ => rel_sz = val,
+                DT_RELENT => rel_ent = val,
+                DT_RELA => rela_off = val,
+                DT_RELASZ => rela_sz = val,
+                DT_RELAENT => rela_ent = val,
+                DT_JMPREL => pltrel_off = val,
+                DT_PLTRELSZ => pltrel_sz = val,
+                DT_HASH => {
+                    nbucket = *self
+                        .get_ref::<Elf32_Word>(val + 0)
                         .ok_or("cannot read hash bucket count")? as usize;
-                    nchain   = *self.get_ref::<Elf32_Word>(val + 4)
+                    nchain = *self
+                        .get_ref::<Elf32_Word>(val + 4)
                         .ok_or("cannot read hash chain count")? as usize;
                     hash_off = val + 8;
-                    hash_sz  = (nbucket + nchain) * mem::size_of::<Elf32_Word>();
+                    hash_sz = (nbucket + nchain) * mem::size_of::<Elf32_Word>();
                 }
-                _ => ()
+                _ => (),
             }
         }
 
@@ -123,28 +112,28 @@ impl Image {
         let symtab_sz = nchain * mem::size_of::<Elf32_Sym>();
 
         if strtab_off + strtab_sz > self.data.len() {
-            return Err("invalid strtab offset/size")?
+            return Err("invalid strtab offset/size")?;
         }
         if symtab_off + symtab_sz > self.data.len() {
-            return Err("invalid symtab offset/size")?
+            return Err("invalid symtab offset/size")?;
         }
         if sym_ent != mem::size_of::<Elf32_Sym>() {
-            return Err("incorrect symbol entry size")?
+            return Err("incorrect symbol entry size")?;
         }
         if rel_off + rel_sz > self.data.len() {
-            return Err("invalid rel offset/size")?
+            return Err("invalid rel offset/size")?;
         }
         if rel_ent != 0 && rel_ent != mem::size_of::<Elf32_Rel>() {
-            return Err("incorrect relocation entry size")?
+            return Err("incorrect relocation entry size")?;
         }
         if rela_off + rela_sz > self.data.len() {
-            return Err("invalid rela offset/size")?
+            return Err("invalid rela offset/size")?;
         }
         if rela_ent != 0 && rela_ent != mem::size_of::<Elf32_Rela>() {
-            return Err("incorrect relocation entry size")?
+            return Err("incorrect relocation entry size")?;
         }
         if pltrel_off + pltrel_sz > self.data.len() {
-            return Err("invalid pltrel offset/size")?
+            return Err("invalid pltrel offset/size")?;
         }
 
         Ok(DynamicSection {
@@ -165,7 +154,7 @@ impl Image {
 
     pub fn write(&self, offset: usize, value: Elf32_Word) -> Result<(), Error> {
         if offset + mem::size_of::<Elf32_Addr>() > self.data.len() {
-            return Err("relocation out of image bounds")?
+            return Err("relocation out of image bounds")?;
         }
 
         let ptr = (self.data.as_ptr() as usize + offset) as *mut Elf32_Addr;

src/libdyld/src/lib.rs

@@ -1,13 +1,14 @@
 #![no_std]
 
 extern crate alloc;
-extern crate log;
 extern crate libcortex_a9;
+extern crate log;
 
-use core::{convert, fmt, ops::Range, str};
 use alloc::string::String;
-use log::{debug, trace};
+use core::{convert, fmt, ops::Range, str};
+
 use elf::*;
+use log::{debug, trace};
 
 pub mod elf;
 mod file;
@@ -21,11 +22,10 @@ pub enum Arch {
     OpenRisc,
 }
 
-
 #[derive(Debug)]
 pub enum Error {
     Parsing(&'static str),
-    Lookup(String)
+    Lookup(String),
 }
 
 impl convert::From<&'static str> for Error {
@@ -37,10 +37,8 @@ impl convert::From<&'static str> for Error {
 impl fmt::Display for Error {
     fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
         match self {
-            &Error::Parsing(desc) =>
-                write!(f, "parse error: {}", desc),
-            &Error::Lookup(ref sym) =>
-                write!(f, "symbol lookup error: {}", sym),
+            &Error::Parsing(desc) => write!(f, "parse error: {}", desc),
+            &Error::Lookup(ref sym) => write!(f, "symbol lookup error: {}", sym),
         }
     }
 }
@@ -103,20 +101,22 @@ impl Library {
         let mut index = self.hash_bucket()[hash as usize % self.hash_bucket().len()] as usize;
 
         loop {
-            if index == STN_UNDEF { return None }
+            if index == STN_UNDEF {
+                return None;
+            }
 
             let sym = &self.symtab()[index];
             let sym_name_off = sym.st_name as usize;
             match self.strtab().get(sym_name_off..sym_name_off + name.len()) {
                 Some(sym_name) if sym_name == name => {
                     if ELF32_ST_BIND(sym.st_info) & STB_GLOBAL == 0 {
-                        return None
+                        return None;
                     }
 
                     match sym.st_shndx {
                         SHN_UNDEF => return None,
                         SHN_ABS => return Some(self.image.ptr() as u32 + sym.st_value),
-                        _ => return Some(self.image.ptr() as u32 + sym.st_value)
+                        _ => return Some(self.image.ptr() as u32 + sym.st_value),
                     }
                 }
                 _ => (),
@@ -127,10 +127,16 @@ impl Library {
     }
 
     pub fn name_starting_at(&self, offset: usize) -> Result<&[u8], Error> {
-        let size = self.strtab().iter().skip(offset).position(|&x| x == 0)
-                              .ok_or("symbol in symbol table not null-terminated")?;
-        Ok(self.strtab().get(offset..offset + size)
-           .ok_or("cannot read symbol name")?)
+        let size = self
+            .strtab()
+            .iter()
+            .skip(offset)
+            .position(|&x| x == 0)
+            .ok_or("symbol in symbol table not null-terminated")?;
+        Ok(self
+            .strtab()
+            .get(offset..offset + size)
+            .ok_or("cannot read symbol name")?)
     }
 
     /// Rebind Rela by `name` to a new `addr`
@@ -143,53 +149,59 @@ impl Library {
     }
 }
 
-pub fn load(
-    data: &[u8],
-    resolve: &dyn Fn(&[u8]) -> Option<Elf32_Word>
-) -> Result<Library, Error> {
+pub fn load(data: &[u8], resolve: &dyn Fn(&[u8]) -> Option<Elf32_Word>) -> Result<Library, Error> {
     // validate ELF file
-    let file = file::File::new(data)
-        .ok_or("cannot read ELF header")?;
+    let file = file::File::new(data).ok_or("cannot read ELF header")?;
     if file.ehdr.e_type != ET_DYN {
-        return Err("not a shared library")?
+        return Err("not a shared library")?;
     }
-    let arch = file.arch()
-        .ok_or("not for a supported architecture")?;
+    let arch = file.arch().ok_or("not for a supported architecture")?;
 
     // prepare target memory
-    let image_size = file.program_headers()
+    let image_size = file
+        .program_headers()
         .filter_map(|phdr| phdr.map(|phdr| phdr.p_vaddr + phdr.p_memsz))
         .max()
         .unwrap_or(0) as usize;
-    let image_align = file.program_headers()
-        .filter_map(|phdr| phdr.and_then(|phdr| {
-            if phdr.p_type == PT_LOAD {
-                Some(phdr.p_align)
-            } else {
-                None
-            }
-        }))
+    let image_align = file
+        .program_headers()
+        .filter_map(|phdr| {
+            phdr.and_then(|phdr| {
+                if phdr.p_type == PT_LOAD {
+                    Some(phdr.p_align)
+                } else {
+                    None
+                }
+            })
+        })
         .max()
         .unwrap_or(4) as usize;
     // 1 image for all segments
-    let mut image = image::Image::new(image_size, image_align)
-        .map_err(|_| "cannot allocate target image")?;
-    debug!("ELF target: {} bytes, align to {:X}, allocated at {:08X}", image_size, image_align, image.ptr() as usize);
+    let mut image = image::Image::new(image_size, image_align).map_err(|_| "cannot allocate target image")?;
+    debug!(
+        "ELF target: {} bytes, align to {:X}, allocated at {:08X}",
+        image_size,
+        image_align,
+        image.ptr() as usize
+    );
 
     // LOAD
     for phdr in file.program_headers() {
         let phdr = phdr.ok_or("cannot read program header")?;
-        trace!("Program header: {:08X}+{:08X} to {:08X}",
-              phdr.p_offset, phdr.p_filesz,
-              image.ptr() as u32
+        trace!(
+            "Program header: {:08X}+{:08X} to {:08X}",
+            phdr.p_offset,
+            phdr.p_filesz,
+            image.ptr() as u32
         );
         let file_range = phdr.p_offset as usize..(phdr.p_offset + phdr.p_filesz) as usize;
         match phdr.p_type {
             PT_LOAD => {
-                let src = file.get(file_range)
+                let src = file
+                    .get(file_range)
                     .ok_or("program header requests an out of bounds load (in file)")?;
-                let dst = image.get_mut(phdr.p_vaddr as usize..
-                                        (phdr.p_vaddr + phdr.p_filesz) as usize)
+                let dst = image
+                    .get_mut(phdr.p_vaddr as usize..(phdr.p_vaddr + phdr.p_filesz) as usize)
                     .ok_or("program header requests an out of bounds load (in target)")?;
                 dst.copy_from_slice(src);
             }
@@ -203,9 +215,9 @@ pub fn load(
         let shdr = shdr.ok_or("cannot read section header")?;
         match shdr.sh_type as usize {
             SHT_ARM_EXIDX => {
-                let range = shdr.sh_addr as usize..
-                    (shdr.sh_addr + shdr.sh_size) as usize;
-                let _ = image.get(range.clone())
+                let range = shdr.sh_addr as usize..(shdr.sh_addr + shdr.sh_size) as usize;
+                let _ = image
+                    .get(range.clone())
                     .ok_or("section header specifies EXIDX outside of image (in target)")?;
                 exidx = Some(range);
             }
@@ -214,11 +226,14 @@ pub fn load(
     }
 
     // relocate DYNAMIC
-    let dyn_range = file.dyn_header_vaddr()
-        .ok_or("cannot find a dynamic header")?;
+    let dyn_range = file.dyn_header_vaddr().ok_or("cannot find a dynamic header")?;
     let dyn_section = image.dyn_section(dyn_range.clone())?;
-    debug!("Relocating {} rela, {} rel, {} pltrel",
-           dyn_section.rela.len(), dyn_section.rel.len(), dyn_section.pltrel.len());
+    debug!(
+        "Relocating {} rela, {} rel, {} pltrel",
+        dyn_section.rela.len(),
+        dyn_section.rel.len(),
+        dyn_section.pltrel.len()
+    );
     let lib = Library {
         arch,
         image,

src/libdyld/src/reloc.rs

@@ -1,16 +1,10 @@
 use alloc::string::String;
+
+use libcortex_a9::{asm::{dsb, isb},
+                   cache::{bpiall, dcci_slice, iciallu}};
 use log::trace;
-use super::{
-    Arch,
-    elf::*,
-    Error,
-    image::Image,
-    Library,
-};
-use libcortex_a9::{
-    cache::{dcci_slice, iciallu, bpiall},
-    asm::{dsb, isb},
-};
+
+use super::{elf::*, image::Image, Arch, Error, Library};
 
 pub trait Relocatable {
     fn offset(&self) -> usize;
@@ -66,25 +60,18 @@ enum RelType {
 impl RelType {
     pub fn new(arch: Arch, type_info: u8) -> Option<Self> {
         match type_info {
-            R_OR1K_NONE if arch == Arch::OpenRisc =>
-                Some(RelType::None),
-            R_ARM_NONE if arch == Arch::Arm =>
-                Some(RelType::None),
+            R_OR1K_NONE if arch == Arch::OpenRisc => Some(RelType::None),
+            R_ARM_NONE if arch == Arch::Arm => Some(RelType::None),
 
-            R_OR1K_RELATIVE if arch == Arch::OpenRisc =>
-                Some(RelType::Relative),
-            R_ARM_RELATIVE if arch == Arch::Arm =>
-                Some(RelType::Relative),
+            R_OR1K_RELATIVE if arch == Arch::OpenRisc => Some(RelType::Relative),
+            R_ARM_RELATIVE if arch == Arch::Arm => Some(RelType::Relative),
 
-            R_OR1K_32 | R_OR1K_GLOB_DAT | R_OR1K_JMP_SLOT
-                if arch == Arch::OpenRisc => Some(RelType::LookupAbs),
-            R_ARM_GLOB_DAT | R_ARM_JUMP_SLOT | R_ARM_ABS32
-                if arch == Arch::Arm => Some(RelType::LookupAbs),
+            R_OR1K_32 | R_OR1K_GLOB_DAT | R_OR1K_JMP_SLOT if arch == Arch::OpenRisc => Some(RelType::LookupAbs),
+            R_ARM_GLOB_DAT | R_ARM_JUMP_SLOT | R_ARM_ABS32 if arch == Arch::Arm => Some(RelType::LookupAbs),
 
             R_ARM_PREL31 if arch == Arch::Arm => Some(RelType::LookupRel),
 
-            _ =>
-                None
+            _ => None,
         }
     }
 }
@@ -96,22 +83,25 @@ fn format_sym_name(sym_name: &[u8]) -> String {
 }
 
 pub fn relocate<R: Relocatable>(
-    arch: Arch, lib: &Library,
-    rel: &R, resolve: &dyn Fn(&[u8]) -> Option<Elf32_Word>
+    arch: Arch,
+    lib: &Library,
+    rel: &R,
+    resolve: &dyn Fn(&[u8]) -> Option<Elf32_Word>,
 ) -> Result<(), Error> {
     let sym;
     if rel.sym_info() == 0 {
         sym = None;
     } else {
-        sym = Some(lib.symtab().get(rel.sym_info() as usize)
-                   .ok_or("symbol out of bounds of symbol table")?)
+        sym = Some(
+            lib.symtab()
+                .get(rel.sym_info() as usize)
+                .ok_or("symbol out of bounds of symbol table")?,
+        )
     }
 
-    let rel_type = RelType::new(arch, rel.type_info())
-        .ok_or("unsupported relocation type")?;
+    let rel_type = RelType::new(arch, rel.type_info()).ok_or("unsupported relocation type")?;
     let value = match rel_type {
-        RelType::None =>
-            return Ok(()),
+        RelType::None => return Ok(()),
 
         RelType::Relative => {
             let addend = rel.addend(&lib.image);
@@ -132,42 +122,48 @@ pub fn relocate<R: Relocatable>(
                 addr
             } else {
                 // We couldn't find it anywhere.
-                return Err(Error::Lookup(format_sym_name(sym_name)))
+                return Err(Error::Lookup(format_sym_name(sym_name)));
             };
 
             match rel_type {
                 RelType::LookupAbs => sym_addr,
-                RelType::LookupRel =>
-                    sym_addr.wrapping_sub(
-                        lib.image.ptr().wrapping_offset(rel.offset() as isize) as Elf32_Addr),
-                _ => unreachable!()
+                RelType::LookupRel => {
+                    sym_addr.wrapping_sub(lib.image.ptr().wrapping_offset(rel.offset() as isize) as Elf32_Addr)
+                }
+                _ => unreachable!(),
             }
         }
     };
 
     match rel.type_info() {
         R_ARM_PREL31 => {
-            let reloc_word = lib.image.get_ref::<Elf32_Word>(rel.offset())
-                                .ok_or("relocation offset cannot be read")?;
-            lib.image.write(rel.offset(), (reloc_word & 0x80000000) | (value & 0x7FFFFFFF))
-        },
+            let reloc_word = lib
+                .image
+                .get_ref::<Elf32_Word>(rel.offset())
+                .ok_or("relocation offset cannot be read")?;
+            lib.image
+                .write(rel.offset(), (reloc_word & 0x80000000) | (value & 0x7FFFFFFF))
+        }
 
         _ => lib.image.write(rel.offset(), value),
     }
 }
 
-pub fn rebind(
-    arch: Arch, lib: &Library, name: &[u8], value: Elf32_Word
-) -> Result<(), Error> {
+pub fn rebind(arch: Arch, lib: &Library, name: &[u8], value: Elf32_Word) -> Result<(), Error> {
     fn rebind_symbol_to_value<R: Relocatable>(
-        arch: Arch, lib: &Library,name: &[u8], value: Elf32_Word, relocs: &[R]
+        arch: Arch,
+        lib: &Library,
+        name: &[u8],
+        value: Elf32_Word,
+        relocs: &[R],
     ) -> Result<(), Error> {
         for reloc in relocs {
-            let rel_type = RelType::new(arch, reloc.type_info())
-                .ok_or("unsupported relocation type")?;
+            let rel_type = RelType::new(arch, reloc.type_info()).ok_or("unsupported relocation type")?;
             match rel_type {
                 RelType::LookupAbs => {
-                    let sym = lib.symtab().get(reloc.sym_info() as usize)
+                    let sym = lib
+                        .symtab()
+                        .get(reloc.sym_info() as usize)
                         .ok_or("symbol out of bounds of symbol table")?;
                     let sym_name = lib.name_starting_at(sym.st_name as usize)?;
 

src/libio/Cargo.toml.tpl

@@ -11,7 +11,7 @@ path = "lib.rs"
 core_io = { version = "0.1", features = ["collections"] }
 byteorder = { version = "1.0", default-features = false, optional = true }
 
-libsupport_zynq = { default-features = false, features = ["alloc_core"], git = "https://git.m-labs.hk/M-Labs/zynq-rs.git" }
+libsupport_zynq = { path = "@@ZYNQ_RS@@/libsupport_zynq", default-features = false, features = ["alloc_core"] }
 
 [features]
 alloc = []

src/libio/cursor.rs

@@ -1,9 +1,12 @@
-use core_io::{Read, Write, Error as IoError};
+#[cfg(feature = "alloc")]
+use alloc::vec::Vec;
+
+use core_io::{Error as IoError, Read, Write};
 
 #[derive(Debug, Clone)]
 pub struct Cursor<T> {
     inner: T,
-    pos:   usize
+    pos: usize,
 }
 
 impl<T> Cursor<T> {
@@ -39,7 +42,6 @@ impl<T> Cursor<T> {
 }
 
 impl<T: AsRef<[u8]>> Read for Cursor<T> {
-
     fn read(&mut self, buf: &mut [u8]) -> Result<usize, IoError> {
         let data = &self.inner.as_ref()[self.pos..];
         let len = buf.len().min(data.len());
@@ -50,10 +52,9 @@ impl<T: AsRef<[u8]>> Read for Cursor<T> {
 }
 
 impl Write for Cursor<&mut [u8]> {
-
     fn write(&mut self, buf: &[u8]) -> Result<usize, IoError> {
         let data = &mut self.inner[self.pos..];
-        let len  = buf.len().min(data.len());
+        let len = buf.len().min(data.len());
         data[..len].copy_from_slice(&buf[..len]);
         self.pos += len;
         Ok(len)
@@ -66,8 +67,7 @@ impl Write for Cursor<&mut [u8]> {
 }
 
 #[cfg(feature = "alloc")]
-impl Write for Cursor<::alloc::Vec<u8>> {
-
+impl Write for Cursor<Vec<u8>> {
     #[inline]
     fn write(&mut self, buf: &[u8]) -> Result<usize, IoError> {
         self.inner.extend_from_slice(buf);

src/libio/lib.rs

@@ -1,13 +1,10 @@
 #![no_std]
 #![feature(never_type)]
-#![cfg_attr(feature = "alloc", feature(alloc))]
 
+#[cfg(feature = "alloc")]
 extern crate alloc;
 extern crate core_io;
 
-#[cfg(feature = "alloc")]
-#[macro_use]
-use alloc;
 #[cfg(feature = "byteorder")]
 extern crate byteorder;
 
@@ -16,7 +13,7 @@ pub mod cursor;
 pub mod proto;
 
 pub use cursor::Cursor;
-#[cfg(feature = "byteorder")]
-pub use proto::{ProtoRead, ProtoWrite};
 #[cfg(all(feature = "byteorder", feature = "alloc"))]
 pub use proto::ReadStringError;
+#[cfg(feature = "byteorder")]
+pub use proto::{ProtoRead, ProtoWrite};

src/libio/proto.rs

@@ -1,15 +1,15 @@
+#[cfg(feature = "alloc")]
+use alloc::{string::String, vec};
 use core::str::Utf8Error;
-use byteorder::{ByteOrder, NativeEndian};
-use alloc::vec;
-use alloc::string::String;
 
-use core_io::{Read, Write, Error as IoError};
+use byteorder::{ByteOrder, NativeEndian};
+use core_io::{Error as IoError, Read, Write};
 
 #[allow(dead_code)]
 #[derive(Debug, Clone, PartialEq)]
 pub enum ReadStringError<T> {
     Utf8(Utf8Error),
-    Other(T)
+    Other(T),
 }
 
 pub trait ProtoRead {
@@ -51,7 +51,8 @@ pub trait ProtoRead {
     }
 
     #[inline]
-    fn read_bytes(&mut self) -> Result<::alloc::vec::Vec<u8>, Self::ReadError> {
+    #[cfg(feature = "alloc")]
+    fn read_bytes(&mut self) -> Result<vec::Vec<u8>, Self::ReadError> {
         let length = self.read_u32()?;
         let mut value = vec![0; length as usize];
         self.read_exact(&mut value)?;
@@ -59,7 +60,8 @@ pub trait ProtoRead {
     }
 
     #[inline]
-    fn read_string(&mut self) -> Result<::alloc::string::String, ReadStringError<Self::ReadError>> {
+    #[cfg(feature = "alloc")]
+    fn read_string(&mut self) -> Result<String, ReadStringError<Self::ReadError>> {
         let bytes = self.read_bytes().map_err(ReadStringError::Other)?;
         String::from_utf8(bytes).map_err(|err| ReadStringError::Utf8(err.utf8_error()))
     }
@@ -136,12 +138,15 @@ pub trait ProtoWrite {
     }
 
     #[inline]
+    #[cfg(feature = "alloc")]
     fn write_string(&mut self, value: &str) -> Result<(), Self::WriteError> {
         self.write_bytes(value.as_bytes())
     }
 }
 
-impl<T> ProtoRead for T where T: Read + ?Sized {
+impl<T> ProtoRead for T
+where T: Read + ?Sized
+{
     type ReadError = IoError;
 
     fn read_exact(&mut self, buf: &mut [u8]) -> Result<(), Self::ReadError> {
@@ -149,7 +154,9 @@ impl<T> ProtoRead for T where T: Read + ?Sized {
     }
 }
 
-impl<T> ProtoWrite for T where T: Write + ?Sized {
+impl<T> ProtoWrite for T
+where T: Write + ?Sized
+{
     type WriteError = IoError;
 
     fn write_all(&mut self, buf: &[u8]) -> Result<(), Self::WriteError> {

src/libksupport/Cargo.toml.tpl

@@ -0,0 +1,34 @@
+[package]
+name = "ksupport"
+description = "Kernel support for Zynq-based platforms"
+version = "0.1.0"
+authors = ["M-Labs"]
+edition = "2018"
+
+[build-dependencies]
+build_zynq = { path = "../libbuild_zynq" }
+
+[dependencies]
+cslice = "0.3"
+log = "0.4"
+nb = "0.1"
+core_io = { version = "0.1", features = ["collections"] }
+byteorder = { version = "1.3", default-features = false }
+void = { version = "1", default-features = false }
+log_buffer = { version = "1.2" }
+libm = { version = "0.2", features = ["unstable"] }
+vcell = "0.1"
+
+libboard_zynq = { path = "@@ZYNQ_RS@@/libboard_zynq", features = ["ipv6"]}
+libsupport_zynq = { path = "@@ZYNQ_RS@@/libsupport_zynq", default-features = false, features = ["alloc_core"] }
+libcortex_a9 = { path = "@@ZYNQ_RS@@/libcortex_a9" }
+libasync = { path = "@@ZYNQ_RS@@/libasync" }
+libregister = { path = "@@ZYNQ_RS@@/libregister" }
+libconfig = { path = "@@ZYNQ_RS@@/libconfig", features = ["fat_lfn", "ipv6"] }
+
+dyld = { path = "../libdyld" }
+dwarf = { path = "../libdwarf" }
+unwind = { path = "../libunwind" }
+libc = { path = "../libc" }
+io = { path = "../libio" }
+libboard_artiq = { path = "../libboard_artiq" }

src/libksupport/build.rs

@@ -0,0 +1,5 @@
+extern crate build_zynq;
+
+fn main() {
+    build_zynq::cfg();
+}

src/libksupport/src/eh_artiq.rs

@@ -13,14 +13,14 @@
 #![allow(non_camel_case_types)]
 
 use core::mem;
+
 use cslice::CSlice;
-use unwind as uw;
-use libc::{c_int, c_void, uintptr_t};
-use log::{trace, error};
-use crate::kernel::KERNEL_IMAGE;
-
 use dwarf::eh::{self, EHAction, EHContext};
+use libc::{c_int, c_void, uintptr_t};
+use log::{error, trace};
+use unwind as uw;
 
+use crate::kernel::KERNEL_IMAGE;
 
 const EXCEPTION_CLASS: uw::_Unwind_Exception_Class = 0x4d_4c_42_53_41_52_54_51; /* 'MLBSARTQ' */
 
@@ -32,13 +32,13 @@ const UNWIND_DATA_REG: (i32, i32) = (0, 1); // R0, R1
 #[repr(C)]
 #[derive(Clone, Copy)]
 pub struct Exception<'a> {
-    pub id:       u32,
-    pub file:     CSlice<'a, u8>,
-    pub line:     u32,
-    pub column:   u32,
+    pub id: u32,
+    pub file: CSlice<'a, u8>,
+    pub line: u32,
+    pub column: u32,
     pub function: CSlice<'a, u8>,
-    pub message:  CSlice<'a, u8>,
-    pub param:    [i64; 3]
+    pub message: CSlice<'a, u8>,
+    pub param: [i64; 3],
 }
 
 fn str_err(_: core::str::Utf8Error) -> core::fmt::Error {
@@ -55,12 +55,16 @@ fn exception_str<'a>(s: &'a CSlice<'a, u8>) -> Result<&'a str, core::str::Utf8Er
 
 impl<'a> core::fmt::Debug for Exception<'a> {
     fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result {
-        write!(f, "Exception {} from {} in {}:{}:{}, message: {}",
+        write!(
+            f,
+            "Exception {} from {} in {}:{}:{}, message: {}",
             self.id,
             exception_str(&self.function).map_err(str_err)?,
             exception_str(&self.file).map_err(str_err)?,
-            self.line, self.column,
-            exception_str(&self.message).map_err(str_err)?)
+            self.line,
+            self.column,
+            exception_str(&self.message).map_err(str_err)?
+        )
     }
 }
 
@@ -91,9 +95,9 @@ struct ExceptionBuffer {
 
 static mut EXCEPTION_BUFFER: ExceptionBuffer = ExceptionBuffer {
     uw_exceptions: [uw::_Unwind_Exception {
-        exception_class:   EXCEPTION_CLASS,
+        exception_class: EXCEPTION_CLASS,
         exception_cleanup: cleanup,
-        private:           [0; uw::unwinder_private_data_size],
+        private: [0; uw::unwinder_private_data_size],
     }; MAX_INFLIGHT_EXCEPTIONS],
     exceptions: [None; MAX_INFLIGHT_EXCEPTIONS + 1],
     exception_stack: [-1; MAX_INFLIGHT_EXCEPTIONS + 1],
@@ -102,17 +106,17 @@ static mut EXCEPTION_BUFFER: ExceptionBuffer = ExceptionBuffer {
     stack_pointers: [StackPointerBacktrace {
         stack_pointer: 0,
         initial_backtrace_size: 0,
-        current_backtrace_size: 0
+        current_backtrace_size: 0,
     }; MAX_INFLIGHT_EXCEPTIONS + 1],
-    exception_count: 0
+    exception_count: 0,
 };
 
-pub unsafe extern fn reset_exception_buffer() {
+pub unsafe extern "C" fn reset_exception_buffer() {
     trace!("reset exception buffer");
     EXCEPTION_BUFFER.uw_exceptions = [uw::_Unwind_Exception {
-        exception_class:   EXCEPTION_CLASS,
+        exception_class: EXCEPTION_CLASS,
         exception_cleanup: cleanup,
-        private:           [0; uw::unwinder_private_data_size],
+        private: [0; uw::unwinder_private_data_size],
     }; MAX_INFLIGHT_EXCEPTIONS];
     EXCEPTION_BUFFER.exceptions = [None; MAX_INFLIGHT_EXCEPTIONS + 1];
     EXCEPTION_BUFFER.exception_stack = [-1; MAX_INFLIGHT_EXCEPTIONS + 1];
@@ -120,26 +124,25 @@ pub unsafe extern fn reset_exception_buffer() {
     EXCEPTION_BUFFER.exception_count = 0;
 }
 
-type _Unwind_Stop_Fn = extern "C" fn(version: c_int,
-                                     actions: i32,
-                                     exception_class: uw::_Unwind_Exception_Class,
-                                     exception_object: *mut uw::_Unwind_Exception,
-                                     context: *mut uw::_Unwind_Context,
-                                     stop_parameter: *mut c_void)
-                                    -> uw::_Unwind_Reason_Code;
+type _Unwind_Stop_Fn = extern "C" fn(
+    version: c_int,
+    actions: i32,
+    exception_class: uw::_Unwind_Exception_Class,
+    exception_object: *mut uw::_Unwind_Exception,
+    context: *mut uw::_Unwind_Context,
+    stop_parameter: *mut c_void,
+) -> uw::_Unwind_Reason_Code;
 
-extern {
+extern "C" {
     // not defined in EHABI, but LLVM added it and is useful to us
-    fn _Unwind_ForcedUnwind(exception: *mut uw::_Unwind_Exception,
-                            stop_fn: _Unwind_Stop_Fn,
-                            stop_parameter: *mut c_void) -> uw::_Unwind_Reason_Code;
+    fn _Unwind_ForcedUnwind(
+        exception: *mut uw::_Unwind_Exception,
+        stop_fn: _Unwind_Stop_Fn,
+        stop_parameter: *mut c_void,
+    ) -> uw::_Unwind_Reason_Code;
 }
 
-unsafe fn find_eh_action(
-    context: *mut uw::_Unwind_Context,
-    foreign_exception: bool,
-    id: u32,
-) -> Result<EHAction, ()> {
+unsafe fn find_eh_action(context: *mut uw::_Unwind_Context, foreign_exception: bool, id: u32) -> Result<EHAction, ()> {
     let lsda = uw::_Unwind_GetLanguageSpecificData(context) as *const u8;
     let mut ip_before_instr: c_int = 0;
     let ip = uw::_Unwind_GetIPInfo(context, &mut ip_before_instr);
@@ -154,10 +157,11 @@ unsafe fn find_eh_action(
     eh::find_eh_action(lsda, &eh_context, foreign_exception, id)
 }
 
-pub unsafe fn artiq_personality(_state: uw::_Unwind_State,
-                                exception_object: *mut uw::_Unwind_Exception,
-                                context: *mut uw::_Unwind_Context)
-                                -> uw::_Unwind_Reason_Code {
+pub unsafe fn artiq_personality(
+    _state: uw::_Unwind_State,
+    exception_object: *mut uw::_Unwind_Exception,
+    context: *mut uw::_Unwind_Context,
+) -> uw::_Unwind_Reason_Code {
     // we will only do phase 2 forced unwinding now
     // The DWARF unwinder assumes that _Unwind_Context holds things like the function
     // and LSDA pointers, however ARM EHABI places them into the exception object.
@@ -165,9 +169,7 @@ pub unsafe fn artiq_personality(_state: uw::_Unwind_State,
     // take only the context pointer, GCC personality routines stash a pointer to
     // exception_object in the context, using location reserved for ARM's
     // "scratch register" (r12).
-    uw::_Unwind_SetGR(context,
-                      uw::UNWIND_POINTER_REG,
-                      exception_object as uw::_Unwind_Ptr);
+    uw::_Unwind_SetGR(context, uw::UNWIND_POINTER_REG, exception_object as uw::_Unwind_Ptr);
     // ...A more principled approach would be to provide the full definition of ARM's
     // _Unwind_Context in our libunwind bindings and fetch the required data from there
     // directly, bypassing DWARF compatibility functions.
@@ -186,10 +188,8 @@ pub unsafe fn artiq_personality(_state: uw::_Unwind_State,
     };
     match eh_action {
         EHAction::None => return continue_unwind(exception_object, context),
-        EHAction::Cleanup(lpad) |
-        EHAction::Catch(lpad) => {
-            uw::_Unwind_SetGR(context, UNWIND_DATA_REG.0,
-                              exception_object as uintptr_t);
+        EHAction::Cleanup(lpad) | EHAction::Catch(lpad) => {
+            uw::_Unwind_SetGR(context, UNWIND_DATA_REG.0, exception_object as uintptr_t);
             uw::_Unwind_SetGR(context, UNWIND_DATA_REG.1, exception as *const _ as uw::_Unwind_Word);
             uw::_Unwind_SetIP(context, lpad);
             return uw::_URC_INSTALL_CONTEXT;
@@ -199,9 +199,10 @@ pub unsafe fn artiq_personality(_state: uw::_Unwind_State,
 
     // On ARM EHABI the personality routine is responsible for actually
     // unwinding a single stack frame before returning (ARM EHABI Sec. 6.1).
-    unsafe fn continue_unwind(exception_object: *mut uw::_Unwind_Exception,
-                              context: *mut uw::_Unwind_Context)
-                              -> uw::_Unwind_Reason_Code {
+    unsafe fn continue_unwind(
+        exception_object: *mut uw::_Unwind_Exception,
+        context: *mut uw::_Unwind_Context,
+    ) -> uw::_Unwind_Reason_Code {
         let reason = __gnu_unwind_frame(exception_object, context);
         if reason == uw::_URC_NO_REASON {
             uw::_URC_CONTINUE_UNWIND
@@ -211,13 +212,14 @@ pub unsafe fn artiq_personality(_state: uw::_Unwind_State,
     }
     // defined in libgcc
     extern "C" {
-        fn __gnu_unwind_frame(exception_object: *mut uw::_Unwind_Exception,
-                              context: *mut uw::_Unwind_Context)
-                              -> uw::_Unwind_Reason_Code;
+        fn __gnu_unwind_frame(
+            exception_object: *mut uw::_Unwind_Exception,
+            context: *mut uw::_Unwind_Context,
+        ) -> uw::_Unwind_Reason_Code;
     }
 }
 
-pub unsafe extern fn raise(exception: *const Exception) -> ! {
+pub unsafe extern "C" fn raise(exception: *const Exception) -> ! {
     use cslice::AsCSlice;
 
     let count = EXCEPTION_BUFFER.exception_count;
@@ -244,8 +246,11 @@ pub unsafe extern fn raise(exception: *const Exception) -> ! {
             }
         }
         assert!(found);
-        let _result = _Unwind_ForcedUnwind(&mut EXCEPTION_BUFFER.uw_exceptions[stack[count - 1] as usize],
-                                           stop_fn, core::ptr::null_mut());
+        let _result = _Unwind_ForcedUnwind(
+            &mut EXCEPTION_BUFFER.uw_exceptions[stack[count - 1] as usize],
+            stop_fn,
+            core::ptr::null_mut(),
+        );
     } else {
         if count < MAX_INFLIGHT_EXCEPTIONS {
             trace!("raising exception at level {}", count);
@@ -253,34 +258,33 @@ pub unsafe extern fn raise(exception: *const Exception) -> ! {
             for (i, slot) in EXCEPTION_BUFFER.exceptions.iter_mut().enumerate() {
                 // we should always be able to find a slot
                 if slot.is_none() {
-                    *slot = Some(
-                        *mem::transmute::<*const Exception, *const Exception<'static>>
-                        (exception));
+                    *slot = Some(*mem::transmute::<*const Exception, *const Exception<'static>>(
+                        exception,
+                    ));
                     EXCEPTION_BUFFER.exception_stack[count] = i as isize;
-                    EXCEPTION_BUFFER.uw_exceptions[i].private =
-                        [0; uw::unwinder_private_data_size];
+                    EXCEPTION_BUFFER.uw_exceptions[i].private = [0; uw::unwinder_private_data_size];
                     EXCEPTION_BUFFER.stack_pointers[i] = StackPointerBacktrace {
                         stack_pointer: 0,
                         initial_backtrace_size: EXCEPTION_BUFFER.backtrace_size,
                         current_backtrace_size: 0,
                     };
                     EXCEPTION_BUFFER.exception_count += 1;
-                    let _result = _Unwind_ForcedUnwind(&mut EXCEPTION_BUFFER.uw_exceptions[i],
-                                                       stop_fn, core::ptr::null_mut());
+                    let _result =
+                        _Unwind_ForcedUnwind(&mut EXCEPTION_BUFFER.uw_exceptions[i], stop_fn, core::ptr::null_mut());
                 }
             }
         } else {
             error!("too many nested exceptions");
             // TODO: better reporting?
             let exception = Exception {
-                id:       get_exception_id("RuntimeError"),
-                file:     file!().as_c_slice(),
-                line:     line!(),
-                column:   column!(),
+                id: get_exception_id("RuntimeError"),
+                file: file!().as_c_slice(),
+                line: line!(),
+                column: column!(),
                 // https://github.com/rust-lang/rfcs/pull/1719
                 function: "__artiq_raise".as_c_slice(),
-                message:  "too many nested exceptions".as_c_slice(),
-                param:    [0, 0, 0]
+                message: "too many nested exceptions".as_c_slice(),
+                param: [0, 0, 0],
             };
             EXCEPTION_BUFFER.exceptions[MAX_INFLIGHT_EXCEPTIONS] = Some(mem::transmute(exception));
             EXCEPTION_BUFFER.stack_pointers[MAX_INFLIGHT_EXCEPTIONS] = Default::default();
@@ -291,17 +295,20 @@ pub unsafe extern fn raise(exception: *const Exception) -> ! {
     unreachable!();
 }
 
-pub unsafe extern fn resume() -> ! {
+pub unsafe extern "C" fn resume() -> ! {
     trace!("resume");
     assert!(EXCEPTION_BUFFER.exception_count != 0);
     let i = EXCEPTION_BUFFER.exception_stack[EXCEPTION_BUFFER.exception_count - 1];
     assert!(i != -1);
-    let _result = _Unwind_ForcedUnwind(&mut EXCEPTION_BUFFER.uw_exceptions[i as usize],
-                                       stop_fn, core::ptr::null_mut());
+    let _result = _Unwind_ForcedUnwind(
+        &mut EXCEPTION_BUFFER.uw_exceptions[i as usize],
+        stop_fn,
+        core::ptr::null_mut(),
+    );
     unreachable!()
 }
 
-pub unsafe extern fn end_catch() {
+pub unsafe extern "C" fn end_catch() {
     let mut count = EXCEPTION_BUFFER.exception_count;
     assert!(count != 0);
     // we remove all exceptions with SP <= current exception SP
@@ -309,8 +316,7 @@ pub unsafe extern fn end_catch() {
     let index = EXCEPTION_BUFFER.exception_stack[count - 1] as usize;
     EXCEPTION_BUFFER.exception_stack[count - 1] = -1;
     EXCEPTION_BUFFER.exceptions[index] = None;
-    let outer_sp = EXCEPTION_BUFFER.stack_pointers
-        [index].stack_pointer;
+    let outer_sp = EXCEPTION_BUFFER.stack_pointers[index].stack_pointer;
     count -= 1;
     for i in (0..count).rev() {
         let index = EXCEPTION_BUFFER.exception_stack[i];
@@ -334,8 +340,7 @@ pub unsafe extern fn end_catch() {
     };
 }
 
-extern fn cleanup(_unwind_code: uw::_Unwind_Reason_Code,
-                  _uw_exception: *mut uw::_Unwind_Exception) {
+extern "C" fn cleanup(_unwind_code: uw::_Unwind_Reason_Code, _uw_exception: *mut uw::_Unwind_Exception) {
     unimplemented!()
 }
 
@@ -345,18 +350,26 @@ fn uncaught_exception() -> ! {
         for i in 0..EXCEPTION_BUFFER.exception_count {
             if EXCEPTION_BUFFER.exception_stack[i] != i as isize {
                 // find the correct index
-                let index = EXCEPTION_BUFFER.exception_stack
+                let index = EXCEPTION_BUFFER
+                    .exception_stack
                     .iter()
-                    .position(|v| *v == i as isize).unwrap();
+                    .position(|v| *v == i as isize)
+                    .unwrap();
                 let a = EXCEPTION_BUFFER.exception_stack[index];
                 let b = EXCEPTION_BUFFER.exception_stack[i];
                 assert!(a != -1 && b != -1);
-                core::mem::swap(&mut EXCEPTION_BUFFER.exception_stack[index],
-                                &mut EXCEPTION_BUFFER.exception_stack[i]);
-                core::mem::swap(&mut EXCEPTION_BUFFER.exceptions[a as usize],
-                                &mut EXCEPTION_BUFFER.exceptions[b as usize]);
-                core::mem::swap(&mut EXCEPTION_BUFFER.stack_pointers[a as usize],
-                                &mut EXCEPTION_BUFFER.stack_pointers[b as usize]);
+                core::mem::swap(
+                    &mut EXCEPTION_BUFFER.exception_stack[index],
+                    &mut EXCEPTION_BUFFER.exception_stack[i],
+                );
+                core::mem::swap(
+                    &mut EXCEPTION_BUFFER.exceptions[a as usize],
+                    &mut EXCEPTION_BUFFER.exceptions[b as usize],
+                );
+                core::mem::swap(
+                    &mut EXCEPTION_BUFFER.stack_pointers[a as usize],
+                    &mut EXCEPTION_BUFFER.stack_pointers[b as usize],
+                );
             }
         }
     }
@@ -364,17 +377,20 @@ fn uncaught_exception() -> ! {
         crate::kernel::core1::terminate(
             EXCEPTION_BUFFER.exceptions[..EXCEPTION_BUFFER.exception_count].as_ref(),
             EXCEPTION_BUFFER.stack_pointers[..EXCEPTION_BUFFER.exception_count].as_ref(),
-            EXCEPTION_BUFFER.backtrace[..EXCEPTION_BUFFER.backtrace_size].as_mut())
+            EXCEPTION_BUFFER.backtrace[..EXCEPTION_BUFFER.backtrace_size].as_mut(),
+        )
     }
 }
 
 // stop function which would be executed when we unwind each frame
-extern fn stop_fn(_version: c_int,
-                  actions: i32,
-                  _uw_exception_class: uw::_Unwind_Exception_Class,
-                  _uw_exception: *mut uw::_Unwind_Exception,
-                  context: *mut uw::_Unwind_Context,
-                  _stop_parameter: *mut c_void) -> uw::_Unwind_Reason_Code {
+extern "C" fn stop_fn(
+    _version: c_int,
+    actions: i32,
+    _uw_exception_class: uw::_Unwind_Exception_Class,
+    _uw_exception: *mut uw::_Unwind_Exception,
+    context: *mut uw::_Unwind_Context,
+    _stop_parameter: *mut c_void,
+) -> uw::_Unwind_Reason_Code {
     unsafe {
         let load_addr = KERNEL_IMAGE.as_ref().unwrap().get_load_addr();
         let backtrace_size = EXCEPTION_BUFFER.backtrace_size;
@@ -406,7 +422,7 @@ extern fn stop_fn(_version: c_int,
 }
 
 // Must be kept in sync with preallocate_runtime_exception_names() in artiq/language/embedding_map.py
-static EXCEPTION_ID_LOOKUP: [(&str, u32); 11] = [
+static EXCEPTION_ID_LOOKUP: [(&str, u32); 12] = [
     ("RuntimeError", 0),
     ("RTIOUnderflow", 1),
     ("RTIOOverflow", 2),
@@ -418,12 +434,13 @@ static EXCEPTION_ID_LOOKUP: [(&str, u32); 11] = [
     ("ZeroDivisionError", 8),
     ("IndexError", 9),
     ("UnwrapNoneError", 10),
+    ("SubkernelError", 11),
 ];
 
 pub fn get_exception_id(name: &str) -> u32 {
     for (n, id) in EXCEPTION_ID_LOOKUP.iter() {
         if *n == name {
-            return *id
+            return *id;
         }
     }
     unimplemented!("unallocated internal exception id")
@@ -431,25 +448,24 @@ pub fn get_exception_id(name: &str) -> u32 {
 
 #[macro_export]
 macro_rules! artiq_raise {
-    ($name:expr, $message:expr, $param0:expr, $param1:expr, $param2:expr) => ({
+    ($name:expr, $message:expr, $param0:expr, $param1:expr, $param2:expr) => {{
         use cslice::AsCSlice;
         let name_id = $crate::eh_artiq::get_exception_id($name);
+        let message_cl = $message.clone();
         let exn = $crate::eh_artiq::Exception {
-            id:       name_id,
-            file:     file!().as_c_slice(),
-            line:     line!(),
-            column:   column!(),
+            id: name_id,
+            file: file!().as_c_slice(),
+            line: line!(),
+            column: column!(),
             // https://github.com/rust-lang/rfcs/pull/1719
             function: "(Rust function)".as_c_slice(),
-            message:  $message.as_c_slice(),
-            param:    [$param0, $param1, $param2]
+            message: message_cl.as_c_slice(),
+            param: [$param0, $param1, $param2],
         };
         #[allow(unused_unsafe)]
         unsafe {
             $crate::eh_artiq::raise(&exn)
         }
-    });
-    ($name:expr, $message:expr) => ({
-        artiq_raise!($name, $message, 0, 0, 0)
-    });
+    }};
+    ($name:expr, $message:expr) => {{ artiq_raise!($name, $message, 0, 0, 0) }};
 }

src/libksupport/src/i2c.rs

@@ -1,9 +1,10 @@
 use libboard_zynq;
+
 use crate::artiq_raise;
 
 pub static mut I2C_BUS: Option<libboard_zynq::i2c::I2c> = None;
 
-pub extern fn start(busno: i32) {
+pub extern "C" fn start(busno: i32) {
     if busno > 0 {
         artiq_raise!("I2CError", "I2C bus could not be accessed");
     }
@@ -14,7 +15,7 @@ pub extern fn start(busno: i32) {
     }
 }
 
-pub extern fn restart(busno: i32) {
+pub extern "C" fn restart(busno: i32) {
     if busno > 0 {
         artiq_raise!("I2CError", "I2C bus could not be accessed");
     }
@@ -25,7 +26,7 @@ pub extern fn restart(busno: i32) {
     }
 }
 
-pub extern fn stop(busno: i32) {
+pub extern "C" fn stop(busno: i32) {
     if busno > 0 {
         artiq_raise!("I2CError", "I2C bus could not be accessed");
     }
@@ -36,7 +37,7 @@ pub extern fn stop(busno: i32) {
     }
 }
 
-pub extern fn write(busno: i32, data: i32) -> bool {
+pub extern "C" fn write(busno: i32, data: i32) -> bool {
     if busno > 0 {
         artiq_raise!("I2CError", "I2C bus could not be accessed");
     }
@@ -48,7 +49,7 @@ pub extern fn write(busno: i32, data: i32) -> bool {
     }
 }
 
-pub extern fn read(busno: i32, ack: bool) -> i32 {
+pub extern "C" fn read(busno: i32, ack: bool) -> i32 {
     if busno > 0 {
         artiq_raise!("I2CError", "I2C bus could not be accessed");
     }
@@ -60,11 +61,12 @@ pub extern fn read(busno: i32, ack: bool) -> i32 {
     }
 }
 
-pub extern fn switch_select(busno: i32, address: i32, mask: i32) {
+pub extern "C" fn switch_select(busno: i32, address: i32, mask: i32) {
     if busno > 0 {
         artiq_raise!("I2CError", "I2C bus could not be accessed");
     }
-    let ch = match mask { //decode from mainline, PCA9548-centric API
+    let ch = match mask {
+        //decode from mainline, PCA9548-centric API
         0x00 => None,
         0x01 => Some(0),
         0x02 => Some(1),
@@ -74,10 +76,15 @@ pub extern fn switch_select(busno: i32, address: i32, mask: i32) {
         0x20 => Some(5),
         0x40 => Some(6),
         0x80 => Some(7),
-        _ => artiq_raise!("I2CError", "switch select supports only one channel")
+        _ => artiq_raise!("I2CError", "switch select supports only one channel"),
     };
     unsafe {
-        if (&mut I2C_BUS).as_mut().unwrap().pca954x_select(address as u8, ch).is_err() {
+        if (&mut I2C_BUS)
+            .as_mut()
+            .unwrap()
+            .pca954x_select(address as u8, ch)
+            .is_err()
+        {
             artiq_raise!("I2CError", "switch select failed");
         }
     }

src/libksupport/src/irq.rs

@@ -1,12 +1,9 @@
-use libboard_zynq::{gic, mpcore, println, stdio};
-use libcortex_a9::{
-    asm, interrupt_handler,
-    regs::MPIDR,
-    spin_lock_yield, notify_spin_lock
-};
-use libregister::RegisterR;
 use core::sync::atomic::{AtomicBool, Ordering};
 
+use libboard_zynq::{gic, mpcore, println, stdio};
+use libcortex_a9::{asm, interrupt_handler, notify_spin_lock, regs::MPIDR, spin_lock_yield};
+use libregister::RegisterR;
+
 extern "C" {
     static mut __stack1_start: u32;
     fn main_core1() -> !;

src/libksupport/src/kernel/api.rs

@@ -1,29 +1,27 @@
-use core::ffi::VaList;
-use core::ptr;
-use core::str;
+use alloc::vec;
+use core::{ffi::VaList, ptr, str};
+
 use libc::{c_char, c_int, size_t};
 use libm;
 use log::{info, warn};
 
-use alloc::vec;
-
-use crate::eh_artiq;
-use crate::rtio;
-use crate::i2c;
-use super::rpc::{rpc_send, rpc_send_async, rpc_recv};
-use super::dma;
-use super::cache;
-use super::core1::rtio_get_destination_status;
+#[cfg(has_drtio)]
+use super::subkernel;
+use super::{cache,
+            core1::rtio_get_destination_status,
+            dma,
+            rpc::{rpc_recv, rpc_send, rpc_send_async}};
+use crate::{eh_artiq, i2c, rtio};
 
 extern "C" {
     fn vsnprintf_(buffer: *mut c_char, count: size_t, format: *const c_char, va: VaList) -> c_int;
 }
 
-unsafe extern fn core_log(fmt: *const c_char, mut args: ...) {
+unsafe extern "C" fn core_log(fmt: *const c_char, mut args: ...) {
     let size = vsnprintf_(ptr::null_mut(), 0, fmt, args.as_va_list()) as usize;
     let mut buf = vec![0; size + 1];
     vsnprintf_(buf.as_mut_ptr() as *mut i8, size + 1, fmt, args.as_va_list());
-    let buf: &[u8] = &buf.as_slice()[..size-1]; // strip \n and NUL
+    let buf: &[u8] = &buf.as_slice()[..size - 1]; // strip \n and NUL
     match str::from_utf8(buf) {
         Ok(s) => info!("kernel: {}", s),
         Err(e) => {
@@ -33,14 +31,13 @@ unsafe extern fn core_log(fmt: *const c_char, mut args: ...) {
     }
 }
 
-unsafe extern fn rtio_log(fmt: *const c_char, mut args: ...) {
+unsafe extern "C" fn rtio_log(fmt: *const c_char, mut args: ...) {
     let size = vsnprintf_(ptr::null_mut(), 0, fmt, args.as_va_list()) as usize;
     let mut buf = vec![0; size + 1];
     vsnprintf_(buf.as_mut_ptr(), size + 1, fmt, args.as_va_list());
     rtio::write_log(buf.as_slice());
 }
 
-
 macro_rules! api {
     ($i:ident) => ({
         extern { static $i: u8; }
@@ -56,24 +53,25 @@ macro_rules! api {
 }
 
 macro_rules! api_libm_f64f64 {
-    ($i:ident) => ({
-        extern fn $i(x: f64) -> f64 {
+    ($i:ident) => {{
+        extern "C" fn $i(x: f64) -> f64 {
             libm::$i(x)
         }
         api!($i = $i)
-    })
+    }};
 }
 
 macro_rules! api_libm_f64f64f64 {
-    ($i:ident) => ({
-        extern fn $i(x: f64, y: f64) -> f64 {
+    ($i:ident) => {{
+        extern "C" fn $i(x: f64, y: f64) -> f64 {
             libm::$i(x, y)
         }
         api!($i = $i)
-    })
+    }};
 }
 
 pub fn resolve(required: &[u8]) -> Option<u32> {
+    #[rustfmt::skip]
     let api = &[
         // timing
         api!(now_mu = rtio::now_mu),
@@ -118,12 +116,23 @@ pub fn resolve(required: &[u8]) -> Option<u32> {
         api!(i2c_read = i2c::read),
         api!(i2c_switch_select = i2c::switch_select),
 
+        // subkernel
+        #[cfg(has_drtio)]
+        api!(subkernel_load_run = subkernel::load_run),
+        #[cfg(has_drtio)]
+        api!(subkernel_await_finish = subkernel::await_finish),
+        #[cfg(has_drtio)]
+        api!(subkernel_send_message = subkernel::send_message),
+        #[cfg(has_drtio)]
+        api!(subkernel_await_message = subkernel::await_message),
+
         // Double-precision floating-point arithmetic helper functions
         // RTABI chapter 4.1.2, Table 2
         api!(__aeabi_dadd),
         api!(__aeabi_ddiv),
         api!(__aeabi_dmul),
         api!(__aeabi_dsub),
+
         // Double-precision floating-point comparison helper functions
         // RTABI chapter 4.1.2, Table 3
         api!(__aeabi_dcmpeq),
@@ -133,12 +142,14 @@ pub fn resolve(required: &[u8]) -> Option<u32> {
         api!(__aeabi_dcmpge),
         api!(__aeabi_dcmpgt),
         api!(__aeabi_dcmpun),
+
         // Single-precision floating-point arithmetic helper functions
         // RTABI chapter 4.1.2, Table 4
         api!(__aeabi_fadd),
         api!(__aeabi_fdiv),
         api!(__aeabi_fmul),
         api!(__aeabi_fsub),
+
         // Single-precision floating-point comparison helper functions
         // RTABI chapter 4.1.2, Table 5
         api!(__aeabi_fcmpeq),
@@ -148,6 +159,7 @@ pub fn resolve(required: &[u8]) -> Option<u32> {
         api!(__aeabi_fcmpge),
         api!(__aeabi_fcmpgt),
         api!(__aeabi_fcmpun),
+
         // Floating-point to integer conversions.
         // RTABI chapter 4.1.2, Table 6
         api!(__aeabi_d2iz),
@@ -158,9 +170,11 @@ pub fn resolve(required: &[u8]) -> Option<u32> {
         api!(__aeabi_f2uiz),
         api!(__aeabi_f2lz),
         api!(__aeabi_f2ulz),
+
         // Conversions between floating types.
         // RTABI chapter 4.1.2, Table 7
         api!(__aeabi_f2d),
+
         // Integer to floating-point conversions.
         // RTABI chapter 4.1.2, Table 8
         api!(__aeabi_i2d),
@@ -171,12 +185,14 @@ pub fn resolve(required: &[u8]) -> Option<u32> {
         api!(__aeabi_ui2f),
         api!(__aeabi_l2f),
         api!(__aeabi_ul2f),
+
         // Long long helper functions
         // RTABI chapter 4.2, Table 9
         api!(__aeabi_lmul),
         api!(__aeabi_llsl),
         api!(__aeabi_llsr),
         api!(__aeabi_lasr),
+
         // Integer division functions
         // RTABI chapter 4.3.1
         api!(__aeabi_idiv),
@@ -184,6 +200,7 @@ pub fn resolve(required: &[u8]) -> Option<u32> {
         api!(__aeabi_idivmod),
         api!(__aeabi_uidiv),
         api!(__aeabi_uldivmod),
+
         // 4.3.4 Memory copying, clearing, and setting
         api!(__aeabi_memcpy8),
         api!(__aeabi_memcpy4),
@@ -199,10 +216,30 @@ pub fn resolve(required: &[u8]) -> Option<u32> {
         api!(__aeabi_memclr),
 
         // libc
-        api!(memcpy, extern { fn memcpy(dest: *mut u8, src: *const u8, n: usize) -> *mut u8; }),
-        api!(memmove, extern { fn memmove(dest: *mut u8, src: *const u8, n: usize) -> *mut u8; }),
-        api!(memset, extern { fn memset(s: *mut u8, c: i32, n: usize) -> *mut u8; }),
-        api!(memcmp, extern { fn memcmp(s1: *const u8, s2: *const u8, n: usize) -> i32; }),
+        api!(
+            memcpy,
+            extern "C" {
+                fn memcpy(dest: *mut u8, src: *const u8, n: usize) -> *mut u8;
+            }
+        ),
+        api!(
+            memmove,
+            extern "C" {
+                fn memmove(dest: *mut u8, src: *const u8, n: usize) -> *mut u8;
+            }
+        ),
+        api!(
+            memset,
+            extern "C" {
+                fn memset(s: *mut u8, c: i32, n: usize) -> *mut u8;
+            }
+        ),
+        api!(
+            memcmp,
+            extern "C" {
+                fn memcmp(s1: *const u8, s2: *const u8, n: usize) -> i32;
+            }
+        ),
 
         // exceptions
         api!(_Unwind_Resume = unwind::_Unwind_Resume),
@@ -210,6 +247,7 @@ pub fn resolve(required: &[u8]) -> Option<u32> {
         api!(__nac3_raise = eh_artiq::raise),
         api!(__nac3_resume = eh_artiq::resume),
         api!(__nac3_end_catch = eh_artiq::end_catch),
+
         // legacy exception symbols
         api!(__artiq_personality = eh_artiq::artiq_personality),
         api!(__artiq_raise = eh_artiq::raise),
@@ -241,7 +279,7 @@ pub fn resolve(required: &[u8]) -> Option<u32> {
         api_libm_f64f64!(fabs),
         api_libm_f64f64!(floor),
         {
-            extern fn fma(x: f64, y: f64, z: f64) -> f64 {
+            extern "C" fn fma(x: f64, y: f64, z: f64) -> f64 {
                 libm::fma(x, y, z)
             }
             api!(fma = fma)
@@ -253,7 +291,7 @@ pub fn resolve(required: &[u8]) -> Option<u32> {
         api_libm_f64f64!(j0),
         api_libm_f64f64!(j1),
         {
-            extern fn jn(n: i32, x: f64) -> f64 {
+            extern "C" fn jn(n: i32, x: f64) -> f64 {
                 libm::jn(n, x)
             }
             api!(jn = jn)
@@ -275,13 +313,13 @@ pub fn resolve(required: &[u8]) -> Option<u32> {
         api_libm_f64f64!(y0),
         api_libm_f64f64!(y1),
         {
-            extern fn yn(n: i32, x: f64) -> f64 {
+            extern "C" fn yn(n: i32, x: f64) -> f64 {
                 libm::yn(n, x)
             }
             api!(yn = yn)
         },
     ];
     api.iter()
-       .find(|&&(exported, _)| exported.as_bytes() == required)
-       .map(|&(_, ptr)| ptr as u32)
+        .find(|&&(exported, _)| exported.as_bytes() == required)
+        .map(|&(_, ptr)| ptr as u32)
 }

src/libksupport/src/kernel/cache.rs

@@ -1,12 +1,17 @@
-use alloc::{string::String, boxed::Box};
-use cslice::{CSlice, AsCSlice};
-use core::mem::{transmute, forget};
-use super::{KERNEL_CHANNEL_0TO1, KERNEL_CHANNEL_1TO0, Message};
+use alloc::{boxed::Box, string::String};
+use core::mem::{forget, transmute};
 
-pub extern fn get(key: CSlice<u8>) -> &CSlice<'static, i32> {
+use cslice::{AsCSlice, CSlice};
+
+use super::{Message, KERNEL_CHANNEL_0TO1, KERNEL_CHANNEL_1TO0};
+
+pub extern "C" fn get(key: CSlice<u8>) -> &CSlice<'static, i32> {
     let key = String::from_utf8(key.as_ref().to_vec()).unwrap();
     unsafe {
-        KERNEL_CHANNEL_1TO0.as_mut().unwrap().send(Message::CacheGetRequest(key));
+        KERNEL_CHANNEL_1TO0
+            .as_mut()
+            .unwrap()
+            .send(Message::CacheGetRequest(key));
         let msg = KERNEL_CHANNEL_0TO1.as_mut().unwrap().recv();
         if let Message::CacheGetReply(v) = msg {
             let leaked = Box::new(v.as_c_slice());
@@ -20,11 +25,13 @@ pub extern fn get(key: CSlice<u8>) -> &CSlice<'static, i32> {
     }
 }
 
-pub extern fn put(key: CSlice<u8>, list: &CSlice<i32>) {
+pub extern "C" fn put(key: CSlice<u8>, list: &CSlice<i32>) {
     let key = String::from_utf8(key.as_ref().to_vec()).unwrap();
     let value = list.as_ref().to_vec();
     unsafe {
-        KERNEL_CHANNEL_1TO0.as_mut().unwrap().send(Message::CachePutRequest(key, value));
+        KERNEL_CHANNEL_1TO0
+            .as_mut()
+            .unwrap()
+            .send(Message::CachePutRequest(key, value));
     }
 }
-

src/libksupport/src/kernel/control.rs

@@ -1,11 +1,11 @@
-use libcortex_a9::sync_channel::{Sender, Receiver};
+use core::mem::{forget, replace};
+
+use libcortex_a9::sync_channel::{Receiver, Sender};
 use libsupport_zynq::boot::Core1;
 
-use super::{CHANNEL_0TO1, CHANNEL_1TO0, CHANNEL_SEM, INIT_LOCK, Message};
+use super::{Message, CHANNEL_0TO1, CHANNEL_1TO0, CHANNEL_SEM, INIT_LOCK};
 use crate::irq::restart_core1;
 
-use core::mem::{forget, replace};
-
 pub struct Control {
     pub tx: Sender<'static, Message>,
     pub rx: Receiver<'static, Message>,
@@ -53,4 +53,3 @@ impl Control {
         forget(replace(&mut self.rx, core0_rx));
     }
 }
-

src/libksupport/src/kernel/core1.rs

@@ -1,31 +1,20 @@
 //! Kernel prologue/epilogue that runs on the 2nd CPU core
 
-use core::{mem, ptr, cell::UnsafeCell};
 use alloc::borrow::ToOwned;
-use log::{debug, info, error};
-use cslice::CSlice;
+use core::{cell::UnsafeCell, mem, ptr};
 
-use libcortex_a9::{
-    enable_fpu,
-    cache::{dcci_slice, iciallu, bpiall},
-    asm::{dsb, isb},
-    sync_channel,
-};
-use libboard_zynq::{mpcore, gic};
+use cslice::CSlice;
+use dyld::{self, elf::EXIDX_Entry, Library};
+use libboard_zynq::{gic, mpcore};
+use libcortex_a9::{asm::{dsb, isb},
+                   cache::{bpiall, dcci_slice, iciallu},
+                   enable_fpu, sync_channel};
 use libsupport_zynq::ram;
-use dyld::{self, Library, elf::EXIDX_Entry};
+use log::{debug, error, info};
+
+use super::{api::resolve, dma, rpc::rpc_send_async, Message, CHANNEL_0TO1, CHANNEL_1TO0, CHANNEL_SEM, INIT_LOCK,
+            KERNEL_CHANNEL_0TO1, KERNEL_CHANNEL_1TO0, KERNEL_IMAGE};
 use crate::{eh_artiq, get_async_errors, rtio};
-use super::{
-    api::resolve,
-    rpc::rpc_send_async,
-    INIT_LOCK,
-    CHANNEL_0TO1, CHANNEL_1TO0,
-    CHANNEL_SEM,
-    KERNEL_CHANNEL_0TO1, KERNEL_CHANNEL_1TO0,
-    KERNEL_IMAGE,
-    Message,
-    dma,
-};
 
 // linker symbols
 extern "C" {
@@ -38,13 +27,13 @@ extern "C" {
 unsafe fn attribute_writeback(typeinfo: *const ()) {
     struct Attr {
         offset: usize,
-        tag:    CSlice<'static, u8>,
-        name:   CSlice<'static, u8>
+        tag: CSlice<'static, u8>,
+        name: CSlice<'static, u8>,
     }
 
     struct Type {
         attributes: *const *const Attr,
-        objects:    *const *const ()
+        objects: *const *const (),
     }
 
     let mut tys = typeinfo as *const *const Type;
@@ -63,11 +52,16 @@ unsafe fn attribute_writeback(typeinfo: *const ()) {
                 attributes = attributes.offset(1);
 
                 if (*attribute).tag.len() > 0 {
-                    rpc_send_async(0, &(*attribute).tag, [
-                        &object as *const _ as *const (),
-                        &(*attribute).name as *const _ as *const (),
-                        (object as usize + (*attribute).offset) as *const ()
-                    ].as_ptr());
+                    rpc_send_async(
+                        0,
+                        &(*attribute).tag,
+                        [
+                            &object as *const _ as *const (),
+                            &(*attribute).name as *const _ as *const (),
+                            (object as usize + (*attribute).offset) as *const (),
+                        ]
+                        .as_ptr(),
+                    );
                 }
             }
         }
@@ -82,7 +76,8 @@ pub struct KernelImage {
 
 impl KernelImage {
     pub fn new(library: Library) -> Result<Self, dyld::Error> {
-        let __modinit__ = library.lookup(b"__modinit__")
+        let __modinit__ = library
+            .lookup(b"__modinit__")
             .ok_or(dyld::Error::Lookup("__modinit__".to_owned()))?;
         let typeinfo = library.lookup(b"typeinfo");
 
@@ -90,8 +85,7 @@ impl KernelImage {
         let bss_start = library.lookup(b"__bss_start");
         let end = library.lookup(b"_end");
         if let Some(bss_start) = bss_start {
-            let end = end
-                .ok_or(dyld::Error::Lookup("_end".to_owned()))?;
+            let end = end.ok_or(dyld::Error::Lookup("_end".to_owned()))?;
             unsafe {
                 ptr::write_bytes(bss_start as *mut u8, 0, (end - bss_start) as usize);
             }
@@ -126,9 +120,7 @@ impl KernelImage {
     }
 
     pub fn get_load_addr(&self) -> usize {
-        unsafe {
-            self.library.get().as_ref().unwrap().image.as_ptr() as usize
-        }
+        unsafe { self.library.get().as_ref().unwrap().image.as_ptr() as usize }
     }
 }
 
@@ -164,20 +156,19 @@ pub extern "C" fn main_core1() {
         let message = core1_rx.recv();
         match message {
             Message::LoadRequest(data) => {
-                let result = dyld::load(&data, &resolve)
-                    .and_then(KernelImage::new);
+                let result = dyld::load(&data, &resolve).and_then(KernelImage::new);
                 match result {
                     Ok(kernel) => {
                         loaded_kernel = Some(kernel);
                         debug!("kernel loaded");
                         core1_tx.send(Message::LoadCompleted);
-                    },
+                    }
                     Err(error) => {
                         error!("failed to load shared library: {}", error);
                         core1_tx.send(Message::LoadFailed);
                     }
                 }
-            },
+            }
             Message::StartRequest => {
                 info!("kernel starting");
                 if let Some(kernel) = loaded_kernel.take() {
@@ -202,9 +193,11 @@ pub extern "C" fn main_core1() {
 }
 
 /// Called by eh_artiq
-pub fn terminate(exceptions: &'static [Option<eh_artiq::Exception<'static>>],
-                 stack_pointers: &'static [eh_artiq::StackPointerBacktrace],
-                 backtrace: &'static mut [(usize, usize)]) -> ! {
+pub fn terminate(
+    exceptions: &'static [Option<eh_artiq::Exception<'static>>],
+    stack_pointers: &'static [eh_artiq::StackPointerBacktrace],
+    backtrace: &'static mut [(usize, usize)],
+) -> ! {
     {
         let core1_tx = unsafe { KERNEL_CHANNEL_1TO0.as_mut().unwrap() };
         let errors = unsafe { get_async_errors() };
@@ -215,26 +208,34 @@ pub fn terminate(exceptions: &'static [Option<eh_artiq::Exception<'static>>],
 
 /// Called by llvm_libunwind
 #[no_mangle]
-extern fn dl_unwind_find_exidx(pc: *const u32, len_ptr: *mut u32) -> *const u32 {
+extern "C" fn dl_unwind_find_exidx(pc: *const u32, len_ptr: *mut u32) -> *const u32 {
     let length;
     let start: *const EXIDX_Entry;
     unsafe {
         if &__text_start as *const u32 <= pc && pc < &__text_end as *const u32 {
             length = (&__exidx_end as *const EXIDX_Entry).offset_from(&__exidx_start) as u32;
             start = &__exidx_start;
-        } else {
-            let exidx = KERNEL_IMAGE.as_ref()
+        } else if KERNEL_IMAGE != ptr::null() {
+            let exidx = KERNEL_IMAGE
+                .as_ref()
                 .expect("dl_unwind_find_exidx kernel image")
-                .library.get().as_ref().unwrap().exidx();
+                .library
+                .get()
+                .as_ref()
+                .unwrap()
+                .exidx();
             length = exidx.len() as u32;
             start = exidx.as_ptr();
+        } else {
+            length = 0;
+            start = ptr::null();
         }
         *len_ptr = length;
     }
     start as *const u32
 }
 
-pub extern fn rtio_get_destination_status(destination: i32) -> bool {
+pub extern "C" fn rtio_get_destination_status(destination: i32) -> bool {
     #[cfg(has_drtio)]
     if destination > 0 && destination < 255 {
         let reply = unsafe {
@@ -245,7 +246,7 @@ pub extern fn rtio_get_destination_status(destination: i32) -> bool {
         };
         return match reply {
             Message::UpDestinationsReply(x) => x,
-            _ => panic!("received unexpected reply to UpDestinationsRequest: {:?}", reply)
+            _ => panic!("received unexpected reply to UpDestinationsRequest: {:?}", reply),
         };
     }
 

src/libksupport/src/kernel/dma.rs

@@ -0,0 +1,254 @@
+use alloc::{string::String, vec::Vec};
+use core::mem;
+
+use cslice::CSlice;
+
+use super::{Message, KERNEL_CHANNEL_0TO1, KERNEL_CHANNEL_1TO0, KERNEL_IMAGE};
+use crate::{artiq_raise, pl::csr, rtio};
+
+#[repr(C)]
+pub struct DmaTrace {
+    duration: i64,
+    address: i32,
+    uses_ddma: bool,
+}
+
+#[derive(Clone, Debug)]
+pub struct DmaRecorder {
+    pub name: String,
+    pub buffer: Vec<u8>,
+    pub duration: i64,
+    pub enable_ddma: bool,
+}
+
+static mut RECORDER: Option<DmaRecorder> = None;
+
+pub unsafe fn init_dma_recorder() {
+    // as static would remain after restart, we have to reset it,
+    // without running its destructor.
+    mem::forget(mem::replace(&mut RECORDER, None));
+}
+
+pub extern "C" fn dma_record_start(name: CSlice<u8>) {
+    let name = String::from_utf8(name.as_ref().to_vec()).unwrap();
+    unsafe {
+        KERNEL_CHANNEL_1TO0
+            .as_mut()
+            .unwrap()
+            .send(Message::DmaEraseRequest(name.clone()));
+    }
+    unsafe {
+        if RECORDER.is_some() {
+            artiq_raise!("DMAError", "DMA is already recording")
+        }
+
+        let library = KERNEL_IMAGE.as_ref().unwrap();
+        library.rebind(b"rtio_output", dma_record_output as *const ()).unwrap();
+        library
+            .rebind(b"rtio_output_wide", dma_record_output_wide as *const ())
+            .unwrap();
+
+        RECORDER = Some(DmaRecorder {
+            name,
+            buffer: Vec::new(),
+            duration: 0,
+            enable_ddma: false,
+        });
+    }
+}
+
+pub extern "C" fn dma_record_stop(duration: i64, enable_ddma: bool) {
+    unsafe {
+        if RECORDER.is_none() {
+            artiq_raise!("DMAError", "DMA is not recording")
+        }
+
+        let library = KERNEL_IMAGE.as_ref().unwrap();
+        library.rebind(b"rtio_output", rtio::output as *const ()).unwrap();
+        library
+            .rebind(b"rtio_output_wide", rtio::output_wide as *const ())
+            .unwrap();
+
+        let mut recorder = RECORDER.take().unwrap();
+        recorder.duration = duration;
+        recorder.enable_ddma = enable_ddma;
+        KERNEL_CHANNEL_1TO0
+            .as_mut()
+            .unwrap()
+            .send(Message::DmaPutRequest(recorder));
+    }
+}
+
+#[inline(always)]
+unsafe fn dma_record_output_prepare(timestamp: i64, target: i32, words: usize) {
+    // See gateware/rtio/dma.py.
+    const HEADER_LENGTH: usize = /*length*/1 + /*channel*/3 + /*timestamp*/8 + /*address*/1;
+    let length = HEADER_LENGTH + /*data*/words * 4;
+
+    let buffer = &mut RECORDER.as_mut().unwrap().buffer;
+    buffer.reserve(length);
+    buffer.extend_from_slice(&[
+        (length >> 0) as u8,
+        (target >> 8) as u8,
+        (target >> 16) as u8,
+        (target >> 24) as u8,
+        (timestamp >> 0) as u8,
+        (timestamp >> 8) as u8,
+        (timestamp >> 16) as u8,
+        (timestamp >> 24) as u8,
+        (timestamp >> 32) as u8,
+        (timestamp >> 40) as u8,
+        (timestamp >> 48) as u8,
+        (timestamp >> 56) as u8,
+        (target >> 0) as u8,
+    ]);
+}
+
+pub extern "C" fn dma_record_output(target: i32, word: i32) {
+    unsafe {
+        let timestamp = rtio::now_mu();
+        dma_record_output_prepare(timestamp, target, 1);
+        RECORDER.as_mut().unwrap().buffer.extend_from_slice(&[
+            (word >> 0) as u8,
+            (word >> 8) as u8,
+            (word >> 16) as u8,
+            (word >> 24) as u8,
+        ]);
+    }
+}
+
+pub extern "C" fn dma_record_output_wide(target: i32, words: &CSlice<i32>) {
+    assert!(words.len() <= 16); // enforce the hardware limit
+
+    unsafe {
+        let timestamp = rtio::now_mu();
+        dma_record_output_prepare(timestamp, target, words.len());
+        let buffer = &mut RECORDER.as_mut().unwrap().buffer;
+        for word in words.as_ref().iter() {
+            buffer.extend_from_slice(&[
+                (word >> 0) as u8,
+                (word >> 8) as u8,
+                (word >> 16) as u8,
+                (word >> 24) as u8,
+            ]);
+        }
+    }
+}
+
+pub extern "C" fn dma_erase(name: CSlice<u8>) {
+    let name = String::from_utf8(name.as_ref().to_vec()).unwrap();
+    unsafe {
+        KERNEL_CHANNEL_1TO0
+            .as_mut()
+            .unwrap()
+            .send(Message::DmaEraseRequest(name));
+    }
+}
+
+pub extern "C" fn dma_retrieve(name: CSlice<u8>) -> DmaTrace {
+    let name = String::from_utf8(name.as_ref().to_vec()).unwrap();
+    unsafe {
+        KERNEL_CHANNEL_1TO0.as_mut().unwrap().send(Message::DmaGetRequest(name));
+    }
+    match unsafe { KERNEL_CHANNEL_0TO1.as_mut().unwrap() }.recv() {
+        Message::DmaGetReply(None) => (),
+        Message::DmaGetReply(Some((address, duration, uses_ddma))) => {
+            return DmaTrace {
+                address,
+                duration,
+                uses_ddma,
+            };
+        }
+        _ => panic!("Expected DmaGetReply after DmaGetRequest!"),
+    }
+    // we have to defer raising error as we have to drop the message first...
+    artiq_raise!("DMAError", "DMA trace not found");
+}
+
+pub extern "C" fn dma_playback(timestamp: i64, ptr: i32, _uses_ddma: bool) {
+    unsafe {
+        csr::rtio_dma::base_address_write(ptr as u32);
+        csr::rtio_dma::time_offset_write(timestamp as u64);
+
+        csr::cri_con::selected_write(1);
+        csr::rtio_dma::enable_write(1);
+        #[cfg(has_drtio)]
+        if _uses_ddma {
+            KERNEL_CHANNEL_1TO0
+                .as_mut()
+                .unwrap()
+                .send(Message::DmaStartRemoteRequest {
+                    id: ptr,
+                    timestamp: timestamp,
+                });
+        }
+        while csr::rtio_dma::enable_read() != 0 {}
+        csr::cri_con::selected_write(0);
+
+        let error = csr::rtio_dma::error_read();
+        if error != 0 {
+            let timestamp = csr::rtio_dma::error_timestamp_read();
+            let channel = csr::rtio_dma::error_channel_read();
+            csr::rtio_dma::error_write(1);
+            if error & 1 != 0 {
+                artiq_raise!(
+                    "RTIOUnderflow",
+                    "RTIO underflow at {1} mu, channel {rtio_channel_info:0}",
+                    channel as i64,
+                    timestamp as i64,
+                    0
+                );
+            }
+            if error & 2 != 0 {
+                artiq_raise!(
+                    "RTIODestinationUnreachable",
+                    "RTIO destination unreachable, output, at {1} mu, channel {rtio_channel_info:0}",
+                    channel as i64,
+                    timestamp as i64,
+                    0
+                );
+            }
+        }
+        #[cfg(has_drtio)]
+        if _uses_ddma {
+            KERNEL_CHANNEL_1TO0
+                .as_mut()
+                .unwrap()
+                .send(Message::DmaAwaitRemoteRequest(ptr));
+            match KERNEL_CHANNEL_0TO1.as_mut().unwrap().recv() {
+                Message::DmaAwaitRemoteReply {
+                    timeout,
+                    error,
+                    channel,
+                    timestamp,
+                } => {
+                    if timeout {
+                        artiq_raise!(
+                            "DMAError",
+                            "Error running DMA on satellite device, timed out waiting for results"
+                        );
+                    }
+                    if error & 1 != 0 {
+                        artiq_raise!(
+                            "RTIOUnderflow",
+                            "RTIO underflow at {1} mu, channel {rtio_channel_info:0}",
+                            channel as i64,
+                            timestamp as i64,
+                            0
+                        );
+                    }
+                    if error & 2 != 0 {
+                        artiq_raise!(
+                            "RTIODestinationUnreachable",
+                            "RTIO destination unreachable, output, at {1} mu, channel {rtio_channel_info:0}",
+                            channel as i64,
+                            timestamp as i64,
+                            0
+                        );
+                    }
+                }
+                _ => panic!("Expected DmaAwaitRemoteReply after DmaAwaitRemoteRequest!"),
+            }
+        }
+    }
+}

src/libksupport/src/kernel/mod.rs

@@ -0,0 +1,124 @@
+use alloc::{string::String, vec::Vec};
+use core::ptr;
+
+use libcortex_a9::{mutex::Mutex, semaphore::Semaphore, sync_channel};
+
+use crate::{eh_artiq, RPCException};
+
+mod control;
+pub use control::Control;
+mod api;
+pub mod core1;
+mod dma;
+mod rpc;
+pub use dma::DmaRecorder;
+mod cache;
+#[cfg(has_drtio)]
+mod subkernel;
+
+#[cfg(has_drtio)]
+#[derive(Debug, Clone)]
+pub enum SubkernelStatus {
+    NoError,
+    Timeout,
+    IncorrectState,
+    CommLost,
+    OtherError,
+}
+
+#[derive(Debug, Clone)]
+pub enum Message {
+    LoadRequest(Vec<u8>),
+    LoadCompleted,
+    LoadFailed,
+    StartRequest,
+    KernelFinished(u8),
+    KernelException(
+        &'static [Option<eh_artiq::Exception<'static>>],
+        &'static [eh_artiq::StackPointerBacktrace],
+        &'static [(usize, usize)],
+        u8,
+    ),
+    RpcSend {
+        is_async: bool,
+        data: Vec<u8>,
+    },
+    RpcRecvRequest(*mut ()),
+    RpcRecvReply(Result<usize, RPCException>),
+
+    CacheGetRequest(String),
+    CacheGetReply(Vec<i32>),
+    CachePutRequest(String, Vec<i32>),
+
+    DmaPutRequest(DmaRecorder),
+    DmaEraseRequest(String),
+    DmaGetRequest(String),
+    DmaGetReply(Option<(i32, i64, bool)>),
+    #[cfg(has_drtio)]
+    DmaStartRemoteRequest {
+        id: i32,
+        timestamp: i64,
+    },
+    #[cfg(has_drtio)]
+    DmaAwaitRemoteRequest(i32),
+    #[cfg(has_drtio)]
+    DmaAwaitRemoteReply {
+        timeout: bool,
+        error: u8,
+        channel: u32,
+        timestamp: u64,
+    },
+
+    #[cfg(has_drtio)]
+    UpDestinationsRequest(i32),
+    #[cfg(has_drtio)]
+    UpDestinationsReply(bool),
+
+    #[cfg(has_drtio)]
+    SubkernelLoadRunRequest {
+        id: u32,
+        run: bool,
+    },
+    #[cfg(has_drtio)]
+    SubkernelLoadRunReply {
+        succeeded: bool,
+    },
+    #[cfg(has_drtio)]
+    SubkernelAwaitFinishRequest {
+        id: u32,
+        timeout: u64,
+    },
+    #[cfg(has_drtio)]
+    SubkernelAwaitFinishReply {
+        status: SubkernelStatus,
+    },
+    #[cfg(has_drtio)]
+    SubkernelMsgSend {
+        id: u32,
+        data: Vec<u8>,
+    },
+    #[cfg(has_drtio)]
+    SubkernelMsgSent,
+    #[cfg(has_drtio)]
+    SubkernelMsgRecvRequest {
+        id: u32,
+        timeout: u64,
+        tags: Vec<u8>,
+    },
+    #[cfg(has_drtio)]
+    SubkernelMsgRecvReply {
+        status: SubkernelStatus,
+        count: u8,
+    },
+}
+
+static CHANNEL_0TO1: Mutex<Option<sync_channel::Sender<'static, Message>>> = Mutex::new(None);
+static CHANNEL_1TO0: Mutex<Option<sync_channel::Receiver<'static, Message>>> = Mutex::new(None);
+static CHANNEL_SEM: Semaphore = Semaphore::new(0, 1);
+
+static mut KERNEL_CHANNEL_0TO1: Option<sync_channel::Receiver<'static, Message>> = None;
+static mut KERNEL_CHANNEL_1TO0: Option<sync_channel::Sender<'static, Message>> = None;
+
+pub static mut KERNEL_IMAGE: *const core1::KernelImage = ptr::null();
+
+static INIT_LOCK: Mutex<()> = Mutex::new(());

src/libksupport/src/kernel/rpc.rs

@@ -1,31 +1,28 @@
 //! Kernel-side RPC API
 
 use alloc::vec::Vec;
+
 use cslice::CSlice;
 
-use crate::eh_artiq;
-use crate::rpc::send_args;
-use super::{
-    KERNEL_CHANNEL_0TO1, KERNEL_CHANNEL_1TO0,
-    Message,
-};
+use super::{Message, KERNEL_CHANNEL_0TO1, KERNEL_CHANNEL_1TO0};
+use crate::{eh_artiq, rpc::send_args};
 
 fn rpc_send_common(is_async: bool, service: u32, tag: &CSlice<u8>, data: *const *const ()) {
     let core1_tx = unsafe { KERNEL_CHANNEL_1TO0.as_mut().unwrap() };
     let mut buffer = Vec::<u8>::new();
-    send_args(&mut buffer, service, tag.as_ref(), data).expect("RPC encoding failed");
+    send_args(&mut buffer, service, tag.as_ref(), data, true).expect("RPC encoding failed");
     core1_tx.send(Message::RpcSend { is_async, data: buffer });
 }
 
-pub extern fn rpc_send(service: u32, tag: &CSlice<u8>, data: *const *const ()) {
+pub extern "C" fn rpc_send(service: u32, tag: &CSlice<u8>, data: *const *const ()) {
     rpc_send_common(false, service, tag, data);
 }
 
-pub extern fn rpc_send_async(service: u32, tag: &CSlice<u8>, data: *const *const ()) {
+pub extern "C" fn rpc_send_async(service: u32, tag: &CSlice<u8>, data: *const *const ()) {
     rpc_send_common(true, service, tag, data);
 }
 
-pub extern fn rpc_recv(slot: *mut ()) -> usize {
+pub extern "C" fn rpc_recv(slot: *mut ()) -> usize {
     let reply = unsafe {
         let core1_rx = KERNEL_CHANNEL_0TO1.as_mut().unwrap();
         let core1_tx = KERNEL_CHANNEL_1TO0.as_mut().unwrap();
@@ -36,15 +33,15 @@ pub extern fn rpc_recv(slot: *mut ()) -> usize {
         Message::RpcRecvReply(Ok(alloc_size)) => alloc_size,
         Message::RpcRecvReply(Err(exception)) => unsafe {
             eh_artiq::raise(&eh_artiq::Exception {
-                id:       exception.id,
-                file:     CSlice::new(exception.file as *const u8, usize::MAX),
-                line:     exception.line as u32,
-                column:   exception.column as u32,
+                id: exception.id,
+                file: CSlice::new(exception.file as *const u8, usize::MAX),
+                line: exception.line as u32,
+                column: exception.column as u32,
                 function: CSlice::new(exception.function as *const u8, usize::MAX),
-                message:  CSlice::new(exception.message as *const u8, usize::MAX),
-                param:    exception.param
+                message: CSlice::new(exception.message as *const u8, usize::MAX),
+                param: exception.param,
             })
         },
-        _ => panic!("received unexpected reply to RpcRecvRequest: {:?}", reply)
+        _ => panic!("received unexpected reply to RpcRecvRequest: {:?}", reply),
     }
 }

src/libksupport/src/kernel/subkernel.rs

@@ -0,0 +1,110 @@
+use alloc::vec::Vec;
+
+use cslice::CSlice;
+
+use super::{Message, SubkernelStatus, KERNEL_CHANNEL_0TO1, KERNEL_CHANNEL_1TO0};
+use crate::{artiq_raise, rpc::send_args};
+
+pub extern "C" fn load_run(id: u32, run: bool) {
+    unsafe {
+        KERNEL_CHANNEL_1TO0
+            .as_mut()
+            .unwrap()
+            .send(Message::SubkernelLoadRunRequest { id: id, run: run });
+    }
+    match unsafe { KERNEL_CHANNEL_0TO1.as_mut().unwrap() }.recv() {
+        Message::SubkernelLoadRunReply { succeeded: true } => (),
+        Message::SubkernelLoadRunReply { succeeded: false } => {
+            artiq_raise!("SubkernelError", "Error loading or running the subkernel")
+        }
+        _ => panic!("Expected SubkernelLoadRunReply after SubkernelLoadRunRequest!"),
+    }
+}
+
+pub extern "C" fn await_finish(id: u32, timeout: u64) {
+    unsafe {
+        KERNEL_CHANNEL_1TO0
+            .as_mut()
+            .unwrap()
+            .send(Message::SubkernelAwaitFinishRequest {
+                id: id,
+                timeout: timeout,
+            });
+    }
+    match unsafe { KERNEL_CHANNEL_0TO1.as_mut().unwrap() }.recv() {
+        Message::SubkernelAwaitFinishReply {
+            status: SubkernelStatus::NoError,
+        } => (),
+        Message::SubkernelAwaitFinishReply {
+            status: SubkernelStatus::IncorrectState,
+        } => artiq_raise!("SubkernelError", "Subkernel not running"),
+        Message::SubkernelAwaitFinishReply {
+            status: SubkernelStatus::Timeout,
+        } => artiq_raise!("SubkernelError", "Subkernel timed out"),
+        Message::SubkernelAwaitFinishReply {
+            status: SubkernelStatus::CommLost,
+        } => artiq_raise!("SubkernelError", "Lost communication with satellite"),
+        Message::SubkernelAwaitFinishReply {
+            status: SubkernelStatus::OtherError,
+        } => artiq_raise!("SubkernelError", "An error occurred during subkernel operation"),
+        _ => panic!("expected SubkernelAwaitFinishReply after SubkernelAwaitFinishRequest"),
+    }
+}
+
+pub extern "C" fn send_message(id: u32, count: u8, tag: &CSlice<u8>, data: *const *const ()) {
+    let mut buffer = Vec::<u8>::new();
+    send_args(&mut buffer, 0, tag.as_ref(), data, false).expect("RPC encoding failed");
+    // overwrite service tag, include how many tags are in the message
+    buffer[3] = count;
+    unsafe {
+        KERNEL_CHANNEL_1TO0.as_mut().unwrap().send(Message::SubkernelMsgSend {
+            id: id,
+            data: buffer[3..].to_vec(),
+        });
+    }
+    match unsafe { KERNEL_CHANNEL_0TO1.as_mut().unwrap() }.recv() {
+        Message::SubkernelMsgSent => (),
+        _ => panic!("expected SubkernelMsgSent after SubkernelMsgSend"),
+    }
+}
+
+pub extern "C" fn await_message(id: u32, timeout: u64, tags: &CSlice<u8>, min: u8, max: u8) {
+    unsafe {
+        KERNEL_CHANNEL_1TO0
+            .as_mut()
+            .unwrap()
+            .send(Message::SubkernelMsgRecvRequest {
+                id: id,
+                timeout: timeout,
+                tags: tags.as_ref().to_vec(),
+            });
+    }
+    match unsafe { KERNEL_CHANNEL_0TO1.as_mut().unwrap() }.recv() {
+        Message::SubkernelMsgRecvReply {
+            status: SubkernelStatus::NoError,
+            count,
+        } => {
+            if min > count || count > max {
+                artiq_raise!("SubkernelError", "Received more or less arguments than required")
+            }
+        }
+        Message::SubkernelMsgRecvReply {
+            status: SubkernelStatus::IncorrectState,
+            ..
+        } => artiq_raise!("SubkernelError", "Subkernel not running"),
+        Message::SubkernelMsgRecvReply {
+            status: SubkernelStatus::Timeout,
+            ..
+        } => artiq_raise!("SubkernelError", "Subkernel timed out"),
+        Message::SubkernelMsgRecvReply {
+            status: SubkernelStatus::CommLost,
+            ..
+        } => artiq_raise!("SubkernelError", "Lost communication with satellite"),
+        Message::SubkernelMsgRecvReply {
+            status: SubkernelStatus::OtherError,
+            ..
+        } => artiq_raise!("SubkernelError", "An error occurred during subkernel operation"),
+        _ => panic!("expected SubkernelMsgRecvReply after SubkernelMsgRecvRequest"),
+    }
+    // RpcRecvRequest should be called after this to receive message data
+}

src/libksupport/src/lib.rs

@@ -0,0 +1,163 @@
+#![no_std]
+#![feature(c_variadic)]
+#![feature(const_btree_new)]
+#![feature(const_in_array_repeat_expressions)]
+#![feature(naked_functions)]
+#![feature(asm)]
+
+#[macro_use]
+extern crate alloc;
+
+use alloc::{collections::BTreeMap, string::String};
+
+use io::{Cursor, ProtoRead};
+use libasync::block_async;
+use libconfig::Config;
+use log::{error, warn};
+#[cfg(has_drtiosat)]
+pub use pl::csr::drtiosat as rtio_core;
+#[cfg(has_rtio_core)]
+pub use pl::csr::rtio_core;
+use void::Void;
+
+pub mod eh_artiq;
+pub mod i2c;
+pub mod irq;
+pub mod kernel;
+pub mod rpc;
+#[cfg(ki_impl = "csr")]
+#[path = "rtio_csr.rs"]
+pub mod rtio;
+#[cfg(ki_impl = "acp")]
+#[path = "rtio_acp.rs"]
+pub mod rtio;
+#[rustfmt::skip]
+#[path = "../../../build/pl.rs"]
+pub mod pl;
+
+
+#[derive(Debug, Clone)]
+pub struct RPCException {
+    pub id: u32,
+    pub message: u32,
+    pub param: [i64; 3],
+    pub file: u32,
+    pub line: i32,
+    pub column: i32,
+    pub function: u32,
+}
+
+pub static mut SEEN_ASYNC_ERRORS: u8 = 0;
+
+pub const ASYNC_ERROR_COLLISION: u8 = 1 << 0;
+pub const ASYNC_ERROR_BUSY: u8 = 1 << 1;
+pub const ASYNC_ERROR_SEQUENCE_ERROR: u8 = 1 << 2;
+
+pub unsafe fn get_async_errors() -> u8 {
+    let errors = SEEN_ASYNC_ERRORS;
+    SEEN_ASYNC_ERRORS = 0;
+    errors
+}
+
+fn wait_for_async_rtio_error() -> nb::Result<(), Void> {
+    unsafe {
+        #[cfg(has_rtio_core)]
+        let errors = rtio_core::async_error_read();
+        #[cfg(has_drtiosat)]
+        let errors = rtio_core::protocol_error_read();
+        if errors != 0 {
+            Ok(())
+        } else {
+            Err(nb::Error::WouldBlock)
+        }
+    }
+}
+
+pub async fn report_async_rtio_errors() {
+    loop {
+        let _ = block_async!(wait_for_async_rtio_error()).await;
+        unsafe {
+            #[cfg(has_rtio_core)]
+            let errors = rtio_core::async_error_read();
+            #[cfg(has_drtiosat)]
+            let errors = rtio_core::protocol_error_read();
+            if errors & ASYNC_ERROR_COLLISION != 0 {
+                let channel = rtio_core::collision_channel_read();
+                error!(
+                    "RTIO collision involving channel 0x{:04x}:{}",
+                    channel,
+                    resolve_channel_name(channel as u32)
+                );
+            }
+            if errors & ASYNC_ERROR_BUSY != 0 {
+                let channel = rtio_core::busy_channel_read();
+                error!(
+                    "RTIO busy error involving channel 0x{:04x}:{}",
+                    channel,
+                    resolve_channel_name(channel as u32)
+                );
+            }
+            if errors & ASYNC_ERROR_SEQUENCE_ERROR != 0 {
+                let channel = rtio_core::sequence_error_channel_read();
+                error!(
+                    "RTIO sequence error involving channel 0x{:04x}:{}",
+                    channel,
+                    resolve_channel_name(channel as u32)
+                );
+            }
+            SEEN_ASYNC_ERRORS = errors;
+            #[cfg(has_rtio_core)]
+            rtio_core::async_error_write(errors);
+            #[cfg(has_drtiosat)]
+            rtio_core::protocol_error_write(errors);
+        }
+    }
+}
+
+static mut RTIO_DEVICE_MAP: BTreeMap<u32, String> = BTreeMap::new();
+
+fn read_device_map(cfg: &Config) -> BTreeMap<u32, String> {
+    let mut device_map: BTreeMap<u32, String> = BTreeMap::new();
+    let _ = cfg
+        .read("device_map")
+        .and_then(|raw_bytes| {
+            let mut bytes_cr = Cursor::new(raw_bytes);
+            let size = bytes_cr.read_u32().unwrap();
+            for _ in 0..size {
+                let channel = bytes_cr.read_u32().unwrap();
+                let device_name = bytes_cr.read_string().unwrap();
+                if let Some(old_entry) = device_map.insert(channel, device_name.clone()) {
+                    warn!(
+                        "conflicting device map entries for RTIO channel {}: '{}' and '{}'",
+                        channel, old_entry, device_name
+                    );
+                }
+            }
+            Ok(())
+        })
+        .or_else(|err| {
+            warn!(
+                "error reading device map ({}), device names will not be available in RTIO error messages",
+                err
+            );
+            Err(err)
+        });
+    device_map
+}
+
+fn _resolve_channel_name(channel: u32, device_map: &BTreeMap<u32, String>) -> String {
+    match device_map.get(&channel) {
+        Some(val) => val.clone(),
+        None => String::from("unknown"),
+    }
+}
+
+pub fn resolve_channel_name(channel: u32) -> String {
+    _resolve_channel_name(channel, unsafe { &RTIO_DEVICE_MAP })
+}
+
+pub fn setup_device_map(cfg: &Config) {
+    unsafe {
+        RTIO_DEVICE_MAP = read_device_map(cfg);
+    }
+}

src/libksupport/src/rpc.rs

@@ -0,0 +1,591 @@
+use core::str;
+
+use byteorder::{ByteOrder, NativeEndian};
+use core_io::{Error, Read, Write};
+use cslice::{CMutSlice, CSlice};
+use io::{ProtoRead, ProtoWrite};
+use log::trace;
+
+use self::tag::{split_tag, Tag, TagIterator};
+
+#[inline]
+pub fn round_up(val: usize, power_of_two: usize) -> usize {
+    assert!(power_of_two.is_power_of_two());
+    let max_rem = power_of_two - 1;
+    (val + max_rem) & (!max_rem)
+}
+
+#[inline]
+pub unsafe fn round_up_mut<T>(ptr: *mut T, power_of_two: usize) -> *mut T {
+    round_up(ptr as usize, power_of_two) as *mut T
+}
+
+#[inline]
+pub unsafe fn round_up_const<T>(ptr: *const T, power_of_two: usize) -> *const T {
+    round_up(ptr as usize, power_of_two) as *const T
+}
+
+#[inline]
+pub unsafe fn align_ptr<T>(ptr: *const ()) -> *const T {
+    round_up_const(ptr, core::mem::align_of::<T>()) as *const T
+}
+
+#[inline]
+pub unsafe fn align_ptr_mut<T>(ptr: *mut ()) -> *mut T {
+    round_up_mut(ptr, core::mem::align_of::<T>()) as *mut T
+}
+
+// versions for reader rather than TcpStream
+// they will be made into sync for satellite subkernels later
+unsafe fn recv_elements<F, R>(
+    reader: &mut R,
+    elt_tag: Tag,
+    length: usize,
+    storage: *mut (),
+    alloc: &mut F,
+) -> Result<(), Error>
+where
+    F: FnMut(usize) -> *mut (),
+    R: Read + ?Sized,
+{
+    match elt_tag {
+        Tag::Bool => {
+            let dest = core::slice::from_raw_parts_mut(storage as *mut u8, length);
+            reader.read_exact(dest)?;
+        }
+        Tag::Int32 => {
+            let ptr = storage as *mut u32;
+            let dest = core::slice::from_raw_parts_mut(ptr as *mut u8, length * 4);
+            reader.read_exact(dest)?;
+            drop(dest);
+            let dest = core::slice::from_raw_parts_mut(ptr, length);
+            NativeEndian::from_slice_u32(dest);
+        }
+        Tag::Int64 | Tag::Float64 => {
+            let ptr = storage as *mut u64;
+            let dest = core::slice::from_raw_parts_mut(ptr as *mut u8, length * 8);
+            reader.read_exact(dest)?;
+            drop(dest);
+            let dest = core::slice::from_raw_parts_mut(ptr, length);
+            NativeEndian::from_slice_u64(dest);
+        }
+        _ => {
+            let mut data = storage;
+            for _ in 0..length {
+                recv_value(reader, elt_tag, &mut data, alloc)?
+            }
+        }
+    }
+    Ok(())
+}
+
+unsafe fn recv_value<F, R>(reader: &mut R, tag: Tag, data: &mut *mut (), alloc: &mut F) -> Result<(), Error>
+where
+    F: FnMut(usize) -> *mut (),
+    R: Read + ?Sized,
+{
+    macro_rules! consume_value {
+        ($ty:ty, | $ptr:ident | $map:expr) => {{
+            let $ptr = align_ptr_mut::<$ty>(*data);
+            *data = $ptr.offset(1) as *mut ();
+            $map
+        }};
+    }
+
+    match tag {
+        Tag::None => Ok(()),
+        Tag::Bool => consume_value!(i8, |ptr| {
+            *ptr = reader.read_u8()? as i8;
+            Ok(())
+        }),
+        Tag::Int32 => consume_value!(i32, |ptr| {
+            *ptr = reader.read_u32()? as i32;
+            Ok(())
+        }),
+        Tag::Int64 | Tag::Float64 => consume_value!(i64, |ptr| {
+            *ptr = reader.read_u64()? as i64;
+            Ok(())
+        }),
+        Tag::String | Tag::Bytes | Tag::ByteArray => {
+            consume_value!(CMutSlice<u8>, |ptr| {
+                let length = reader.read_u32()? as usize;
+                *ptr = CMutSlice::new(alloc(length) as *mut u8, length);
+                reader.read_exact((*ptr).as_mut())?;
+                Ok(())
+            })
+        }
+        Tag::Tuple(it, arity) => {
+            let alignment = tag.alignment();
+            *data = round_up_mut(*data, alignment);
+            let mut it = it.clone();
+            for _ in 0..arity {
+                let tag = it.next().expect("truncated tag");
+                recv_value(reader, tag, data, alloc)?
+            }
+            *data = round_up_mut(*data, alignment);
+            Ok(())
+        }
+        Tag::List(it) => {
+            #[repr(C)]
+            struct List {
+                elements: *mut (),
+                length: usize,
+            }
+            consume_value!(*mut List, |ptr_to_list| {
+                let tag = it.clone().next().expect("truncated tag");
+                let length = reader.read_u32()? as usize;
+
+                let list_size = 4 + 4;
+                let storage_offset = round_up(list_size, tag.alignment());
+                let storage_size = tag.size() * length;
+
+                let allocation = alloc(storage_offset + storage_size) as *mut u8;
+                *ptr_to_list = allocation as *mut List;
+                let storage = allocation.offset(storage_offset as isize) as *mut ();
+
+                (**ptr_to_list).length = length;
+                (**ptr_to_list).elements = storage;
+                recv_elements(reader, tag, length, storage, alloc)
+            })
+        }
+        Tag::Array(it, num_dims) => {
+            consume_value!(*mut (), |buffer| {
+                let mut total_len: usize = 1;
+                for _ in 0..num_dims {
+                    let len = reader.read_u32()? as usize;
+                    total_len *= len;
+                    consume_value!(usize, |ptr| *ptr = len)
+                }
+
+                let elt_tag = it.clone().next().expect("truncated tag");
+                *buffer = alloc(elt_tag.size() * total_len);
+                recv_elements(reader, elt_tag, total_len, *buffer, alloc)
+            })
+        }
+        Tag::Range(it) => {
+            *data = round_up_mut(*data, tag.alignment());
+            let tag = it.clone().next().expect("truncated tag");
+            recv_value(reader, tag, data, alloc)?;
+            recv_value(reader, tag, data, alloc)?;
+            recv_value(reader, tag, data, alloc)?;
+            Ok(())
+        }
+        Tag::Keyword(_) => unreachable!(),
+        Tag::Object => unreachable!(),
+    }
+}
+
+pub fn recv_return<'a, F, R>(
+    reader: &mut R,
+    tag_bytes: &'a [u8],
+    data: *mut (),
+    alloc: &mut F,
+) -> Result<&'a [u8], Error>
+where
+    F: FnMut(usize) -> *mut (),
+    R: Read + ?Sized,
+{
+    let mut it = TagIterator::new(tag_bytes);
+    trace!("recv ...->{}", it);
+
+    let tag = it.next().expect("truncated tag");
+    let mut data = data;
+    unsafe { recv_value(reader, tag, &mut data, alloc)? };
+
+    Ok(it.data)
+}
+
+unsafe fn send_elements<W>(
+    writer: &mut W,
+    elt_tag: Tag,
+    length: usize,
+    data: *const (),
+    write_tags: bool,
+) -> Result<(), Error>
+where
+    W: Write + ?Sized,
+{
+    if write_tags {
+        writer.write_u8(elt_tag.as_u8())?;
+    }
+    match elt_tag {
+        // we cannot use NativeEndian::from_slice_i32 as the data is not mutable,
+        // and that is not needed as the data is already in native endian
+        Tag::Bool => {
+            let slice = core::slice::from_raw_parts(data as *const u8, length);
+            writer.write_all(slice)?;
+        }
+        Tag::Int32 => {
+            let slice = core::slice::from_raw_parts(data as *const u8, length * 4);
+            writer.write_all(slice)?;
+        }
+        Tag::Int64 | Tag::Float64 => {
+            let slice = core::slice::from_raw_parts(data as *const u8, length * 8);
+            writer.write_all(slice)?;
+        }
+        _ => {
+            let mut data = data;
+            for _ in 0..length {
+                send_value(writer, elt_tag, &mut data, write_tags)?;
+            }
+        }
+    }
+    Ok(())
+}
+
+unsafe fn send_value<W>(writer: &mut W, tag: Tag, data: &mut *const (), write_tags: bool) -> Result<(), Error>
+where W: Write + ?Sized {
+    macro_rules! consume_value {
+        ($ty:ty, | $ptr:ident | $map:expr) => {{
+            let $ptr = align_ptr::<$ty>(*data);
+            *data = $ptr.offset(1) as *const ();
+            $map
+        }};
+    }
+
+    if write_tags {
+        writer.write_u8(tag.as_u8())?;
+    }
+    match tag {
+        Tag::None => Ok(()),
+        Tag::Bool => consume_value!(u8, |ptr| writer.write_u8(*ptr)),
+        Tag::Int32 => consume_value!(u32, |ptr| writer.write_u32(*ptr)),
+        Tag::Int64 | Tag::Float64 => consume_value!(u64, |ptr| writer.write_u64(*ptr)),
+        Tag::String => consume_value!(CSlice<u8>, |ptr| {
+            writer.write_string(str::from_utf8((*ptr).as_ref()).unwrap())
+        }),
+        Tag::Bytes | Tag::ByteArray => consume_value!(CSlice<u8>, |ptr| writer.write_bytes((*ptr).as_ref())),
+        Tag::Tuple(it, arity) => {
+            let mut it = it.clone();
+            if write_tags {
+                writer.write_u8(arity)?;
+            }
+            let mut max_alignment = 0;
+            for _ in 0..arity {
+                let tag = it.next().expect("truncated tag");
+                max_alignment = core::cmp::max(max_alignment, tag.alignment());
+                send_value(writer, tag, data, write_tags)?
+            }
+            *data = round_up_const(*data, max_alignment);
+            Ok(())
+        }
+        Tag::List(it) => {
+            #[repr(C)]
+            struct List {
+                elements: *const (),
+                length: u32,
+            }
+            consume_value!(&List, |ptr| {
+                let length = (**ptr).length as usize;
+                writer.write_u32((*ptr).length)?;
+                let tag = it.clone().next().expect("truncated tag");
+                send_elements(writer, tag, length, (**ptr).elements, write_tags)
+            })
+        }
+        Tag::Array(it, num_dims) => {
+            if write_tags {
+                writer.write_u8(num_dims)?;
+            }
+            consume_value!(*const (), |buffer| {
+                let elt_tag = it.clone().next().expect("truncated tag");
+
+                let mut total_len = 1;
+                for _ in 0..num_dims {
+                    consume_value!(u32, |len| {
+                        writer.write_u32(*len)?;
+                        total_len *= *len;
+                    })
+                }
+                let length = total_len as usize;
+                send_elements(writer, elt_tag, length, *buffer, write_tags)
+            })
+        }
+        Tag::Range(it) => {
+            let tag = it.clone().next().expect("truncated tag");
+            send_value(writer, tag, data, write_tags)?;
+            send_value(writer, tag, data, write_tags)?;
+            send_value(writer, tag, data, write_tags)?;
+            Ok(())
+        }
+        Tag::Keyword(it) => {
+            #[repr(C)]
+            struct Keyword<'a> {
+                name: CSlice<'a, u8>,
+            }
+            consume_value!(Keyword, |ptr| {
+                writer.write_string(str::from_utf8((*ptr).name.as_ref()).unwrap())?;
+                let tag = it.clone().next().expect("truncated tag");
+                let mut data = ptr.offset(1) as *const ();
+                send_value(writer, tag, &mut data, write_tags)
+            })
+            // Tag::Keyword never appears in composite types, so we don't have
+            // to accurately advance data.
+        }
+        Tag::Object => {
+            #[repr(C)]
+            struct Object {
+                id: u32,
+            }
+            consume_value!(*const Object, |ptr| writer.write_u32((**ptr).id))
+        }
+    }
+}
+
+pub fn send_args<W>(
+    writer: &mut W,
+    service: u32,
+    tag_bytes: &[u8],
+    data: *const *const (),
+    write_tags: bool,
+) -> Result<(), Error>
+where
+    W: Write + ?Sized,
+{
+    let (arg_tags_bytes, return_tag_bytes) = split_tag(tag_bytes);
+
+    let mut args_it = TagIterator::new(arg_tags_bytes);
+    let return_it = TagIterator::new(return_tag_bytes);
+    trace!("send<{}>({})->{}", service, args_it, return_it);
+
+    writer.write_u32(service)?;
+    for index in 0.. {
+        if let Some(arg_tag) = args_it.next() {
+            let mut data = unsafe { *data.offset(index) };
+            unsafe { send_value(writer, arg_tag, &mut data, write_tags)? };
+        } else {
+            break;
+        }
+    }
+    writer.write_u8(0)?;
+    writer.write_bytes(return_tag_bytes)?;
+
+    Ok(())
+}
+
+pub mod tag {
+    use core::fmt;
+
+    pub fn split_tag(tag_bytes: &[u8]) -> (&[u8], &[u8]) {
+        let tag_separator = tag_bytes
+            .iter()
+            .position(|&b| b == b':')
+            .expect("tag without a return separator");
+        let (arg_tags_bytes, rest) = tag_bytes.split_at(tag_separator);
+        let return_tag_bytes = &rest[1..];
+        (arg_tags_bytes, return_tag_bytes)
+    }
+
+    #[derive(Debug, Clone, Copy)]
+    pub enum Tag<'a> {
+        None,
+        Bool,
+        Int32,
+        Int64,
+        Float64,
+        String,
+        Bytes,
+        ByteArray,
+        Tuple(TagIterator<'a>, u8),
+        List(TagIterator<'a>),
+        Array(TagIterator<'a>, u8),
+        Range(TagIterator<'a>),
+        Keyword(TagIterator<'a>),
+        Object,
+    }
+
+    impl<'a> Tag<'a> {
+        pub fn as_u8(self) -> u8 {
+            match self {
+                Tag::None => b'n',
+                Tag::Bool => b'b',
+                Tag::Int32 => b'i',
+                Tag::Int64 => b'I',
+                Tag::Float64 => b'f',
+                Tag::String => b's',
+                Tag::Bytes => b'B',
+                Tag::ByteArray => b'A',
+                Tag::Tuple(_, _) => b't',
+                Tag::List(_) => b'l',
+                Tag::Array(_, _) => b'a',
+                Tag::Range(_) => b'r',
+                Tag::Keyword(_) => b'k',
+                Tag::Object => b'O',
+            }
+        }
+
+        pub fn alignment(self) -> usize {
+            use cslice::CSlice;
+            match self {
+                Tag::None => 1,
+                Tag::Bool => core::mem::align_of::<u8>(),
+                Tag::Int32 => core::mem::align_of::<i32>(),
+                Tag::Int64 => core::mem::align_of::<i64>(),
+                Tag::Float64 => core::mem::align_of::<f64>(),
+                // struct type: align to largest element
+                Tag::Tuple(it, arity) => {
+                    let it = it.clone();
+                    it.take(arity.into()).map(|t| t.alignment()).max().unwrap()
+                }
+                Tag::Range(it) => {
+                    let it = it.clone();
+                    it.take(3).map(|t| t.alignment()).max().unwrap()
+                }
+                // the ptr/length(s) pair is basically CSlice
+                Tag::Bytes | Tag::String | Tag::ByteArray | Tag::List(_) | Tag::Array(_, _) => {
+                    core::mem::align_of::<CSlice<()>>()
+                }
+                Tag::Keyword(_) => unreachable!("Tag::Keyword should not appear in composite types"),
+                Tag::Object => core::mem::align_of::<u32>(),
+            }
+        }
+
+        pub fn size(self) -> usize {
+            match self {
+                Tag::None => 0,
+                Tag::Bool => 1,
+                Tag::Int32 => 4,
+                Tag::Int64 => 8,
+                Tag::Float64 => 8,
+                Tag::String => 8,
+                Tag::Bytes => 8,
+                Tag::ByteArray => 8,
+                Tag::Tuple(it, arity) => {
+                    let mut size = 0;
+                    let mut max_alignment = 0;
+                    let mut it = it.clone();
+                    for _ in 0..arity {
+                        let tag = it.next().expect("truncated tag");
+                        let alignment = tag.alignment();
+                        max_alignment = core::cmp::max(max_alignment, alignment);
+                        size = super::round_up(size, alignment);
+                        size += tag.size();
+                    }
+                    // Take into account any tail padding (if element(s) with largest
+                    // alignment are not at the end).
+                    size = super::round_up(size, max_alignment);
+                    size
+                }
+                Tag::List(_) => 4,
+                Tag::Array(_, num_dims) => 4 * (1 + num_dims as usize),
+                Tag::Range(it) => {
+                    let tag = it.clone().next().expect("truncated tag");
+                    tag.size() * 3
+                }
+                Tag::Keyword(_) => unreachable!(),
+                Tag::Object => unreachable!(),
+            }
+        }
+    }
+
+    #[derive(Debug, Clone, Copy)]
+    pub struct TagIterator<'a> {
+        pub data: &'a [u8],
+    }
+
+    impl<'a> TagIterator<'a> {
+        pub fn new(data: &'a [u8]) -> TagIterator<'a> {
+            TagIterator { data }
+        }
+
+        fn sub(&mut self, count: u8) -> TagIterator<'a> {
+            let data = self.data;
+            for _ in 0..count {
+                self.next().expect("truncated tag");
+            }
+            TagIterator {
+                data: &data[..(data.len() - self.data.len())],
+            }
+        }
+    }
+
+    impl<'a> core::iter::Iterator for TagIterator<'a> {
+        type Item = Tag<'a>;
+
+        fn next(&mut self) -> Option<Tag<'a>> {
+            if self.data.len() == 0 {
+                return None;
+            }
+
+            let tag_byte = self.data[0];
+            self.data = &self.data[1..];
+            Some(match tag_byte {
+                b'n' => Tag::None,
+                b'b' => Tag::Bool,
+                b'i' => Tag::Int32,
+                b'I' => Tag::Int64,
+                b'f' => Tag::Float64,
+                b's' => Tag::String,
+                b'B' => Tag::Bytes,
+                b'A' => Tag::ByteArray,
+                b't' => {
+                    let count = self.data[0];
+                    self.data = &self.data[1..];
+                    Tag::Tuple(self.sub(count), count)
+                }
+                b'l' => Tag::List(self.sub(1)),
+                b'a' => {
+                    let count = self.data[0];
+                    self.data = &self.data[1..];
+                    Tag::Array(self.sub(1), count)
+                }
+                b'r' => Tag::Range(self.sub(1)),
+                b'k' => Tag::Keyword(self.sub(1)),
+                b'O' => Tag::Object,
+                _ => unreachable!(),
+            })
+        }
+    }
+
+    impl<'a> fmt::Display for TagIterator<'a> {
+        fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+            let mut it = self.clone();
+            let mut first = true;
+            while let Some(tag) = it.next() {
+                if first {
+                    first = false
+                } else {
+                    write!(f, ", ")?
+                }
+
+                match tag {
+                    Tag::None => write!(f, "None")?,
+                    Tag::Bool => write!(f, "Bool")?,
+                    Tag::Int32 => write!(f, "Int32")?,
+                    Tag::Int64 => write!(f, "Int64")?,
+                    Tag::Float64 => write!(f, "Float64")?,
+                    Tag::String => write!(f, "String")?,
+                    Tag::Bytes => write!(f, "Bytes")?,
+                    Tag::ByteArray => write!(f, "ByteArray")?,
+                    Tag::Tuple(it, _) => {
+                        write!(f, "Tuple(")?;
+                        it.fmt(f)?;
+                        write!(f, ")")?;
+                    }
+                    Tag::List(it) => {
+                        write!(f, "List(")?;
+                        it.fmt(f)?;
+                        write!(f, ")")?;
+                    }
+                    Tag::Array(it, num_dims) => {
+                        write!(f, "Array(")?;
+                        it.fmt(f)?;
+                        write!(f, ", {})", num_dims)?;
+                    }
+                    Tag::Range(it) => {
+                        write!(f, "Range(")?;
+                        it.fmt(f)?;
+                        write!(f, ")")?;
+                    }
+                    Tag::Keyword(it) => {
+                        write!(f, "Keyword(")?;
+                        it.fmt(f)?;
+                        write!(f, ")")?;
+                    }
+                    Tag::Object => write!(f, "Object")?,
+                }
+            }
+
+            Ok(())
+        }
+    }
+}

src/libksupport/src/rtio_acp.rs

@@ -1,18 +1,19 @@
-use cslice::CSlice;
-use vcell::VolatileCell;
-use libcortex_a9::asm;
-use crate::artiq_raise;
 use core::sync::atomic::{fence, Ordering};
 
-use crate::pl::csr;
+use cslice::CSlice;
+use libcortex_a9::asm;
+use vcell::VolatileCell;
 
-pub const RTIO_O_STATUS_WAIT:                      i32 = 1;
-pub const RTIO_O_STATUS_UNDERFLOW:                 i32 = 2;
-pub const RTIO_O_STATUS_DESTINATION_UNREACHABLE:   i32 = 4;
-pub const RTIO_I_STATUS_WAIT_EVENT:                i32 = 1;
-pub const RTIO_I_STATUS_OVERFLOW:                  i32 = 2;
-pub const RTIO_I_STATUS_WAIT_STATUS:               i32 = 4; // TODO
-pub const RTIO_I_STATUS_DESTINATION_UNREACHABLE:   i32 = 8;
+use crate::{artiq_raise, pl::csr, resolve_channel_name, rtio_core};
+
+pub const RTIO_O_STATUS_WAIT: i32 = 1;
+pub const RTIO_O_STATUS_UNDERFLOW: i32 = 2;
+pub const RTIO_O_STATUS_DESTINATION_UNREACHABLE: i32 = 4;
+pub const RTIO_I_STATUS_WAIT_EVENT: i32 = 1;
+pub const RTIO_I_STATUS_OVERFLOW: i32 = 2;
+#[allow(unused)]
+pub const RTIO_I_STATUS_WAIT_STATUS: i32 = 4; // TODO
+pub const RTIO_I_STATUS_DESTINATION_UNREACHABLE: i32 = 8;
 
 #[repr(C)]
 pub struct TimestampedData {
@@ -46,18 +47,18 @@ static mut TRANSACTION_BUFFER: Transaction = Transaction {
     reply_timestamp: VolatileCell::new(0),
     padding0: [0; 2],
     padding1: [0; 2],
-    padding2: [0; 2]
+    padding2: [0; 2],
 };
 
-pub extern fn init() {
+pub extern "C" fn init() {
     unsafe {
-        csr::rtio_core::reset_write(1);
+        rtio_core::reset_write(1);
         csr::rtio::engine_addr_base_write(&TRANSACTION_BUFFER as *const Transaction as u32);
         csr::rtio::enable_write(1);
     }
 }
 
-pub extern fn get_counter() -> i64 {
+pub extern "C" fn get_counter() -> i64 {
     unsafe {
         csr::rtio::counter_update_write(1);
         csr::rtio::counter_read() as i64
@@ -66,15 +67,15 @@ pub extern fn get_counter() -> i64 {
 
 static mut NOW: i64 = 0;
 
-pub extern fn now_mu() -> i64 {
+pub extern "C" fn now_mu() -> i64 {
     unsafe { NOW }
 }
 
-pub extern fn at_mu(t: i64) {
+pub extern "C" fn at_mu(t: i64) {
     unsafe { NOW = t }
 }
 
-pub extern fn delay_mu(dt: i64) {
+pub extern "C" fn delay_mu(dt: i64) {
     unsafe { NOW += dt }
 }
 
@@ -86,18 +87,34 @@ unsafe fn process_exceptional_status(channel: i32, status: i32) {
         while csr::rtio::o_status_read() as i32 & RTIO_O_STATUS_WAIT != 0 {}
     }
     if status & RTIO_O_STATUS_UNDERFLOW != 0 {
-        artiq_raise!("RTIOUnderflow",
-            "RTIO underflow at {0} mu, channel {1}, slack {2} mu",
-            timestamp, channel as i64, timestamp - get_counter());
+        artiq_raise!(
+            "RTIOUnderflow",
+            format!(
+                "RTIO underflow at {{1}} mu, channel 0x{:04x}:{}, slack {{2}} mu",
+                channel,
+                resolve_channel_name(channel as u32)
+            ),
+            channel as i64,
+            timestamp,
+            timestamp - get_counter()
+        );
     }
     if status & RTIO_O_STATUS_DESTINATION_UNREACHABLE != 0 {
-        artiq_raise!("RTIODestinationUnreachable",
-            "RTIO destination unreachable, output, at {0} mu, channel {1}",
-            timestamp, channel as i64, 0);
+        artiq_raise!(
+            "RTIODestinationUnreachable",
+            format!(
+                "RTIO destination unreachable, output, at {{0}} mu, channel 0x{:04x}:{}",
+                channel,
+                resolve_channel_name(channel as u32)
+            ),
+            timestamp,
+            channel as i64,
+            0
+        );
     }
 }
 
-pub extern fn output(target: i32, data: i32) {
+pub extern "C" fn output(target: i32, data: i32) {
     unsafe {
         // Clear status so we can observe response
         TRANSACTION_BUFFER.reply_status.set(0);
@@ -125,7 +142,7 @@ pub extern fn output(target: i32, data: i32) {
     }
 }
 
-pub extern fn output_wide(target: i32, data: CSlice<i32>) {
+pub extern "C" fn output_wide(target: i32, data: CSlice<i32>) {
     unsafe {
         // Clear status so we can observe response
         TRANSACTION_BUFFER.reply_status.set(0);
@@ -142,7 +159,7 @@ pub extern fn output_wide(target: i32, data: CSlice<i32>) {
         loop {
             status = TRANSACTION_BUFFER.reply_status.get();
             if status != 0 {
-                break
+                break;
             }
         }
 
@@ -153,8 +170,8 @@ pub extern fn output_wide(target: i32, data: CSlice<i32>) {
     }
 }
 
-pub extern fn input_timestamp(timeout: i64, channel: i32) -> i64 {
-   unsafe {
+pub extern "C" fn input_timestamp(timeout: i64, channel: i32) -> i64 {
+    unsafe {
         // Clear status so we can observe response
         TRANSACTION_BUFFER.reply_status.set(0);
 
@@ -170,29 +187,45 @@ pub extern fn input_timestamp(timeout: i64, channel: i32) -> i64 {
         loop {
             status = TRANSACTION_BUFFER.reply_status.get();
             if status != 0 {
-                break
+                break;
             }
         }
 
         if status & RTIO_I_STATUS_OVERFLOW != 0 {
-            artiq_raise!("RTIOOverflow",
-                         "RTIO input overflow on channel {0}",
-                         channel as i64, 0, 0);
+            artiq_raise!(
+                "RTIOOverflow",
+                format!(
+                    "RTIO input overflow on channel 0x{:04x}:{}",
+                    channel,
+                    resolve_channel_name(channel as u32)
+                ),
+                channel as i64,
+                0,
+                0
+            );
         }
         if status & RTIO_I_STATUS_WAIT_EVENT != 0 {
-            return -1
+            return -1;
         }
         if status & RTIO_I_STATUS_DESTINATION_UNREACHABLE != 0 {
-            artiq_raise!("RTIODestinationUnreachable",
-                         "RTIO destination unreachable, input, on channel {0}",
-                         channel as i64, 0, 0);
+            artiq_raise!(
+                "RTIODestinationUnreachable",
+                format!(
+                    "RTIO destination unreachable, input, on channel 0x{:04x}:{}",
+                    channel,
+                    resolve_channel_name(channel as u32)
+                ),
+                channel as i64,
+                0,
+                0
+            );
         }
 
         TRANSACTION_BUFFER.reply_timestamp.get()
     }
 }
 
-pub extern fn input_data(channel: i32) -> i32 {
+pub extern "C" fn input_data(channel: i32) -> i32 {
     unsafe {
         TRANSACTION_BUFFER.reply_status.set(0);
 
@@ -208,26 +241,42 @@ pub extern fn input_data(channel: i32) -> i32 {
         loop {
             status = TRANSACTION_BUFFER.reply_status.get();
             if status != 0 {
-                break
+                break;
             }
         }
 
         if status & RTIO_I_STATUS_OVERFLOW != 0 {
-            artiq_raise!("RTIOOverflow",
-                         "RTIO input overflow on channel {0}",
-                         channel as i64, 0, 0);
+            artiq_raise!(
+                "RTIOOverflow",
+                format!(
+                    "RTIO input overflow on channel 0x{:04x}:{}",
+                    channel,
+                    resolve_channel_name(channel as u32)
+                ),
+                channel as i64,
+                0,
+                0
+            );
         }
         if status & RTIO_I_STATUS_DESTINATION_UNREACHABLE != 0 {
-            artiq_raise!("RTIODestinationUnreachable",
-                         "RTIO destination unreachable, input, on channel {0}",
-                         channel as i64, 0, 0);
+            artiq_raise!(
+                "RTIODestinationUnreachable",
+                format!(
+                    "RTIO destination unreachable, input, on channel 0x{:04x}:{}",
+                    channel,
+                    resolve_channel_name(channel as u32)
+                ),
+                channel as i64,
+                0,
+                0
+            );
         }
 
         TRANSACTION_BUFFER.reply_data.get()
     }
 }
 
-pub extern fn input_timestamped_data(timeout: i64, channel: i32) -> TimestampedData {
+pub extern "C" fn input_timestamped_data(timeout: i64, channel: i32) -> TimestampedData {
     unsafe {
         TRANSACTION_BUFFER.reply_status.set(0);
 
@@ -243,19 +292,35 @@ pub extern fn input_timestamped_data(timeout: i64, channel: i32) -> TimestampedD
         loop {
             status = TRANSACTION_BUFFER.reply_status.get();
             if status != 0 {
-                break
+                break;
             }
         }
 
         if status & RTIO_I_STATUS_OVERFLOW != 0 {
-            artiq_raise!("RTIOOverflow",
-                         "RTIO input overflow on channel {0}",
-                         channel as i64, 0, 0);
+            artiq_raise!(
+                "RTIOOverflow",
+                format!(
+                    "RTIO input overflow on channel 0x{:04x}:{}",
+                    channel,
+                    resolve_channel_name(channel as u32)
+                ),
+                channel as i64,
+                0,
+                0
+            );
         }
         if status & RTIO_I_STATUS_DESTINATION_UNREACHABLE != 0 {
-            artiq_raise!("RTIODestinationUnreachable",
-                         "RTIO destination unreachable, input, on channel {0}",
-                         channel as i64, 0, 0);
+            artiq_raise!(
+                "RTIODestinationUnreachable",
+                format!(
+                    "RTIO destination unreachable, input, on channel 0x{:04x}:{}",
+                    channel,
+                    resolve_channel_name(channel as u32)
+                ),
+                channel as i64,
+                0,
+                0
+            );
         }
 
         TimestampedData {

src/libksupport/src/rtio_csr.rs

@@ -0,0 +1,274 @@
+use core::ptr::{read_volatile, write_volatile};
+
+use cslice::CSlice;
+
+use crate::{artiq_raise, pl::csr, resolve_channel_name, rtio_core};
+
+pub const RTIO_O_STATUS_WAIT: u8 = 1;
+pub const RTIO_O_STATUS_UNDERFLOW: u8 = 2;
+pub const RTIO_O_STATUS_DESTINATION_UNREACHABLE: u8 = 4;
+pub const RTIO_I_STATUS_WAIT_EVENT: u8 = 1;
+pub const RTIO_I_STATUS_OVERFLOW: u8 = 2;
+pub const RTIO_I_STATUS_WAIT_STATUS: u8 = 4;
+pub const RTIO_I_STATUS_DESTINATION_UNREACHABLE: u8 = 8;
+
+#[repr(C)]
+pub struct TimestampedData {
+    timestamp: i64,
+    data: i32,
+}
+
+pub extern "C" fn init() {
+    unsafe {
+        rtio_core::reset_write(1);
+    }
+}
+
+pub extern "C" fn get_counter() -> i64 {
+    unsafe {
+        csr::rtio::counter_update_write(1);
+        csr::rtio::counter_read() as i64
+    }
+}
+
+pub extern "C" fn now_mu() -> i64 {
+    unsafe { csr::rtio::now_read() as i64 }
+}
+
+pub extern "C" fn at_mu(t: i64) {
+    unsafe {
+        csr::rtio::now_write(t as u64);
+    }
+}
+
+pub extern "C" fn delay_mu(dt: i64) {
+    unsafe {
+        csr::rtio::now_write(csr::rtio::now_read() + dt as u64);
+    }
+}
+
+// writing the LSB of o_data (offset=0) triggers the RTIO write
+#[inline(always)]
+pub unsafe fn rtio_o_data_write(offset: usize, data: u32) {
+    write_volatile(
+        csr::rtio::O_DATA_ADDR.offset((csr::rtio::O_DATA_SIZE - 1 - offset) as isize),
+        data,
+    );
+}
+
+#[inline(always)]
+pub unsafe fn rtio_i_data_read(offset: usize) -> u32 {
+    read_volatile(csr::rtio::I_DATA_ADDR.offset((csr::rtio::I_DATA_SIZE - 1 - offset) as isize))
+}
+
+#[inline(never)]
+unsafe fn process_exceptional_status(channel: i32, status: u8) {
+    let timestamp = csr::rtio::now_read() as i64;
+    if status & RTIO_O_STATUS_WAIT != 0 {
+        while csr::rtio::o_status_read() & RTIO_O_STATUS_WAIT != 0 {}
+    }
+    if status & RTIO_O_STATUS_UNDERFLOW != 0 {
+        artiq_raise!(
+            "RTIOUnderflow",
+            format!(
+                "RTIO underflow at {{1}} mu, channel 0x{:04x}:{}, slack {{2}} mu",
+                channel,
+                resolve_channel_name(channel as u32)
+            ),
+            channel as i64,
+            timestamp,
+            timestamp - get_counter()
+        );
+    }
+    if status & RTIO_O_STATUS_DESTINATION_UNREACHABLE != 0 {
+        artiq_raise!(
+            "RTIODestinationUnreachable",
+            format!(
+                "RTIO destination unreachable, output, at {{0}} mu, channel 0x{:04x}:{}",
+                channel,
+                resolve_channel_name(channel as u32)
+            ),
+            timestamp,
+            channel as i64,
+            0
+        );
+    }
+}
+
+pub extern "C" fn output(target: i32, data: i32) {
+    unsafe {
+        csr::rtio::target_write(target as u32);
+        // writing target clears o_data
+        rtio_o_data_write(0, data as _);
+        let status = csr::rtio::o_status_read();
+        if status != 0 {
+            process_exceptional_status(target >> 8, status);
+        }
+    }
+}
+
+pub extern "C" fn output_wide(target: i32, data: &CSlice<i32>) {
+    unsafe {
+        csr::rtio::target_write(target as u32);
+        // writing target clears o_data
+        for i in (0..data.len()).rev() {
+            rtio_o_data_write(i, data[i] as _)
+        }
+        let status = csr::rtio::o_status_read();
+        if status != 0 {
+            process_exceptional_status(target >> 8, status);
+        }
+    }
+}
+
+pub extern "C" fn input_timestamp(timeout: i64, channel: i32) -> i64 {
+    unsafe {
+        csr::rtio::target_write((channel as u32) << 8);
+        csr::rtio::i_timeout_write(timeout as u64);
+
+        let mut status = RTIO_I_STATUS_WAIT_STATUS;
+        while status & RTIO_I_STATUS_WAIT_STATUS != 0 {
+            status = csr::rtio::i_status_read();
+        }
+
+        if status & RTIO_I_STATUS_OVERFLOW != 0 {
+            artiq_raise!(
+                "RTIOOverflow",
+                format!(
+                    "RTIO input overflow on channel 0x{:04x}:{}",
+                    channel,
+                    resolve_channel_name(channel as u32)
+                ),
+                channel as i64,
+                0,
+                0
+            );
+        }
+        if status & RTIO_I_STATUS_WAIT_EVENT != 0 {
+            return -1;
+        }
+        if status & RTIO_I_STATUS_DESTINATION_UNREACHABLE != 0 {
+            artiq_raise!(
+                "RTIODestinationUnreachable",
+                format!(
+                    "RTIO destination unreachable, input, on channel 0x{:04x}:{}",
+                    channel,
+                    resolve_channel_name(channel as u32)
+                ),
+                channel as i64,
+                0,
+                0
+            );
+        }
+
+        csr::rtio::i_timestamp_read() as i64
+    }
+}
+
+pub extern "C" fn input_data(channel: i32) -> i32 {
+    unsafe {
+        csr::rtio::target_write((channel as u32) << 8);
+        csr::rtio::i_timeout_write(0xffffffff_ffffffff);
+
+        let mut status = RTIO_I_STATUS_WAIT_STATUS;
+        while status & RTIO_I_STATUS_WAIT_STATUS != 0 {
+            status = csr::rtio::i_status_read();
+        }
+
+        if status & RTIO_I_STATUS_OVERFLOW != 0 {
+            artiq_raise!(
+                "RTIOOverflow",
+                format!(
+                    "RTIO input overflow on channel 0x{:04x}:{}",
+                    channel,
+                    resolve_channel_name(channel as u32)
+                ),
+                channel as i64,
+                0,
+                0
+            );
+        }
+        if status & RTIO_I_STATUS_DESTINATION_UNREACHABLE != 0 {
+            artiq_raise!(
+                "RTIODestinationUnreachable",
+                format!(
+                    "RTIO destination unreachable, input, on channel 0x{:04x}:{}",
+                    channel,
+                    resolve_channel_name(channel as u32)
+                ),
+                channel as i64,
+                0,
+                0
+            );
+        }
+
+        rtio_i_data_read(0) as i32
+    }
+}
+
+pub extern "C" fn input_timestamped_data(timeout: i64, channel: i32) -> TimestampedData {
+    unsafe {
+        csr::rtio::target_write((channel as u32) << 8);
+        csr::rtio::i_timeout_write(timeout as u64);
+
+        let mut status = RTIO_I_STATUS_WAIT_STATUS;
+        while status & RTIO_I_STATUS_WAIT_STATUS != 0 {
+            status = csr::rtio::i_status_read();
+        }
+
+        if status & RTIO_I_STATUS_OVERFLOW != 0 {
+            artiq_raise!(
+                "RTIOOverflow",
+                format!(
+                    "RTIO input overflow on channel 0x{:04x}:{}",
+                    channel,
+                    resolve_channel_name(channel as u32)
+                ),
+                channel as i64,
+                0,
+                0
+            );
+        }
+        if status & RTIO_I_STATUS_WAIT_EVENT != 0 {
+            return TimestampedData { timestamp: -1, data: 0 };
+        }
+        if status & RTIO_I_STATUS_DESTINATION_UNREACHABLE != 0 {
+            artiq_raise!(
+                "RTIODestinationUnreachable",
+                format!(
+                    "RTIO destination unreachable, input, on channel 0x{:04x}:{}",
+                    channel,
+                    resolve_channel_name(channel as u32)
+                ),
+                channel as i64,
+                0,
+                0
+            );
+        }
+
+        TimestampedData {
+            timestamp: csr::rtio::i_timestamp_read() as i64,
+            data: rtio_i_data_read(0) as i32,
+        }
+    }
+}
+
+pub fn write_log(data: &[i8]) {
+    unsafe {
+        csr::rtio::target_write(csr::CONFIG_RTIO_LOG_CHANNEL << 8);
+
+        let mut word: u32 = 0;
+        for i in 0..data.len() {
+            word <<= 8;
+            word |= data[i] as u32;
+            if i % 4 == 3 {
+                rtio_o_data_write(0, word);
+                word = 0;
+            }
+        }
+
+        if word != 0 {
+            rtio_o_data_write(0, word);
+        }
+    }
+}

src/libunwind/backtrace.rs

@@ -1,27 +1,25 @@
-use libc::{c_void, c_int};
+use libc::{c_int, c_void};
+
 use crate::libunwind as uw;
 
 const UW_REG_SP: c_int = 13;
 
 pub fn backtrace<F>(f: F) -> Result<(), uw::_Unwind_Reason_Code>
-    where F: FnMut(usize) -> ()
-{
+where F: FnMut(usize) -> () {
     struct TraceContext<F> {
         step_fn: F,
-        prev_sp: uw::_Unwind_Word
+        prev_sp: uw::_Unwind_Word,
     }
 
-    extern fn trace<F>(context: *mut uw::_Unwind_Context, arg: *mut c_void)
-                      -> uw::_Unwind_Reason_Code
-        where F: FnMut(usize) -> ()
-    {
+    extern "C" fn trace<F>(context: *mut uw::_Unwind_Context, arg: *mut c_void) -> uw::_Unwind_Reason_Code
+    where F: FnMut(usize) -> () {
         unsafe {
             let trace_context = &mut *(arg as *mut TraceContext<F>);
 
             // Detect the root of a libfringe thread
             let cur_sp = uw::_Unwind_GetGR(context, UW_REG_SP);
             if cur_sp == trace_context.prev_sp {
-                return uw::_URC_END_OF_STACK
+                return uw::_URC_END_OF_STACK;
             } else {
                 trace_context.prev_sp = cur_sp;
             }
@@ -35,7 +33,7 @@ pub fn backtrace<F>(f: F) -> Result<(), uw::_Unwind_Reason_Code>
         let mut trace_context = TraceContext { step_fn: f, prev_sp: 0 };
         match uw::_Unwind_Backtrace(trace::<F>, &mut trace_context as *mut _ as *mut c_void) {
             uw::_URC_NO_REASON => Ok(()),
-            err => Err(err)
+            err => Err(err),
         }
     }
 }

src/libunwind/build.rs

@@ -5,8 +5,7 @@ fn main() {
 }
 
 mod llvm_libunwind {
-    use std::path::Path;
-    use std::env;
+    use std::{env, path::Path};
 
     fn setup_options(cfg: &mut cc::Build) {
         cfg.no_default_flags(true);
@@ -82,11 +81,7 @@ mod llvm_libunwind {
         cfg.flag("-fvisibility=hidden");
         cfg.flag_if_supported("-fvisibility-global-new-delete-hidden");
 
-        let unwind_sources = vec![
-            "Unwind-EHABI.cpp",
-            "Unwind-seh.cpp",
-            "libunwind.cpp"
-        ];
+        let unwind_sources = vec!["Unwind-EHABI.cpp", "Unwind-seh.cpp", "libunwind.cpp"];
 
         let root = Path::new("../llvm_libunwind");
         cfg.include(root.join("include"));

src/libunwind/libunwind.rs

@@ -21,8 +21,7 @@ pub use _Unwind_Reason_Code::*;
 pub type _Unwind_Exception_Class = u64;
 pub type _Unwind_Word = uintptr_t;
 pub type _Unwind_Ptr = uintptr_t;
-pub type _Unwind_Trace_Fn =
-    extern "C" fn(ctx: *mut _Unwind_Context, arg: *mut c_void) -> _Unwind_Reason_Code;
+pub type _Unwind_Trace_Fn = extern "C" fn(ctx: *mut _Unwind_Context, arg: *mut c_void) -> _Unwind_Reason_Code;
 
 #[cfg(target_arch = "x86")]
 pub const unwinder_private_data_size: usize = 5;
@@ -279,7 +278,6 @@ if #[cfg(all(windows, target_arch = "x86_64", target_env = "gnu"))] {
 } // cfg_if!
 
 #[no_mangle]
-extern fn abort() {
+extern "C" fn abort() {
     panic!("Abort!");
 }
-

src/runtime/Cargo.toml.tpl

@@ -18,7 +18,6 @@ num-traits = { version = "0.2", default-features = false }
 num-derive = "0.3"
 cslice = "0.3"
 log = "0.4"
-nb = "0.1"
 embedded-hal = "0.2"
 core_io = { version = "0.1", features = ["collections"] }
 byteorder = { version = "1.3", default-features = false }
@@ -26,19 +25,23 @@ void = { version = "1", default-features = false }
 futures = { version = "0.3", default-features = false, features = ["async-await"] }
 async-recursion = "0.3"
 log_buffer = { version = "1.2" }
-libm = { version = "0.2", features = ["unstable"] }
 vcell = "0.1"
 
-libboard_zynq = { git = "https://git.m-labs.hk/M-Labs/zynq-rs.git", features = ["ipv6"]}
-libsupport_zynq = { default-features = false, features = ["alloc_core"], git = "https://git.m-labs.hk/M-Labs/zynq-rs.git" }
-libcortex_a9 = { git = "https://git.m-labs.hk/M-Labs/zynq-rs.git" }
-libasync = { git = "https://git.m-labs.hk/M-Labs/zynq-rs.git" }
-libregister = { git = "https://git.m-labs.hk/M-Labs/zynq-rs.git" }
-libconfig = { git = "https://git.m-labs.hk/M-Labs/zynq-rs.git", features = ["fat_lfn", "ipv6"] }
+libboard_zynq = { path = "@@ZYNQ_RS@@/libboard_zynq", features = ["ipv6"]}
+libsupport_zynq = { path = "@@ZYNQ_RS@@/libsupport_zynq", default-features = false, features = ["alloc_core"] }
+libcortex_a9 = { path = "@@ZYNQ_RS@@/libcortex_a9" }
+libasync = { path = "@@ZYNQ_RS@@/libasync" }
+libregister = { path = "@@ZYNQ_RS@@/libregister" }
+libconfig = { path = "@@ZYNQ_RS@@/libconfig", features = ["fat_lfn", "ipv6"] }
 
 dyld = { path = "../libdyld" }
 dwarf = { path = "../libdwarf" }
 unwind = { path = "../libunwind" }
 libc = { path = "../libc" }
-io = { path = "../libio" }
+io = { path = "../libio", features = ["alloc"] }
+ksupport = { path = "../libksupport" }
 libboard_artiq = { path = "../libboard_artiq" }
+
+[dependencies.tar-no-std]
+git = "https://git.m-labs.hk/M-Labs/tar-no-std"
+rev = "2ab6dc5"

src/runtime/src/analyzer.rs

@@ -1,10 +1,15 @@
+use alloc::rc::Rc;
+#[cfg(has_drtio)]
+use alloc::vec::Vec;
+use core::cell::RefCell;
+
 use libasync::{smoltcp::TcpStream, task};
-use libboard_zynq::smoltcp::Error;
-use libcortex_a9::cache;
+use libboard_artiq::drtio_routing;
+use libboard_zynq::{smoltcp::Error, timer::GlobalTimer};
+use libcortex_a9::{cache, mutex::Mutex};
 use log::{debug, info, warn};
 
-use crate::proto_async::*;
-use crate::pl;
+use crate::{pl, proto_async::*};
 
 const BUFFER_SIZE: usize = 512 * 1024;
 
@@ -13,9 +18,7 @@ struct Buffer {
     data: [u8; BUFFER_SIZE],
 }
 
-static mut BUFFER: Buffer = Buffer {
-    data: [0; BUFFER_SIZE]
-};
+static mut BUFFER: Buffer = Buffer { data: [0; BUFFER_SIZE] };
 
 fn arm() {
     debug!("arming RTIO analyzer");
@@ -40,13 +43,57 @@ fn disarm() {
     debug!("RTIO analyzer disarmed");
 }
 
+#[cfg(has_drtio)]
+pub mod remote_analyzer {
+    use super::*;
+    use crate::rtio_mgt::drtio;
+
+    pub struct RemoteBuffer {
+        pub total_byte_count: u64,
+        pub sent_bytes: u32,
+        pub error: bool,
+        pub data: Vec<u8>,
+    }
+
+    pub async fn get_data(
+        aux_mutex: &Rc<Mutex<bool>>,
+        routing_table: &drtio_routing::RoutingTable,
+        up_destinations: &Rc<RefCell<[bool; drtio_routing::DEST_COUNT]>>,
+        timer: GlobalTimer,
+    ) -> Result<RemoteBuffer, &'static str> {
+        // gets data from satellites and returns consolidated data
+        let mut remote_data: Vec<u8> = Vec::new();
+        let mut remote_error = false;
+        let mut remote_sent_bytes = 0;
+        let mut remote_total_bytes = 0;
+
+        let data_vec = match drtio::analyzer_query(aux_mutex, routing_table, up_destinations, timer).await {
+            Ok(data_vec) => data_vec,
+            Err(e) => return Err(e),
+        };
+        for data in data_vec {
+            remote_total_bytes += data.total_byte_count;
+            remote_sent_bytes += data.sent_bytes;
+            remote_error |= data.error;
+            remote_data.extend(data.data);
+        }
+
+        Ok(RemoteBuffer {
+            total_byte_count: remote_total_bytes,
+            sent_bytes: remote_sent_bytes,
+            error: remote_error,
+            data: remote_data,
+        })
+    }
+}
+
 #[derive(Debug)]
 struct Header {
     sent_bytes: u32,
     total_byte_count: u64,
     error_occurred: bool,
     log_channel: u8,
-    dds_onehot_sel: bool
+    dds_onehot_sel: bool,
 }
 
 async fn write_header(stream: &mut TcpStream, header: &Header) -> Result<(), Error> {
@@ -59,7 +106,13 @@ async fn write_header(stream: &mut TcpStream, header: &Header) -> Result<(), Err
     Ok(())
 }
 
-async fn handle_connection(stream: &mut TcpStream) -> Result<(), Error> {
+async fn handle_connection(
+    stream: &mut TcpStream,
+    _aux_mutex: &Rc<Mutex<bool>>,
+    _routing_table: &drtio_routing::RoutingTable,
+    _up_destinations: &Rc<RefCell<[bool; drtio_routing::DEST_COUNT]>>,
+    _timer: GlobalTimer,
+) -> Result<(), Error> {
     info!("received connection");
 
     let data = unsafe { &BUFFER.data[..] };
@@ -68,6 +121,11 @@ async fn handle_connection(stream: &mut TcpStream) -> Result<(), Error> {
     let total_byte_count = unsafe { pl::csr::rtio_analyzer::dma_byte_count_read() as u64 };
     let pointer = (total_byte_count % BUFFER_SIZE as u64) as usize;
     let wraparound = total_byte_count >= BUFFER_SIZE as u64;
+    let sent_bytes = if wraparound {
+        BUFFER_SIZE as u32
+    } else {
+        total_byte_count as u32
+    };
 
     if overflow_occurred {
         warn!("overflow occured");
@@ -76,12 +134,42 @@ async fn handle_connection(stream: &mut TcpStream) -> Result<(), Error> {
         warn!("bus error occured");
     }
 
+    #[cfg(has_drtio)]
+    let remote = remote_analyzer::get_data(_aux_mutex, _routing_table, _up_destinations, _timer).await;
+    #[cfg(has_drtio)]
+    let (header, remote_data) = match remote {
+        Ok(remote) => (
+            Header {
+                total_byte_count: total_byte_count + remote.total_byte_count,
+                sent_bytes: sent_bytes + remote.sent_bytes,
+                error_occurred: overflow_occurred | bus_error_occurred | remote.error,
+                log_channel: pl::csr::CONFIG_RTIO_LOG_CHANNEL as u8,
+                dds_onehot_sel: true,
+            },
+            remote.data,
+        ),
+        Err(e) => {
+            warn!("Error getting remote analyzer data: {}", e);
+            (
+                Header {
+                    total_byte_count: total_byte_count,
+                    sent_bytes: sent_bytes,
+                    error_occurred: true,
+                    log_channel: pl::csr::CONFIG_RTIO_LOG_CHANNEL as u8,
+                    dds_onehot_sel: true,
+                },
+                Vec::new(),
+            )
+        }
+    };
+
+    #[cfg(not(has_drtio))]
     let header = Header {
         total_byte_count: total_byte_count,
-        sent_bytes: if wraparound { BUFFER_SIZE as u32 } else { total_byte_count as u32 },
+        sent_bytes: sent_bytes,
         error_occurred: overflow_occurred | bus_error_occurred,
         log_channel: pl::csr::CONFIG_RTIO_LOG_CHANNEL as u8,
-        dds_onehot_sel: true  // kept for backward compatibility of analyzer dumps
+        dds_onehot_sel: true, // kept for backward compatibility of analyzer dumps
     };
     debug!("{:?}", header);
 
@@ -92,17 +180,28 @@ async fn handle_connection(stream: &mut TcpStream) -> Result<(), Error> {
     } else {
         stream.send(data[..pointer].iter().copied()).await?;
     }
+    #[cfg(has_drtio)]
+    stream.send(remote_data.iter().copied()).await?;
 
     Ok(())
 }
 
-pub fn start() {
+pub fn start(
+    aux_mutex: &Rc<Mutex<bool>>,
+    routing_table: &Rc<RefCell<drtio_routing::RoutingTable>>,
+    up_destinations: &Rc<RefCell<[bool; drtio_routing::DEST_COUNT]>>,
+    timer: GlobalTimer,
+) {
+    let aux_mutex = aux_mutex.clone();
+    let routing_table = routing_table.clone();
+    let up_destinations = up_destinations.clone();
     task::spawn(async move {
         loop {
             arm();
             let mut stream = TcpStream::accept(1382, 2048, 2048).await.unwrap();
             disarm();
-            let _ = handle_connection(&mut stream)
+            let routing_table = routing_table.borrow();
+            let _ = handle_connection(&mut stream, &aux_mutex, &routing_table, &up_destinations, timer)
                 .await
                 .map_err(|e| warn!("connection terminated: {:?}", e));
             let _ = stream.flush().await;

src/runtime/src/comms.rs

@@ -1,44 +1,57 @@
-use core::fmt;
-use core::cell::RefCell;
-use alloc::{vec, vec::Vec, string::String, collections::BTreeMap, rc::Rc};
-use log::{info, warn, error};
-use cslice::CSlice;
+use alloc::{collections::BTreeMap, rc::Rc, string::String, vec, vec::Vec};
+use core::{cell::RefCell, fmt, slice, str};
 
+use core_io::Error as IoError;
+use cslice::CSlice;
+use dyld::elf;
+use futures::{future::FutureExt, select_biased};
+#[cfg(has_drtio)]
+use io::Cursor;
+#[cfg(has_drtio)]
+use ksupport::rpc;
+use ksupport::{kernel, resolve_channel_name};
+#[cfg(has_drtio)]
+use libasync::delay;
+use libasync::{smoltcp::{Sockets, TcpStream},
+               task};
+use libboard_artiq::drtio_routing;
+#[cfg(feature = "target_kasli_soc")]
+use libboard_zynq::error_led::ErrorLED;
+#[cfg(has_drtio)]
+use libboard_zynq::time::Milliseconds;
+use libboard_zynq::{self as zynq,
+                    smoltcp::{self,
+                              iface::{EthernetInterfaceBuilder, NeighborCache},
+                              time::Instant,
+                              wire::IpCidr},
+                    timer::GlobalTimer};
+use libconfig::{net_settings, Config};
+use libcortex_a9::{mutex::Mutex,
+                   semaphore::Semaphore,
+                   sync_channel::{Receiver, Sender}};
+use log::{error, info, warn};
 use num_derive::{FromPrimitive, ToPrimitive};
 use num_traits::{FromPrimitive, ToPrimitive};
+#[cfg(has_drtio)]
+use tar_no_std::TarArchiveRef;
 
-use libboard_zynq::{
-    self as zynq,
-    smoltcp::{
-        self,
-        wire::IpCidr,
-        iface::{NeighborCache, EthernetInterfaceBuilder},
-        time::Instant,
-    },
-    timer::GlobalTimer,
-};
-use libcortex_a9::{semaphore::Semaphore, mutex::Mutex, sync_channel::{Sender, Receiver}};
-use futures::{select_biased, future::FutureExt};
-use libasync::{smoltcp::{Sockets, TcpStream}, task};
-use libconfig::{Config, net_settings};
-use libboard_artiq::drtio_routing;
-
-use crate::proto_async::*;
-use crate::kernel;
-use crate::rpc;
-use crate::moninj;
-use crate::mgmt;
-use crate::analyzer;
-use crate::rtio_mgt;
 #[cfg(has_drtio)]
 use crate::pl;
+use crate::{analyzer, mgmt, moninj, proto_async::*, rpc_async, rtio_dma, rtio_mgt};
+#[cfg(has_drtio)]
+use crate::{subkernel, subkernel::Error as SubkernelError};
 
 #[derive(Debug, Clone, Copy, PartialEq, Eq)]
 pub enum Error {
     NetworkError(smoltcp::Error),
+    IoError,
     UnexpectedPattern,
     UnrecognizedPacket,
     BufferExhausted,
+    #[cfg(has_drtio)]
+    SubkernelError(subkernel::Error),
+    #[cfg(has_drtio)]
+    DestinationDown,
 }
 
 pub type Result<T> = core::result::Result<T, Error>;
@@ -47,9 +60,14 @@ impl fmt::Display for Error {
     fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
         match self {
             Error::NetworkError(error) => write!(f, "network error: {}", error),
-            Error::UnexpectedPattern   => write!(f, "unexpected pattern"),
-            Error::UnrecognizedPacket  => write!(f, "unrecognized packet"),
-            Error::BufferExhausted     => write!(f, "buffer exhausted"),
+            Error::IoError => write!(f, "io error"),
+            Error::UnexpectedPattern => write!(f, "unexpected pattern"),
+            Error::UnrecognizedPacket => write!(f, "unrecognized packet"),
+            Error::BufferExhausted => write!(f, "buffer exhausted"),
+            #[cfg(has_drtio)]
+            Error::SubkernelError(error) => write!(f, "subkernel error: {:?}", error),
+            #[cfg(has_drtio)]
+            Error::DestinationDown => write!(f, "subkernel destination down"),
         }
     }
 }
@@ -60,6 +78,19 @@ impl From<smoltcp::Error> for Error {
     }
 }
 
+impl From<IoError> for Error {
+    fn from(_error: IoError) -> Self {
+        Error::IoError
+    }
+}
+
+#[cfg(has_drtio)]
+impl From<subkernel::Error> for Error {
+    fn from(error: subkernel::Error) -> Self {
+        Error::SubkernelError(error)
+    }
+}
+
 #[derive(Debug, FromPrimitive, ToPrimitive)]
 enum Request {
     SystemInfo = 3,
@@ -67,6 +98,7 @@ enum Request {
     RunKernel = 6,
     RPCReply = 7,
     RPCException = 8,
+    UploadSubkernel = 9,
 }
 
 #[derive(Debug, FromPrimitive, ToPrimitive)]
@@ -83,18 +115,18 @@ enum Reply {
 }
 
 static CACHE_STORE: Mutex<BTreeMap<String, Vec<i32>>> = Mutex::new(BTreeMap::new());
-static DMA_RECORD_STORE: Mutex<BTreeMap<String, (Vec<u8>, i64)>> = Mutex::new(BTreeMap::new());
 
 async fn write_header(stream: &TcpStream, reply: Reply) -> Result<()> {
-    stream.send_slice(&[0x5a, 0x5a, 0x5a, 0x5a, reply.to_u8().unwrap()]).await?;
+    stream
+        .send_slice(&[0x5a, 0x5a, 0x5a, 0x5a, reply.to_u8().unwrap()])
+        .await?;
     Ok(())
 }
 
 async fn read_request(stream: &TcpStream, allow_close: bool) -> Result<Option<Request>> {
     match expect(stream, &[0x5a, 0x5a, 0x5a, 0x5a]).await {
         Ok(true) => {}
-        Ok(false) =>
-            return Err(Error::UnexpectedPattern),
+        Ok(false) => return Err(Error::UnexpectedPattern),
         Err(smoltcp::Error::Finished) => {
             if allow_close {
                 info!("peer closed connection");
@@ -103,11 +135,12 @@ async fn read_request(stream: &TcpStream, allow_close: bool) -> Result<Option<Re
                 error!("peer unexpectedly closed connection");
                 return Err(smoltcp::Error::Finished)?;
             }
-        },
-        Err(e) =>
-            return Err(e)?,
+        }
+        Err(e) => return Err(e)?,
     }
-    Ok(Some(FromPrimitive::from_i8(read_i8(&stream).await?).ok_or(Error::UnrecognizedPacket)?))
+    Ok(Some(
+        FromPrimitive::from_i8(read_i8(&stream).await?).ok_or(Error::UnrecognizedPacket)?,
+    ))
 }
 
 async fn read_bytes(stream: &TcpStream, max_length: usize) -> Result<Vec<u8>> {
@@ -126,9 +159,7 @@ async fn fast_send(sender: &mut Sender<'_, kernel::Message>, content: kernel::Me
     let mut content = content;
     for _ in 0..RETRY_LIMIT {
         match sender.try_send(content) {
-            Ok(()) => {
-                return
-            },
+            Ok(()) => return,
             Err(v) => {
                 content = v;
             }
@@ -140,10 +171,8 @@ async fn fast_send(sender: &mut Sender<'_, kernel::Message>, content: kernel::Me
 async fn fast_recv(receiver: &mut Receiver<'_, kernel::Message>) -> kernel::Message {
     for _ in 0..RETRY_LIMIT {
         match receiver.try_recv() {
-            Ok(v) => {
-                return v;
-            },
-            Err(()) => ()
+            Ok(v) => return v,
+            Err(()) => (),
         }
     }
     receiver.async_recv().await
@@ -159,7 +188,14 @@ async fn write_exception_string(stream: &TcpStream, s: CSlice<'static, u8>) -> R
     Ok(())
 }
 
-async fn handle_run_kernel(stream: Option<&TcpStream>, control: &Rc<RefCell<kernel::Control>>, _up_destinations: &Rc<RefCell<[bool; drtio_routing::DEST_COUNT]>>) -> Result<()> {
+async fn handle_run_kernel(
+    stream: Option<&TcpStream>,
+    control: &Rc<RefCell<kernel::Control>>,
+    _up_destinations: &Rc<RefCell<[bool; drtio_routing::DEST_COUNT]>>,
+    aux_mutex: &Rc<Mutex<bool>>,
+    routing_table: &drtio_routing::RoutingTable,
+    timer: GlobalTimer,
+) -> Result<()> {
     control.borrow_mut().tx.async_send(kernel::Message::StartRequest).await;
     loop {
         let reply = control.borrow_mut().rx.async_recv().await;
@@ -167,7 +203,7 @@ async fn handle_run_kernel(stream: Option<&TcpStream>, control: &Rc<RefCell<kern
             kernel::Message::RpcSend { is_async, data } => {
                 if stream.is_none() {
                     error!("Unexpected RPC from startup/idle kernel!");
-                    break
+                    break;
                 }
                 let stream = stream.unwrap();
                 write_header(stream, Reply::RPCRequest).await?;
@@ -182,7 +218,7 @@ async fn handle_run_kernel(stream: Option<&TcpStream>, control: &Rc<RefCell<kern
                                 kernel::Message::RpcRecvRequest(slot) => slot,
                                 other => panic!("expected root value slot from core1, not {:?}", other),
                             };
-                            rpc::recv_return(stream, &tag, slot, &|size| {
+                            rpc_async::recv_return(stream, &tag, slot, &|size| {
                                 let control = control.clone();
                                 async move {
                                     if size == 0 {
@@ -194,46 +230,64 @@ async fn handle_run_kernel(stream: Option<&TcpStream>, control: &Rc<RefCell<kern
                                         fast_send(&mut control.tx, kernel::Message::RpcRecvReply(Ok(size))).await;
                                         match fast_recv(&mut control.rx).await {
                                             kernel::Message::RpcRecvRequest(slot) => slot,
-                                            other => panic!("expected nested value slot from kernel CPU, not {:?}", other),
+                                            other => {
+                                                panic!("expected nested value slot from kernel CPU, not {:?}", other)
+                                            }
                                         }
                                     }
                                 }
-                            }).await?;
-                            control.borrow_mut().tx.async_send(kernel::Message::RpcRecvReply(Ok(0))).await;
-                        },
+                            })
+                            .await?;
+                            control
+                                .borrow_mut()
+                                .tx
+                                .async_send(kernel::Message::RpcRecvReply(Ok(0)))
+                                .await;
+                        }
                         Request::RPCException => {
                             let mut control = control.borrow_mut();
                             match control.rx.async_recv().await {
                                 kernel::Message::RpcRecvRequest(_) => (),
                                 other => panic!("expected (ignored) root value slot from kernel CPU, not {:?}", other),
                             }
-                            let id =       read_i32(stream).await? as u32;
-                            let message =  read_i32(stream).await? as u32;
-                            let param =    [read_i64(stream).await?,
-                                            read_i64(stream).await?,
-                                            read_i64(stream).await?];
-                            let file =     read_i32(stream).await? as u32;
-                            let line =     read_i32(stream).await?;
-                            let column =   read_i32(stream).await?;
+                            let id = read_i32(stream).await? as u32;
+                            let message = read_i32(stream).await? as u32;
+                            let param = [
+                                read_i64(stream).await?,
+                                read_i64(stream).await?,
+                                read_i64(stream).await?,
+                            ];
+                            let file = read_i32(stream).await? as u32;
+                            let line = read_i32(stream).await?;
+                            let column = read_i32(stream).await?;
                             let function = read_i32(stream).await? as u32;
-                            control.tx.async_send(kernel::Message::RpcRecvReply(Err(kernel::RPCException {
-                                id, message, param, file, line, column, function
-                            }))).await;
-                        },
+                            control
+                                .tx
+                                .async_send(kernel::Message::RpcRecvReply(Err(ksupport::RPCException {
+                                    id,
+                                    message,
+                                    param,
+                                    file,
+                                    line,
+                                    column,
+                                    function,
+                                })))
+                                .await;
+                        }
                         _ => {
                             error!("unexpected RPC request from host: {:?}", host_request);
-                            return Err(Error::UnrecognizedPacket)
+                            return Err(Error::UnrecognizedPacket);
                         }
                     }
                 }
-            },
+            }
             kernel::Message::KernelFinished(async_errors) => {
                 if let Some(stream) = stream {
                     write_header(stream, Reply::KernelFinished).await?;
                     write_i8(stream, async_errors as i8).await?;
                 }
                 break;
-            },
+            }
             kernel::Message::KernelException(exceptions, stack_pointers, backtrace, async_errors) => {
                 match stream {
                     Some(stream) => {
@@ -244,7 +298,26 @@ async fn handle_run_kernel(stream: Option<&TcpStream>, control: &Rc<RefCell<kern
                         for exception in exceptions.iter() {
                             let exception = exception.as_ref().unwrap();
                             write_i32(stream, exception.id as i32).await?;
-                            write_exception_string(stream, exception.message).await?;
+
+                            if exception.message.len() == usize::MAX {
+                                // exception with host string
+                                write_exception_string(stream, exception.message).await?;
+                            } else {
+                                let msg = str::from_utf8(unsafe {
+                                    slice::from_raw_parts(exception.message.as_ptr(), exception.message.len())
+                                })
+                                .unwrap()
+                                .replace(
+                                    "{rtio_channel_info:0}",
+                                    &format!(
+                                        "0x{:04x}:{}",
+                                        exception.param[0],
+                                        resolve_channel_name(exception.param[0] as u32)
+                                    ),
+                                );
+                                write_exception_string(stream, unsafe { CSlice::new(msg.as_ptr(), msg.len()) }).await?;
+                            }
+
                             write_i64(stream, exception.param[0] as i64).await?;
                             write_i64(stream, exception.param[1] as i64).await?;
                             write_i64(stream, exception.param[2] as i64).await?;
@@ -264,7 +337,7 @@ async fn handle_run_kernel(stream: Option<&TcpStream>, control: &Rc<RefCell<kern
                             write_i32(stream, sp as i32).await?;
                         }
                         write_i8(stream, async_errors as i8).await?;
-                    },
+                    }
                     None => {
                         error!("Uncaught kernel exceptions: {:?}", exceptions);
                     }
@@ -273,27 +346,200 @@ async fn handle_run_kernel(stream: Option<&TcpStream>, control: &Rc<RefCell<kern
             }
             kernel::Message::CachePutRequest(key, value) => {
                 CACHE_STORE.lock().insert(key, value);
-            },
+            }
             kernel::Message::CacheGetRequest(key) => {
                 const DEFAULT: Vec<i32> = Vec::new();
                 let value = CACHE_STORE.lock().get(&key).unwrap_or(&DEFAULT).clone();
-                control.borrow_mut().tx.async_send(kernel::Message::CacheGetReply(value)).await;
-            },
+                control
+                    .borrow_mut()
+                    .tx
+                    .async_send(kernel::Message::CacheGetReply(value))
+                    .await;
+            }
             kernel::Message::DmaPutRequest(recorder) => {
-                DMA_RECORD_STORE.lock().insert(recorder.name, (recorder.buffer, recorder.duration));
-            },
+                let _id = rtio_dma::put_record(aux_mutex, routing_table, timer, recorder).await;
+                #[cfg(has_drtio)]
+                rtio_dma::remote_dma::upload_traces(aux_mutex, routing_table, timer, _id).await;
+            }
             kernel::Message::DmaEraseRequest(name) => {
                 // prevent possible OOM when we have large DMA record replacement.
-                DMA_RECORD_STORE.lock().remove(&name);
-            },
+                rtio_dma::erase(name, aux_mutex, routing_table, timer).await;
+            }
             kernel::Message::DmaGetRequest(name) => {
-                let result = DMA_RECORD_STORE.lock().get(&name).map(|v| v.clone());
-                control.borrow_mut().tx.async_send(kernel::Message::DmaGetReply(result)).await;
-            },
+                let result = rtio_dma::retrieve(name).await;
+                control
+                    .borrow_mut()
+                    .tx
+                    .async_send(kernel::Message::DmaGetReply(result))
+                    .await;
+            }
+            #[cfg(has_drtio)]
+            kernel::Message::DmaStartRemoteRequest { id, timestamp } => {
+                rtio_dma::remote_dma::playback(aux_mutex, routing_table, timer, id as u32, timestamp as u64).await;
+            }
+            #[cfg(has_drtio)]
+            kernel::Message::DmaAwaitRemoteRequest(id) => {
+                let result = rtio_dma::remote_dma::await_done(id as u32, Some(10_000), timer).await;
+                let reply = match result {
+                    Ok(rtio_dma::remote_dma::RemoteState::PlaybackEnded {
+                        error,
+                        channel,
+                        timestamp,
+                    }) => kernel::Message::DmaAwaitRemoteReply {
+                        timeout: false,
+                        error: error,
+                        channel: channel,
+                        timestamp: timestamp,
+                    },
+                    _ => kernel::Message::DmaAwaitRemoteReply {
+                        timeout: true,
+                        error: 0,
+                        channel: 0,
+                        timestamp: 0,
+                    },
+                };
+                control.borrow_mut().tx.async_send(reply).await;
+            }
+            #[cfg(has_drtio)]
+            kernel::Message::SubkernelLoadRunRequest { id, run } => {
+                let succeeded = match subkernel::load(aux_mutex, routing_table, timer, id, run).await {
+                    Ok(()) => true,
+                    Err(e) => {
+                        error!("Error loading subkernel: {:?}", e);
+                        false
+                    }
+                };
+                control
+                    .borrow_mut()
+                    .tx
+                    .async_send(kernel::Message::SubkernelLoadRunReply { succeeded: succeeded })
+                    .await;
+            }
+            #[cfg(has_drtio)]
+            kernel::Message::SubkernelAwaitFinishRequest { id, timeout } => {
+                let res = subkernel::await_finish(aux_mutex, routing_table, timer, id, timeout).await;
+                let status = match res {
+                    Ok(ref res) => {
+                        if res.status == subkernel::FinishStatus::CommLost {
+                            kernel::SubkernelStatus::CommLost
+                        } else if let Some(exception) = &res.exception {
+                            error!("Exception in subkernel");
+                            match stream {
+                                None => (),
+                                Some(stream) => {
+                                    write_chunk(stream, exception).await?;
+                                }
+                            }
+                            // will not be called after exception is served
+                            kernel::SubkernelStatus::OtherError
+                        } else {
+                            kernel::SubkernelStatus::NoError
+                        }
+                    }
+                    Err(SubkernelError::Timeout) => kernel::SubkernelStatus::Timeout,
+                    Err(SubkernelError::IncorrectState) => kernel::SubkernelStatus::IncorrectState,
+                    Err(_) => kernel::SubkernelStatus::OtherError,
+                };
+                control
+                    .borrow_mut()
+                    .tx
+                    .async_send(kernel::Message::SubkernelAwaitFinishReply { status: status })
+                    .await;
+            }
+            #[cfg(has_drtio)]
+            kernel::Message::SubkernelMsgSend { id, data } => {
+                let res = subkernel::message_send(aux_mutex, routing_table, timer, id, data).await;
+                match res {
+                    Ok(_) => (),
+                    Err(e) => {
+                        error!("error sending subkernel message: {:?}", e)
+                    }
+                };
+                control
+                    .borrow_mut()
+                    .tx
+                    .async_send(kernel::Message::SubkernelMsgSent)
+                    .await;
+            }
+            #[cfg(has_drtio)]
+            kernel::Message::SubkernelMsgRecvRequest { id, timeout, tags } => {
+                let message_received = subkernel::message_await(id, timeout, timer).await;
+                let (status, count) = match message_received {
+                    Ok(ref message) => (kernel::SubkernelStatus::NoError, message.count),
+                    Err(SubkernelError::Timeout) => (kernel::SubkernelStatus::Timeout, 0),
+                    Err(SubkernelError::IncorrectState) => (kernel::SubkernelStatus::IncorrectState, 0),
+                    Err(SubkernelError::CommLost) => (kernel::SubkernelStatus::CommLost, 0),
+                    Err(SubkernelError::SubkernelException) => {
+                        error!("Exception in subkernel");
+                        // just retrieve the exception
+                        let status = subkernel::await_finish(aux_mutex, routing_table, timer, id, timeout)
+                            .await
+                            .unwrap();
+                        match stream {
+                            None => (),
+                            Some(stream) => {
+                                write_chunk(stream, &status.exception.unwrap()).await?;
+                            }
+                        }
+                        (kernel::SubkernelStatus::OtherError, 0)
+                    }
+                    Err(_) => (kernel::SubkernelStatus::OtherError, 0),
+                };
+                control
+                    .borrow_mut()
+                    .tx
+                    .async_send(kernel::Message::SubkernelMsgRecvReply {
+                        status: status,
+                        count: count,
+                    })
+                    .await;
+                if let Ok(message) = message_received {
+                    // receive code almost identical to RPC recv, except we are not reading from a stream
+                    let mut reader = Cursor::new(message.data);
+                    let mut current_tags: &[u8] = &tags;
+                    let mut i = 0;
+                    loop {
+                        // kernel has to consume all arguments in the whole message
+                        let slot = match fast_recv(&mut control.borrow_mut().rx).await {
+                            kernel::Message::RpcRecvRequest(slot) => slot,
+                            other => panic!("expected root value slot from core1, not {:?}", other),
+                        };
+                        let remaining_tags = rpc::recv_return(&mut reader, &current_tags, slot, &mut |size| {
+                            if size == 0 {
+                                0 as *mut ()
+                            } else {
+                                let mut control = control.borrow_mut();
+                                control.tx.send(kernel::Message::RpcRecvReply(Ok(size)));
+                                match control.rx.recv() {
+                                    kernel::Message::RpcRecvRequest(slot) => slot,
+                                    other => {
+                                        panic!("expected nested value slot from kernel CPU, not {:?}", other)
+                                    }
+                                }
+                            }
+                        })?;
+                        control
+                            .borrow_mut()
+                            .tx
+                            .async_send(kernel::Message::RpcRecvReply(Ok(0)))
+                            .await;
+                        i += 1;
+                        if i < count {
+                            current_tags = remaining_tags;
+                        } else {
+                            break;
+                        }
+                    }
+                }
+            }
             #[cfg(has_drtio)]
             kernel::Message::UpDestinationsRequest(destination) => {
                 let result = _up_destinations.borrow()[destination as usize];
-                control.borrow_mut().tx.async_send(kernel::Message::UpDestinationsReply(result)).await;
+                control
+                    .borrow_mut()
+                    .tx
+                    .async_send(kernel::Message::UpDestinationsReply(result))
+                    .await;
             }
             _ => {
                 panic!("unexpected message from core1 while kernel was running: {:?}", reply);
@@ -303,11 +549,67 @@ async fn handle_run_kernel(stream: Option<&TcpStream>, control: &Rc<RefCell<kern
     Ok(())
 }
 
+async fn handle_flash_kernel(
+    buffer: &Vec<u8>,
+    control: &Rc<RefCell<kernel::Control>>,
+    _up_destinations: &Rc<RefCell<[bool; drtio_routing::DEST_COUNT]>>,
+    _aux_mutex: &Rc<Mutex<bool>>,
+    _routing_table: &drtio_routing::RoutingTable,
+    _timer: GlobalTimer,
+) -> Result<()> {
+    if buffer[0] == elf::ELFMAG0 && buffer[1] == elf::ELFMAG1 && buffer[2] == elf::ELFMAG2 && buffer[3] == elf::ELFMAG3
+    {
+        // assume ELF file, proceed as before
+        load_kernel(buffer, control, None).await
+    } else {
+        #[cfg(has_drtio)]
+        {
+            let archive = TarArchiveRef::new(buffer.as_ref());
+            let entries = archive.entries();
+            let mut main_lib: Vec<u8> = Vec::new();
+            for entry in entries {
+                if entry.filename().as_str() == "main.elf" {
+                    main_lib = entry.data().to_vec();
+                } else {
+                    // subkernel filename must be in format:
+                    // "<subkernel id> <destination>.elf"
+                    let filename = entry.filename();
+                    let mut iter = filename.as_str().split_whitespace();
+                    let sid: u32 = iter.next().unwrap().parse().unwrap();
+                    let dest: u8 = iter.next().unwrap().strip_suffix(".elf").unwrap().parse().unwrap();
+                    let up = _up_destinations.borrow()[dest as usize];
+                    if up {
+                        let subkernel_lib = entry.data().to_vec();
+                        subkernel::add_subkernel(sid, dest, subkernel_lib).await;
+                        match subkernel::upload(_aux_mutex, _routing_table, _timer, sid).await {
+                            Ok(_) => (),
+                            Err(_) => return Err(Error::UnexpectedPattern),
+                        }
+                    } else {
+                        return Err(Error::DestinationDown);
+                    }
+                }
+            }
+            load_kernel(&main_lib, control, None).await
+        }
+        #[cfg(not(has_drtio))]
+        {
+            panic!("multi-kernel libraries are not supported in standalone systems");
+        }
+    }
+}
 
-async fn load_kernel(buffer: &Vec<u8>, control: &Rc<RefCell<kernel::Control>>, stream: Option<&TcpStream>) -> Result<()> {
+async fn load_kernel(
+    buffer: &Vec<u8>,
+    control: &Rc<RefCell<kernel::Control>>,
+    stream: Option<&TcpStream>,
+) -> Result<()> {
     let mut control = control.borrow_mut();
     control.restart();
-    control.tx.async_send(kernel::Message::LoadRequest(buffer.to_vec())).await;
+    control
+        .tx
+        .async_send(kernel::Message::LoadRequest(buffer.to_vec()))
+        .await;
     let reply = control.rx.async_recv().await;
     match reply {
         kernel::Message::LoadCompleted => {
@@ -315,7 +617,7 @@ async fn load_kernel(buffer: &Vec<u8>, control: &Rc<RefCell<kernel::Control>>, s
                 write_header(stream, Reply::LoadCompleted).await?;
             }
             Ok(())
-        },
+        }
         kernel::Message::LoadFailed => {
             if let Some(stream) = stream {
                 write_header(stream, Reply::LoadFailed).await?;
@@ -324,7 +626,7 @@ async fn load_kernel(buffer: &Vec<u8>, control: &Rc<RefCell<kernel::Control>>, s
                 error!("Kernel load failed");
             }
             Err(Error::UnexpectedPattern)
-        },
+        }
         _ => {
             error!("unexpected message from core1: {:?}", reply);
             if let Some(stream) = stream {
@@ -336,16 +638,27 @@ async fn load_kernel(buffer: &Vec<u8>, control: &Rc<RefCell<kernel::Control>>, s
     }
 }
 
-async fn handle_connection(stream: &mut TcpStream, control: Rc<RefCell<kernel::Control>>, up_destinations: &Rc<RefCell<[bool; drtio_routing::DEST_COUNT]>>) -> Result<()> {
+async fn handle_connection(
+    stream: &mut TcpStream,
+    control: Rc<RefCell<kernel::Control>>,
+    up_destinations: &Rc<RefCell<[bool; drtio_routing::DEST_COUNT]>>,
+    aux_mutex: &Rc<Mutex<bool>>,
+    routing_table: &drtio_routing::RoutingTable,
+    timer: GlobalTimer,
+) -> Result<()> {
     stream.set_ack_delay(None);
 
     if !expect(stream, b"ARTIQ coredev\n").await? {
         return Err(Error::UnexpectedPattern);
     }
     stream.send_slice("e".as_bytes()).await?;
+    #[cfg(has_drtio)]
+    subkernel::clear_subkernels().await;
     loop {
         let request = read_request(stream, true).await?;
         if request.is_none() {
+            #[cfg(has_drtio)]
+            subkernel::clear_subkernels().await;
             return Ok(());
         }
         let request = request.unwrap();
@@ -353,17 +666,48 @@ async fn handle_connection(stream: &mut TcpStream, control: Rc<RefCell<kernel::C
             Request::SystemInfo => {
                 write_header(stream, Reply::SystemInfo).await?;
                 stream.send_slice("ARZQ".as_bytes()).await?;
-            },
+            }
             Request::LoadKernel => {
-                let buffer = read_bytes(stream, 1024*1024).await?;
+                let buffer = read_bytes(stream, 1024 * 1024).await?;
                 load_kernel(&buffer, &control, Some(stream)).await?;
-            },
+            }
             Request::RunKernel => {
-                handle_run_kernel(Some(stream), &control, &up_destinations).await?;
-            },
+                handle_run_kernel(
+                    Some(stream),
+                    &control,
+                    &up_destinations,
+                    aux_mutex,
+                    routing_table,
+                    timer,
+                )
+                .await?;
+            }
+            Request::UploadSubkernel => {
+                #[cfg(has_drtio)]
+                {
+                    let id = read_i32(stream).await? as u32;
+                    let destination = read_i8(stream).await? as u8;
+                    let buffer = read_bytes(stream, 1024 * 1024).await?;
+                    subkernel::add_subkernel(id, destination, buffer).await;
+                    match subkernel::upload(aux_mutex, routing_table, timer, id).await {
+                        Ok(_) => write_header(stream, Reply::LoadCompleted).await?,
+                        Err(_) => {
+                            write_header(stream, Reply::LoadFailed).await?;
+                            write_chunk(stream, b"subkernel failed to load").await?;
+                            return Err(Error::UnexpectedPattern);
+                        }
+                    }
+                }
+                #[cfg(not(has_drtio))]
+                {
+                    write_header(stream, Reply::LoadFailed).await?;
+                    write_chunk(stream, b"No DRTIO on this system, subkernels are not supported").await?;
+                    return Err(Error::UnexpectedPattern);
+                }
+            }
             _ => {
                 error!("unexpected request from host: {:?}", request);
-                return Err(Error::UnrecognizedPacket)
+                return Err(Error::UnrecognizedPacket);
             }
         }
     }
@@ -387,34 +731,35 @@ pub fn main(timer: GlobalTimer, cfg: Config) {
             let ip_addrs = [
                 IpCidr::new(net_addresses.ipv4_addr, 0),
                 IpCidr::new(net_addresses.ipv6_ll_addr, 0),
-                IpCidr::new(addr, 0)
+                IpCidr::new(addr, 0),
             ];
             EthernetInterfaceBuilder::new(&mut eth)
-                       .ethernet_addr(net_addresses.hardware_addr)
-                       .ip_addrs(ip_addrs)
-                       .neighbor_cache(neighbor_cache)
-                       .finalize()
+                .ethernet_addr(net_addresses.hardware_addr)
+                .ip_addrs(ip_addrs)
+                .neighbor_cache(neighbor_cache)
+                .finalize()
         }
         None => {
             let ip_addrs = [
                 IpCidr::new(net_addresses.ipv4_addr, 0),
-                IpCidr::new(net_addresses.ipv6_ll_addr, 0)
+                IpCidr::new(net_addresses.ipv6_ll_addr, 0),
             ];
             EthernetInterfaceBuilder::new(&mut eth)
-                       .ethernet_addr(net_addresses.hardware_addr)
-                       .ip_addrs(ip_addrs)
-                       .neighbor_cache(neighbor_cache)
-                       .finalize()
+                .ethernet_addr(net_addresses.hardware_addr)
+                .ip_addrs(ip_addrs)
+                .neighbor_cache(neighbor_cache)
+                .finalize()
         }
     };
 
     Sockets::init(32);
 
-    // before, mutex was on io, but now that io isn't used...?
     let aux_mutex: Rc<Mutex<bool>> = Rc::new(Mutex::new(false));
     #[cfg(has_drtio)]
-    let drtio_routing_table = Rc::new(RefCell::new(
-        drtio_routing::config_routing_table(pl::csr::DRTIO.len(), &cfg)));
+    let drtio_routing_table = Rc::new(RefCell::new(drtio_routing::config_routing_table(
+        pl::csr::DRTIO.len(),
+        &cfg,
+    )));
     #[cfg(not(has_drtio))]
     let drtio_routing_table = Rc::new(RefCell::new(drtio_routing::RoutingTable::default_empty()));
     let up_destinations = Rc::new(RefCell::new([false; drtio_routing::DEST_COUNT]));
@@ -422,17 +767,33 @@ pub fn main(timer: GlobalTimer, cfg: Config) {
     drtio_routing::interconnect_disable_all();
 
     rtio_mgt::startup(&aux_mutex, &drtio_routing_table, &up_destinations, timer);
+    ksupport::setup_device_map(&cfg);
 
-    analyzer::start();
-    moninj::start(timer, aux_mutex, drtio_routing_table);
+    analyzer::start(&aux_mutex, &drtio_routing_table, &up_destinations, timer);
+    moninj::start(timer, &aux_mutex, &drtio_routing_table);
 
     let control: Rc<RefCell<kernel::Control>> = Rc::new(RefCell::new(kernel::Control::start()));
-    let idle_kernel = Rc::new(cfg.read("idle").ok());
-    if let Ok(buffer) = cfg.read("startup") {
+    let idle_kernel = Rc::new(cfg.read("idle_kernel").ok());
+    if let Ok(buffer) = cfg.read("startup_kernel") {
         info!("Loading startup kernel...");
-        if let Ok(()) = task::block_on(load_kernel(&buffer, &control, None)) {
+        let routing_table = drtio_routing_table.borrow();
+        if let Ok(()) = task::block_on(handle_flash_kernel(
+            &buffer,
+            &control,
+            &up_destinations,
+            &aux_mutex,
+            &routing_table,
+            timer,
+        )) {
             info!("Starting startup kernel...");
-            let _ = task::block_on(handle_run_kernel(None, &control, &up_destinations));
+            let _ = task::block_on(handle_run_kernel(
+                None,
+                &control,
+                &up_destinations,
+                &aux_mutex,
+                &routing_table,
+                timer,
+            ));
             info!("Startup kernel finished!");
         } else {
             error!("Error loading startup kernel!");
@@ -458,21 +819,33 @@ pub fn main(timer: GlobalTimer, cfg: Config) {
             let connection = connection.clone();
             let terminate = terminate.clone();
             let up_destinations = up_destinations.clone();
+            let aux_mutex = aux_mutex.clone();
+            let routing_table = drtio_routing_table.clone();
 
             // we make sure the value of terminate is 0 before we start
             let _ = terminate.try_wait();
             task::spawn(async move {
+                let routing_table = routing_table.borrow();
                 select_biased! {
                     _ = (async {
-                        let _ = handle_connection(&mut stream, control.clone(), &up_destinations)
+                        let _ = handle_connection(&mut stream, control.clone(), &up_destinations, &aux_mutex, &routing_table, timer)
                             .await
                             .map_err(|e| warn!("connection terminated: {}", e));
                         if let Some(buffer) = &*idle_kernel {
                             info!("Loading idle kernel");
-                            let _ = load_kernel(&buffer, &control, None)
-                                .await.map_err(|_| warn!("error loading idle kernel"));
+                            let res = handle_flash_kernel(&buffer, &control, &up_destinations,  &aux_mutex, &routing_table, timer)
+                                .await;
+                            match res {
+                                #[cfg(has_drtio)]
+                                Err(Error::DestinationDown) => {
+                                    let mut countdown = timer.countdown();
+                                    delay(&mut countdown, Milliseconds(500)).await;
+                                }
+                                Err(_) => warn!("error loading idle kernel"),
+                                _ => (),
+                            }
                             info!("Running idle kernel");
-                            let _ = handle_run_kernel(None, &control, &up_destinations)
+                            let _ = handle_run_kernel(None, &control, &up_destinations, &aux_mutex, &routing_table, timer)
                                 .await.map_err(|_| warn!("error running idle kernel"));
                             info!("Idle kernel terminated");
                         }
@@ -486,7 +859,59 @@ pub fn main(timer: GlobalTimer, cfg: Config) {
         }
     });
 
-    Sockets::run(&mut iface, || {
-        Instant::from_millis(timer.get_time().0 as i32)
-    });
+    Sockets::run(&mut iface, || Instant::from_millis(timer.get_time().0 as i32));
+}
+
+pub fn soft_panic_main(timer: GlobalTimer, cfg: Config) -> ! {
+    let net_addresses = net_settings::get_addresses(&cfg);
+    info!("network addresses: {}", net_addresses);
+
+    let eth = zynq::eth::Eth::eth0(net_addresses.hardware_addr.0.clone());
+    const RX_LEN: usize = 64;
+    // Number of transmission buffers (minimum is two because with
+    // one, duplicate packet transmission occurs)
+    const TX_LEN: usize = 64;
+    let eth = eth.start_rx(RX_LEN);
+    let mut eth = eth.start_tx(TX_LEN);
+
+    let neighbor_cache = NeighborCache::new(alloc::collections::BTreeMap::new());
+    let mut iface = match net_addresses.ipv6_addr {
+        Some(addr) => {
+            let ip_addrs = [
+                IpCidr::new(net_addresses.ipv4_addr, 0),
+                IpCidr::new(net_addresses.ipv6_ll_addr, 0),
+                IpCidr::new(addr, 0),
+            ];
+            EthernetInterfaceBuilder::new(&mut eth)
+                .ethernet_addr(net_addresses.hardware_addr)
+                .ip_addrs(ip_addrs)
+                .neighbor_cache(neighbor_cache)
+                .finalize()
+        }
+        None => {
+            let ip_addrs = [
+                IpCidr::new(net_addresses.ipv4_addr, 0),
+                IpCidr::new(net_addresses.ipv6_ll_addr, 0),
+            ];
+            EthernetInterfaceBuilder::new(&mut eth)
+                .ethernet_addr(net_addresses.hardware_addr)
+                .ip_addrs(ip_addrs)
+                .neighbor_cache(neighbor_cache)
+                .finalize()
+        }
+    };
+
+    Sockets::init(32);
+
+    mgmt::start(cfg);
+
+    // getting eth settings disables the LED as it resets GPIO
+    // need to re-enable it here
+    #[cfg(feature = "target_kasli_soc")]
+    {
+        let mut err_led = ErrorLED::error_led();
+        err_led.toggle(true);
+    }
+
+    Sockets::run(&mut iface, || Instant::from_millis(timer.get_time().0 as i32));
 }

src/runtime/src/kernel/dma.rs

@@ -1,212 +0,0 @@
-use crate::{
-    pl::csr,
-    artiq_raise,
-    rtio,
-};
-use alloc::{vec::Vec, string::String, boxed::Box};
-use cslice::CSlice;
-use super::{KERNEL_IMAGE, KERNEL_CHANNEL_0TO1, KERNEL_CHANNEL_1TO0, Message};
-use core::mem;
-
-use libcortex_a9::cache::dcci_slice;
-
-const ALIGNMENT: usize = 16 * 8;
-
-#[repr(C)]
-pub struct DmaTrace {
-    duration: i64,
-    address:  i32,
-}
-
-#[derive(Clone, Debug)]
-pub struct DmaRecorder {
-    pub name: String,
-    pub buffer: Vec<u8>,
-    pub duration: i64,
-}
-
-static mut RECORDER: Option<DmaRecorder> = None;
-
-pub unsafe fn init_dma_recorder() {
-    // as static would remain after restart, we have to reset it,
-    // without running its destructor.
-    mem::forget(mem::replace(&mut RECORDER, None));
-}
-
-pub extern fn dma_record_start(name: CSlice<u8>) {
-    let name = String::from_utf8(name.as_ref().to_vec()).unwrap();
-    unsafe {
-        KERNEL_CHANNEL_1TO0.as_mut().unwrap().send(Message::DmaEraseRequest(name.clone()));
-    }
-    unsafe {
-        if RECORDER.is_some() {
-            artiq_raise!("DMAError", "DMA is already recording")
-        }
-
-        let library = KERNEL_IMAGE.as_ref().unwrap();
-        library.rebind(b"rtio_output",
-                       dma_record_output as *const ()).unwrap();
-        library.rebind(b"rtio_output_wide",
-                       dma_record_output_wide as *const ()).unwrap();
-
-        RECORDER = Some(DmaRecorder {
-            name,
-            buffer: Vec::new(),
-            duration: 0,
-        });
-    }
-}
-
-pub extern fn dma_record_stop(duration: i64) {
-    unsafe {
-        if RECORDER.is_none() {
-            artiq_raise!("DMAError", "DMA is not recording")
-        }
-
-        let library = KERNEL_IMAGE.as_ref().unwrap();
-        library.rebind(b"rtio_output",
-                       rtio::output as *const ()).unwrap();
-        library.rebind(b"rtio_output_wide",
-                       rtio::output_wide as *const ()).unwrap();
-
-        let mut recorder = RECORDER.take().unwrap();
-        recorder.duration = duration;
-        KERNEL_CHANNEL_1TO0.as_mut().unwrap().send(
-            Message::DmaPutRequest(recorder)
-        );
-    }
-}
-
-#[inline(always)]
-unsafe fn dma_record_output_prepare(timestamp: i64, target: i32,
-                                    words: usize) {
-    // See gateware/rtio/dma.py.
-    const HEADER_LENGTH: usize = /*length*/1 + /*channel*/3 + /*timestamp*/8 + /*address*/1;
-    let length = HEADER_LENGTH + /*data*/words * 4;
-
-    let buffer = &mut RECORDER.as_mut().unwrap().buffer;
-    buffer.reserve(length);
-    buffer.extend_from_slice(&[
-        (length    >>  0) as u8,
-        (target    >>  8) as u8,
-        (target    >>  16) as u8,
-        (target    >>  24) as u8,
-        (timestamp >>  0) as u8,
-        (timestamp >>  8) as u8,
-        (timestamp >> 16) as u8,
-        (timestamp >> 24) as u8,
-        (timestamp >> 32) as u8,
-        (timestamp >> 40) as u8,
-        (timestamp >> 48) as u8,
-        (timestamp >> 56) as u8,
-        (target    >>  0) as u8,
-    ]);
-}
-
-pub extern fn dma_record_output(target: i32, word: i32) {
-    unsafe {
-        let timestamp = rtio::now_mu();
-        dma_record_output_prepare(timestamp, target, 1);
-        RECORDER.as_mut().unwrap().buffer.extend_from_slice(&[
-            (word >>  0) as u8,
-            (word >>  8) as u8,
-            (word >> 16) as u8,
-            (word >> 24) as u8,
-        ]);
-    }
-}
-
-pub extern fn dma_record_output_wide(target: i32, words: CSlice<i32>) {
-    assert!(words.len() <= 16); // enforce the hardware limit
-
-    unsafe {
-        let timestamp = rtio::now_mu();
-        dma_record_output_prepare(timestamp, target, words.len());
-        let buffer = &mut RECORDER.as_mut().unwrap().buffer;
-        for word in words.as_ref().iter() {
-            buffer.extend_from_slice(&[
-                (word >>  0) as u8,
-                (word >>  8) as u8,
-                (word >> 16) as u8,
-                (word >> 24) as u8,
-            ]);
-        }
-    }
-}
-
-pub extern fn dma_erase(name: CSlice<u8>) {
-    let name = String::from_utf8(name.as_ref().to_vec()).unwrap();
-    unsafe {
-        KERNEL_CHANNEL_1TO0.as_mut().unwrap().send(Message::DmaEraseRequest(name));
-    }
-}
-
-pub extern fn dma_retrieve(name: CSlice<u8>) -> DmaTrace {
-    let name = String::from_utf8(name.as_ref().to_vec()).unwrap();
-    unsafe {
-        KERNEL_CHANNEL_1TO0.as_mut().unwrap().send(Message::DmaGetRequest(name));
-    }
-    match unsafe {KERNEL_CHANNEL_0TO1.as_mut().unwrap()}.recv() {
-        Message::DmaGetReply(None) => (),
-        Message::DmaGetReply(Some((mut v, duration))) => {
-            v.reserve(ALIGNMENT - 1);
-            let original_length = v.len();
-            let padding = ALIGNMENT - v.as_ptr() as usize % ALIGNMENT;
-            let padding = if padding == ALIGNMENT { 0 } else { padding };
-            for _ in 0..padding {
-                v.push(0);
-            }
-            // trailing zero to indicate end of buffer
-            v.push(0);
-            v.copy_within(0..original_length, padding);
-            dcci_slice(&v);
-            let v = Box::new(v);
-            let address = Box::into_raw(v) as *mut Vec<u8> as i32;
-            return DmaTrace {
-                address,
-                duration,
-            };
-        },
-        _ => panic!("Expected DmaGetReply after DmaGetRequest!"),
-    }
-    // we have to defer raising error as we have to drop the message first...
-    artiq_raise!("DMAError", "DMA trace not found");
-}
-
-pub extern fn dma_playback(timestamp: i64, ptr: i32) {
-    unsafe {
-        let v = Box::from_raw(ptr as *mut Vec<u8>);
-        let padding = ALIGNMENT - v.as_ptr() as usize % ALIGNMENT;
-        let padding = if padding == ALIGNMENT { 0 } else { padding };
-        let ptr = v.as_ptr().add(padding) as i32;
-
-        csr::rtio_dma::base_address_write(ptr as u32);
-        csr::rtio_dma::time_offset_write(timestamp as u64);
-
-        csr::cri_con::selected_write(1);
-        csr::rtio_dma::enable_write(1);
-        while csr::rtio_dma::enable_read() != 0 {}
-        csr::cri_con::selected_write(0);
-
-        // leave the handle as we may try to do playback for another time.
-        mem::forget(v);
-
-        let error = csr::rtio_dma::error_read();
-        if error != 0 {
-            let timestamp = csr::rtio_dma::error_timestamp_read();
-            let channel = csr::rtio_dma::error_channel_read();
-            csr::rtio_dma::error_write(1);
-            if error & 1 != 0 {
-                artiq_raise!("RTIOUnderflow",
-                    "RTIO underflow at {0} mu, channel {1}",
-                    timestamp as i64, channel as i64, 0);
-            }
-            if error & 2 != 0 {
-                artiq_raise!("RTIODestinationUnreachable",
-                    "RTIO destination unreachable, output, at {0} mu, channel {1}",
-                    timestamp as i64, channel as i64, 0);
-            }
-        }
-    }
-}
-

src/runtime/src/kernel/mod.rs

@@ -1,67 +0,0 @@
-use core::ptr;
-use alloc::{vec::Vec, string::String};
-
-use libcortex_a9::{mutex::Mutex, sync_channel, semaphore::Semaphore};
-use crate::eh_artiq;
-
-mod control;
-pub use control::Control;
-pub mod core1;
-mod api;
-mod rpc;
-mod dma;
-pub use dma::DmaRecorder;
-mod cache;
-
-#[derive(Debug, Clone)]
-pub struct RPCException {
-    pub id: u32,
-    pub message: u32,
-    pub param: [i64; 3],
-    pub file: u32,
-    pub line: i32,
-    pub column: i32,
-    pub function: u32
-}
-
-#[derive(Debug, Clone)]
-pub enum Message {
-    LoadRequest(Vec<u8>),
-    LoadCompleted,
-    LoadFailed,
-    StartRequest,
-    KernelFinished(u8),
-    KernelException(&'static [Option<eh_artiq::Exception<'static>>],
-                    &'static [eh_artiq::StackPointerBacktrace],
-                    &'static [(usize, usize)],
-                    u8),
-    RpcSend { is_async: bool, data: Vec<u8> },
-    RpcRecvRequest(*mut ()),
-    RpcRecvReply(Result<usize, RPCException>),
-
-    CacheGetRequest(String),
-    CacheGetReply(Vec<i32>),
-    CachePutRequest(String, Vec<i32>),
-
-    DmaPutRequest(DmaRecorder),
-    DmaEraseRequest(String),
-    DmaGetRequest(String),
-    DmaGetReply(Option<(Vec<u8>, i64)>),
-
-    #[cfg(has_drtio)]
-    UpDestinationsRequest(i32),
-    #[cfg(has_drtio)]
-    UpDestinationsReply(bool),
-}
-
-static CHANNEL_0TO1: Mutex<Option<sync_channel::Sender<'static, Message>>> = Mutex::new(None);
-static CHANNEL_1TO0: Mutex<Option<sync_channel::Receiver<'static, Message>>> = Mutex::new(None);
-static CHANNEL_SEM: Semaphore = Semaphore::new(0, 1);
-
-static mut KERNEL_CHANNEL_0TO1: Option<sync_channel::Receiver<'static, Message>> = None;
-static mut KERNEL_CHANNEL_1TO0: Option<sync_channel::Sender<'static, Message>> = None;
-
-pub static mut KERNEL_IMAGE: *const core1::KernelImage = ptr::null();
-
-static INIT_LOCK: Mutex<()> = Mutex::new(());
-

src/runtime/src/main.rs

@@ -1,104 +1,86 @@
 #![no_std]
 #![no_main]
-#![recursion_limit="1024"]  // for futures_util::select!
+#![recursion_limit = "1024"] // for futures_util::select!
 #![feature(alloc_error_handler)]
-#![feature(panic_info_message)]
-#![feature(c_variadic)]
 #![feature(const_btree_new)]
-#![feature(const_in_array_repeat_expressions)]
-#![feature(naked_functions)]
-#![feature(asm)]
+#![feature(panic_info_message)]
 
+#[macro_use]
 extern crate alloc;
 
-use log::{info, warn, error};
+#[cfg(all(feature = "target_kasli_soc", has_drtio))]
+use core::cell::RefCell;
 
-use libboard_zynq::{timer::GlobalTimer, mpcore, gic};
-use libasync::{task, block_async};
-use libsupport_zynq::ram;
-use nb;
-use void::Void;
+use ksupport;
+use libasync::task;
+#[cfg(has_drtio_eem)]
+use libboard_artiq::drtio_eem;
+#[cfg(all(feature = "target_kasli_soc", has_drtio))]
+use libboard_artiq::io_expander;
+use libboard_artiq::{identifier_read, logger, pl};
+use libboard_zynq::{gic, mpcore, timer::GlobalTimer};
 use libconfig::Config;
 use libcortex_a9::l2c::enable_l2_cache;
-use libboard_artiq::{logger, identifier_read, init_gateware, pl};
+use libsupport_zynq::ram;
+use log::{info, warn};
 
-const ASYNC_ERROR_COLLISION: u8 = 1 << 0;
-const ASYNC_ERROR_BUSY: u8 = 1 << 1;
-const ASYNC_ERROR_SEQUENCE_ERROR: u8 = 1 << 2;
-
-mod proto_async;
-mod comms;
-mod rpc;
-#[cfg(ki_impl = "csr")]
-#[path = "rtio_csr.rs"]
-mod rtio;
-#[cfg(ki_impl = "acp")]
-#[path = "rtio_acp.rs"]
-mod rtio;
-mod rtio_mgt;
-mod rtio_clocking;
-mod kernel;
-mod moninj;
-mod eh_artiq;
-mod panic;
-mod mgmt;
 mod analyzer;
-mod irq;
-mod i2c;
+mod comms;
 
-static mut SEEN_ASYNC_ERRORS: u8 = 0;
+mod mgmt;
+mod moninj;
+mod panic;
+mod proto_async;
+mod rpc_async;
+mod rtio_clocking;
+mod rtio_dma;
+mod rtio_mgt;
+#[cfg(has_drtio)]
+mod subkernel;
 
-pub unsafe fn get_async_errors() -> u8 {
-    let errors = SEEN_ASYNC_ERRORS;
-    SEEN_ASYNC_ERRORS = 0;
-    errors
-}
-
-fn wait_for_async_rtio_error() -> nb::Result<(), Void> {
-    unsafe {
-        if pl::csr::rtio_core::async_error_read() != 0 {
-            Ok(())
-        } else {
-            Err(nb::Error::WouldBlock)
-        }
-    }
-}
-
-async fn report_async_rtio_errors() {
+#[cfg(all(feature = "target_kasli_soc", has_drtio))]
+async fn io_expanders_service(
+    i2c_bus: RefCell<&mut libboard_zynq::i2c::I2c>,
+    io_expander0: RefCell<io_expander::IoExpander>,
+    io_expander1: RefCell<io_expander::IoExpander>,
+) {
     loop {
-        let _ = block_async!(wait_for_async_rtio_error()).await;
-        unsafe {
-            let errors = pl::csr::rtio_core::async_error_read();
-            if errors & ASYNC_ERROR_COLLISION != 0 {
-                error!("RTIO collision involving channel {}",
-                       pl::csr::rtio_core::collision_channel_read());
-            }
-            if errors & ASYNC_ERROR_BUSY != 0 {
-                error!("RTIO busy error involving channel {}",
-                       pl::csr::rtio_core::busy_channel_read());
-            }
-            if errors & ASYNC_ERROR_SEQUENCE_ERROR != 0 {
-                error!("RTIO sequence error involving channel {}",
-                       pl::csr::rtio_core::sequence_error_channel_read());
-            }
-            SEEN_ASYNC_ERRORS = errors;
-            pl::csr::rtio_core::async_error_write(errors);
+        task::r#yield().await;
+        io_expander0
+            .borrow_mut()
+            .service(&mut i2c_bus.borrow_mut())
+            .expect("I2C I/O expander #0 service failed");
+        io_expander1
+            .borrow_mut()
+            .service(&mut i2c_bus.borrow_mut())
+            .expect("I2C I/O expander #1 service failed");
+    }
+}
+
+#[cfg(has_grabber)]
+mod grabber {
+    use libasync::delay;
+    use libboard_artiq::grabber;
+    use libboard_zynq::time::Milliseconds;
+
+    use crate::GlobalTimer;
+    pub async fn grabber_thread(timer: GlobalTimer) {
+        let mut countdown = timer.countdown();
+        loop {
+            grabber::tick();
+            delay(&mut countdown, Milliseconds(200)).await;
         }
     }
 }
 
-
-
-static mut LOG_BUFFER: [u8; 1<<17] = [0; 1<<17];
+static mut LOG_BUFFER: [u8; 1 << 17] = [0; 1 << 17];
 
 #[no_mangle]
 pub fn main_core0() {
     enable_l2_cache(0x8);
     let mut timer = GlobalTimer::start();
 
-    let buffer_logger = unsafe {
-        logger::BufferLogger::new(&mut LOG_BUFFER[..])
-    };
+    let buffer_logger = unsafe { logger::BufferLogger::new(&mut LOG_BUFFER[..]) };
     buffer_logger.set_uart_log_level(log::LevelFilter::Info);
     buffer_logger.register();
     log::set_max_level(log::LevelFilter::Info);
@@ -108,10 +90,33 @@ pub fn main_core0() {
     ram::init_alloc_core0();
     gic::InterruptController::gic(mpcore::RegisterBlock::mpcore()).enable_interrupts();
 
-    init_gateware();
-    info!("detected gateware: {}", identifier_read(&mut [0; 64]));
+    info!("gateware ident: {}", identifier_read(&mut [0; 64]));
 
-    i2c::init();
+    ksupport::i2c::init();
+    #[cfg(all(feature = "target_kasli_soc", has_drtio))]
+    {
+        let i2c_bus = unsafe { (ksupport::i2c::I2C_BUS).as_mut().unwrap() };
+        let mut io_expander0 = io_expander::IoExpander::new(i2c_bus, 0).unwrap();
+        let mut io_expander1 = io_expander::IoExpander::new(i2c_bus, 1).unwrap();
+        io_expander0
+            .init(i2c_bus)
+            .expect("I2C I/O expander #0 initialization failed");
+        io_expander1
+            .init(i2c_bus)
+            .expect("I2C I/O expander #1 initialization failed");
+        // Drive TX_DISABLE to false on SFP0..3
+        io_expander0.set(0, 1, false);
+        io_expander1.set(0, 1, false);
+        io_expander0.set(1, 1, false);
+        io_expander1.set(1, 1, false);
+        io_expander0.service(i2c_bus).unwrap();
+        io_expander1.service(i2c_bus).unwrap();
+        task::spawn(io_expanders_service(
+            RefCell::new(i2c_bus),
+            RefCell::new(io_expander0),
+            RefCell::new(io_expander1),
+        ));
+    }
 
     let cfg = match Config::new() {
         Ok(cfg) => cfg,
@@ -123,7 +128,13 @@ pub fn main_core0() {
 
     rtio_clocking::init(&mut timer, &cfg);
 
-    task::spawn(report_async_rtio_errors());
+    #[cfg(has_drtio_eem)]
+    drtio_eem::init(&mut timer, &cfg);
+
+    #[cfg(has_grabber)]
+    task::spawn(grabber::grabber_thread(timer));
+
+    task::spawn(ksupport::report_async_rtio_errors());
 
     comms::main(timer, cfg);
 }

src/runtime/src/mgmt.rs

@@ -1,16 +1,17 @@
+use alloc::{rc::Rc, string::String, vec::Vec};
+use core::cell::RefCell;
+
 use futures::{future::poll_fn, task::Poll};
 use libasync::{smoltcp::TcpStream, task};
-use libboard_zynq::smoltcp;
-use libconfig::Config;
-use core::cell::RefCell;
-use alloc::{rc::Rc, vec::Vec, string::String};
-use log::{self, info, debug, warn, error, LevelFilter};
-
 use libboard_artiq::logger::{BufferLogger, LogBufferRef};
-use crate::proto_async::*;
+use libboard_zynq::{slcr, smoltcp};
+use libconfig::Config;
+use log::{self, debug, error, info, warn, LevelFilter};
 use num_derive::FromPrimitive;
 use num_traits::FromPrimitive;
 
+use crate::proto_async::*;
+
 #[derive(Debug, Clone, Copy, PartialEq, Eq)]
 pub enum Error {
     NetworkError(smoltcp::Error),
@@ -24,10 +25,10 @@ type Result<T> = core::result::Result<T, Error>;
 impl core::fmt::Display for Error {
     fn fmt(&self, f: &mut core::fmt::Formatter) -> core::fmt::Result {
         match self {
-            &Error::NetworkError(error)  => write!(f, "network error: {}", error),
+            &Error::NetworkError(error) => write!(f, "network error: {}", error),
             &Error::UnknownLogLevel(lvl) => write!(f, "unknown log level {}", lvl),
-            &Error::UnexpectedPattern    => write!(f, "unexpected pattern"),
-            &Error::UnrecognizedPacket   => write!(f, "unrecognized packet"),
+            &Error::UnexpectedPattern => write!(f, "unexpected pattern"),
+            &Error::UnrecognizedPacket => write!(f, "unrecognized packet"),
         }
     }
 }
@@ -44,6 +45,7 @@ pub enum Request {
     ClearLog = 2,
     PullLog = 7,
     SetLogFilter = 3,
+    Reboot = 5,
     SetUartLogFilter = 6,
 
     ConfigRead = 12,
@@ -55,6 +57,7 @@ pub enum Request {
 pub enum Reply {
     Success = 1,
     LogContent = 2,
+    RebootImminent = 3,
     Error = 6,
     ConfigData = 7,
 }
@@ -72,9 +75,7 @@ async fn read_log_level_filter(stream: &mut TcpStream) -> Result<log::LevelFilte
 }
 
 async fn get_logger_buffer_pred<F>(f: F) -> LogBufferRef<'static>
-where
-    F: Fn(&LogBufferRef) -> bool,
-{
+where F: Fn(&LogBufferRef) -> bool {
     poll_fn(|ctx| {
         let logger = unsafe { BufferLogger::get_logger().as_mut().unwrap() };
         match logger.buffer() {
@@ -110,10 +111,7 @@ async fn read_key(stream: &mut TcpStream) -> Result<String> {
     Ok(String::from_utf8(buffer).unwrap())
 }
 
-async fn handle_connection(
-    stream: &mut TcpStream,
-    pull_id: Rc<RefCell<u32>>,
-    cfg: Rc<Config>) -> Result<()> {
+async fn handle_connection(stream: &mut TcpStream, pull_id: Rc<RefCell<u32>>, cfg: Rc<Config>) -> Result<()> {
     if !expect(&stream, b"ARTIQ management\n").await? {
         return Err(Error::UnexpectedPattern);
     }
@@ -161,7 +159,7 @@ async fn handle_connection(
                         logger.set_buffer_log_level(LevelFilter::Trace);
                     }
                 }
-            },
+            }
             Request::SetLogFilter => {
                 let lvl = read_log_level_filter(stream).await?;
                 info!("Changing log level to {}", lvl);
@@ -172,10 +170,7 @@ async fn handle_connection(
                 let lvl = read_log_level_filter(stream).await?;
                 info!("Changing UART log level to {}", lvl);
                 unsafe {
-                    BufferLogger::get_logger()
-                        .as_ref()
-                        .unwrap()
-                        .set_uart_log_level(lvl);
+                    BufferLogger::get_logger().as_ref().unwrap().set_uart_log_level(lvl);
                 }
                 write_i8(stream, Reply::Success as i8).await?;
             }
@@ -191,16 +186,12 @@ async fn handle_connection(
                     warn!("read error: no such key");
                     write_i8(stream, Reply::Error as i8).await?;
                 }
-            },
+            }
             Request::ConfigWrite => {
                 let key = read_key(stream).await?;
                 debug!("write key: {}", key);
                 let len = read_i32(stream).await?;
-                let len = if len <= 0 {
-                    0
-                } else {
-                    len as usize
-                };
+                let len = if len <= 0 { 0 } else { len as usize };
                 let mut buffer = Vec::with_capacity(len);
                 unsafe {
                     buffer.set_len(len);
@@ -215,7 +206,7 @@ async fn handle_connection(
                     error!("failed to write: {:?}", value);
                     write_i8(stream, Reply::Error as i8).await?;
                 }
-            },
+            }
             Request::ConfigRemove => {
                 let key = read_key(stream).await?;
                 debug!("erase key: {}", key);
@@ -228,6 +219,12 @@ async fn handle_connection(
                     write_i8(stream, Reply::Error as i8).await?;
                 }
             }
+            Request::Reboot => {
+                info!("rebooting");
+                write_i8(stream, Reply::RebootImminent as i8).await?;
+                stream.flush().await?;
+                slcr::reboot();
+            }
         }
     }
 }

src/runtime/src/moninj.rs

@@ -1,26 +1,23 @@
-use core::{fmt, cell::RefCell};
 use alloc::{collections::BTreeMap, rc::Rc};
-use log::{debug, info, warn};
-use void::Void;
+use core::{cell::RefCell, fmt};
 
+use futures::{pin_mut, select_biased, FutureExt};
+use libasync::{block_async, nb, smoltcp::TcpStream, task};
 use libboard_artiq::drtio_routing;
-
-use libboard_zynq::{smoltcp, timer::GlobalTimer, time::Milliseconds};
-use libasync::{task, smoltcp::TcpStream, block_async, nb};
+use libboard_zynq::{smoltcp, time::Milliseconds, timer::GlobalTimer};
 use libcortex_a9::mutex::Mutex;
-
+use log::{debug, info, warn};
 use num_derive::{FromPrimitive, ToPrimitive};
 use num_traits::{FromPrimitive, ToPrimitive};
-use futures::{pin_mut, select_biased, FutureExt};
+use void::Void;
 
 use crate::proto_async::*;
 
-
 #[derive(Debug, Clone, Copy, PartialEq, Eq)]
 pub enum Error {
     NetworkError(smoltcp::Error),
     UnexpectedPattern,
-    UnrecognizedPacket
+    UnrecognizedPacket,
 }
 
 pub type Result<T> = core::result::Result<T, Error>;
@@ -29,8 +26,8 @@ impl fmt::Display for Error {
     fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
         match self {
             &Error::NetworkError(error) => write!(f, "network error: {}", error),
-            &Error::UnexpectedPattern   => write!(f, "unexpected pattern"),
-            &Error::UnrecognizedPacket  => write!(f, "unrecognized packet"),
+            &Error::UnexpectedPattern => write!(f, "unexpected pattern"),
+            &Error::UnrecognizedPacket => write!(f, "unrecognized packet"),
         }
     }
 }
@@ -46,60 +43,102 @@ enum HostMessage {
     MonitorProbe = 0,
     MonitorInjection = 3,
     Inject = 1,
-    GetInjectionStatus = 2
+    GetInjectionStatus = 2,
 }
 
 #[derive(Debug, FromPrimitive, ToPrimitive)]
 enum DeviceMessage {
     MonitorStatus = 0,
-    InjectionStatus = 1
+    InjectionStatus = 1,
 }
 
 #[cfg(has_drtio)]
 mod remote_moninj {
-    use super::*;
     use libboard_artiq::drtioaux_async;
-    use crate::rtio_mgt::drtio;
     use log::error;
 
-    pub async fn read_probe(aux_mutex: &Rc<Mutex<bool>>, timer: GlobalTimer, linkno: u8, destination: u8, channel: i32, probe: i8) -> i64 {
-        let reply = drtio::aux_transact(aux_mutex, linkno, &drtioaux_async::Packet::MonitorRequest { 
-            destination: destination,
-            channel: channel as _,
-            probe: probe as _},
-            timer).await;
+    use super::*;
+    use crate::rtio_mgt::drtio;
+
+    pub async fn read_probe(
+        aux_mutex: &Rc<Mutex<bool>>,
+        timer: GlobalTimer,
+        linkno: u8,
+        destination: u8,
+        channel: i32,
+        probe: i8,
+    ) -> i64 {
+        let reply = drtio::aux_transact(
+            aux_mutex,
+            linkno,
+            &drtioaux_async::Packet::MonitorRequest {
+                destination: destination,
+                channel: channel as _,
+                probe: probe as _,
+            },
+            timer,
+        )
+        .await;
         match reply {
             Ok(drtioaux_async::Packet::MonitorReply { value }) => return value as i64,
             Ok(packet) => error!("received unexpected aux packet: {:?}", packet),
-            Err("link went down") => { debug!("link is down"); },
-            Err(e) => error!("aux packet error ({})", e)
+            Err("link went down") => {
+                debug!("link is down");
+            }
+            Err(e) => error!("aux packet error ({})", e),
         }
         0
     }
 
-    pub async fn inject(aux_mutex: &Rc<Mutex<bool>>, _timer: GlobalTimer, linkno: u8, destination: u8, channel: i32, overrd: i8, value: i8) {
+    pub async fn inject(
+        aux_mutex: &Rc<Mutex<bool>>,
+        _timer: GlobalTimer,
+        linkno: u8,
+        destination: u8,
+        channel: i32,
+        overrd: i8,
+        value: i8,
+    ) {
         let _lock = aux_mutex.lock();
-        drtioaux_async::send(linkno, &drtioaux_async::Packet::InjectionRequest {
-            destination: destination,
-            channel: channel as _,
-            overrd: overrd as _,
-            value: value as _
-        }).await.unwrap();
+        drtioaux_async::send(
+            linkno,
+            &drtioaux_async::Packet::InjectionRequest {
+                destination: destination,
+                channel: channel as _,
+                overrd: overrd as _,
+                value: value as _,
+            },
+        )
+        .await
+        .unwrap();
     }
 
-    pub async fn read_injection_status(aux_mutex: &Rc<Mutex<bool>>, timer: GlobalTimer, linkno: u8, destination: u8, channel: i32, overrd: i8) -> i8 {
-        let reply = drtio::aux_transact(aux_mutex, 
+    pub async fn read_injection_status(
+        aux_mutex: &Rc<Mutex<bool>>,
+        timer: GlobalTimer,
+        linkno: u8,
+        destination: u8,
+        channel: i32,
+        overrd: i8,
+    ) -> i8 {
+        let reply = drtio::aux_transact(
+            aux_mutex,
             linkno,
             &drtioaux_async::Packet::InjectionStatusRequest {
                 destination: destination,
                 channel: channel as _,
-                overrd: overrd as _},
-            timer).await;
+                overrd: overrd as _,
+            },
+            timer,
+        )
+        .await;
         match reply {
             Ok(drtioaux_async::Packet::InjectionStatusReply { value }) => return value as i8,
             Ok(packet) => error!("received unexpected aux packet: {:?}", packet),
-            Err("link went down") => { debug!("link is down"); },
-            Err(e) => error!("aux packet error ({})", e)
+            Err("link went down") => {
+                debug!("link is down");
+            }
+            Err(e) => error!("aux packet error ({})", e),
         }
         0
     }
@@ -157,8 +196,12 @@ macro_rules! dispatch {
     }}
 }
 
-async fn handle_connection(stream: &TcpStream, timer: GlobalTimer, 
-        _aux_mutex: &Rc<Mutex<bool>>, _routing_table: &drtio_routing::RoutingTable) -> Result<()> {
+async fn handle_connection(
+    stream: &TcpStream,
+    timer: GlobalTimer,
+    _aux_mutex: &Rc<Mutex<bool>>,
+    _routing_table: &drtio_routing::RoutingTable,
+) -> Result<()> {
     if !expect(&stream, b"ARTIQ moninj\n").await? {
         return Err(Error::UnexpectedPattern);
     }
@@ -255,7 +298,13 @@ async fn handle_connection(stream: &TcpStream, timer: GlobalTimer,
     }
 }
 
-pub fn start(timer: GlobalTimer, aux_mutex: Rc<Mutex<bool>>, routing_table: Rc<RefCell<drtio_routing::RoutingTable>>) {
+pub fn start(
+    timer: GlobalTimer,
+    aux_mutex: &Rc<Mutex<bool>>,
+    routing_table: &Rc<RefCell<drtio_routing::RoutingTable>>,
+) {
+    let aux_mutex = aux_mutex.clone();
+    let routing_table = routing_table.clone();
     task::spawn(async move {
         loop {
             let aux_mutex = aux_mutex.clone();

src/runtime/src/panic.rs

@@ -1,22 +1,22 @@
-use libboard_zynq::{print, println};
-use libregister::RegisterR;
+#[cfg(feature = "target_kasli_soc")]
+use libboard_zynq::error_led::ErrorLED;
+use libboard_zynq::{print, println, timer::GlobalTimer};
+use libconfig::Config;
 use libcortex_a9::regs::MPIDR;
+use libregister::RegisterR;
+use log::error;
 use unwind::backtrace;
 
+use crate::comms::soft_panic_main;
+
 static mut PANICKED: [bool; 2] = [false; 2];
+static mut SOFT_PANICKED: bool = false;
 
 #[panic_handler]
 fn panic(info: &core::panic::PanicInfo) -> ! {
     let id = MPIDR.read().cpu_id() as usize;
-    print!("Core {} ", id);
-    unsafe {
-        if PANICKED[id] {
-            println!("nested panic!");
-            loop {}
-        }
-        PANICKED[id] = true;
-    }
-    print!("panic at ");
+    let soft_panicked = unsafe { SOFT_PANICKED };
+    print!("Core {} panic at ", id);
     if let Some(location) = info.location() {
         print!("{}:{}:{}", location.file(), location.line(), location.column());
     } else {
@@ -27,6 +27,20 @@ fn panic(info: &core::panic::PanicInfo) -> ! {
     } else {
         println!("");
     }
+    unsafe {
+        // soft panics only allowed for core 0
+        if PANICKED[id] && (SOFT_PANICKED || id == 1) {
+            println!("nested panic!");
+            loop {}
+        }
+        SOFT_PANICKED = true;
+        PANICKED[id] = true;
+    }
+    #[cfg(feature = "target_kasli_soc")]
+    {
+        let mut err_led = ErrorLED::error_led();
+        err_led.toggle(true);
+    }
     println!("Backtrace: ");
     let _ = backtrace(|ip| {
         // Backtrace gives us the return address, i.e. the address after the delay slot,
@@ -34,6 +48,26 @@ fn panic(info: &core::panic::PanicInfo) -> ! {
         print!("{:#08x} ", ip - 2 * 4);
     });
     println!("\nEnd backtrace");
-
+    if !soft_panicked && id == 0 {
+        soft_panic(info);
+    }
     loop {}
 }
+
+fn soft_panic(info: &core::panic::PanicInfo) -> ! {
+    // write panic info to log, so coremgmt can also read it
+    if let Some(location) = info.location() {
+        error!("panic at {}:{}:{}", location.file(), location.line(), location.column());
+    } else {
+        error!("panic at unknown location");
+    }
+    if let Some(message) = info.message() {
+        error!("panic message: {}", message);
+    }
+    let timer = GlobalTimer::start();
+    let cfg = match Config::new() {
+        Ok(cfg) => cfg,
+        Err(_) => Config::new_dummy(),
+    };
+    soft_panic_main(timer, cfg);
+}

src/runtime/src/proto_async.rs

@@ -1,8 +1,7 @@
-use core::cmp::min;
-use core::cell::RefCell;
+use core::{cell::RefCell, cmp::min};
 
-use libboard_zynq::smoltcp;
 use libasync::smoltcp::TcpStream;
+use libboard_zynq::smoltcp;
 
 type Result<T> = core::result::Result<T, smoltcp::Error>;
 
@@ -14,25 +13,27 @@ enum RecvState<T> {
 pub async fn expect(stream: &TcpStream, pattern: &[u8]) -> Result<bool> {
     let mut state = RecvState::NeedsMore(0, true);
     loop {
-        state = stream.recv(|buf| {
-            let mut consumed = 0;
-            if let RecvState::NeedsMore(mut cur_index, _) = state {
-                for b in buf.iter() {
-                    consumed += 1;
-                    if *b == pattern[cur_index] {
-                        if cur_index + 1 == pattern.len() {
-                            return (consumed, RecvState::Completed(true));
+        state = stream
+            .recv(|buf| {
+                let mut consumed = 0;
+                if let RecvState::NeedsMore(mut cur_index, _) = state {
+                    for b in buf.iter() {
+                        consumed += 1;
+                        if *b == pattern[cur_index] {
+                            if cur_index + 1 == pattern.len() {
+                                return (consumed, RecvState::Completed(true));
+                            }
+                        } else {
+                            return (consumed, RecvState::Completed(false));
                         }
-                    } else {
-                        return (consumed, RecvState::Completed(false));
+                        cur_index += 1;
                     }
-                    cur_index += 1;
+                    (consumed, RecvState::NeedsMore(cur_index, true))
+                } else {
+                    unreachable!();
                 }
-                (consumed, RecvState::NeedsMore(cur_index, true))
-            } else {
-                unreachable!();
-            }
-        }).await?;
+            })
+            .await?;
         if let RecvState::Completed(result) = state {
             return Ok(result);
         }
@@ -40,15 +41,11 @@ pub async fn expect(stream: &TcpStream, pattern: &[u8]) -> Result<bool> {
 }
 
 pub async fn read_bool(stream: &TcpStream) -> Result<bool> {
-    Ok(stream.recv(|buf| {
-        (1, buf[0] != 0)
-    }).await?)
+    Ok(stream.recv(|buf| (1, buf[0] != 0)).await?)
 }
 
 pub async fn read_i8(stream: &TcpStream) -> Result<i8> {
-    Ok(stream.recv(|buf| {
-        (1, buf[0] as i8)
-    }).await?)
+    Ok(stream.recv(|buf| (1, buf[0] as i8)).await?)
 }
 
 pub async fn read_i32(stream: &TcpStream) -> Result<i32> {
@@ -68,12 +65,14 @@ pub async fn read_chunk(stream: &TcpStream, destination: &mut [u8]) -> Result<()
     let destination = RefCell::new(destination);
     let mut done = 0;
     while done < total {
-        let count = stream.recv(|buf| {
-            let mut destination = destination.borrow_mut();
-            let count = min(total - done, buf.len());
-            destination[done..done + count].copy_from_slice(&buf[..count]);
-            (count, count)
-        }).await?;
+        let count = stream
+            .recv(|buf| {
+                let mut destination = destination.borrow_mut();
+                let count = min(total - done, buf.len());
+                destination[done..done + count].copy_from_slice(&buf[..count]);
+                (count, count)
+            })
+            .await?;
         done += count;
     }
     Ok(())

src/runtime/src/rpc.rs

@@ -1,614 +0,0 @@
-use core::str;
-use core::future::Future;
-use cslice::{CSlice, CMutSlice};
-use log::trace;
-use byteorder::{NativeEndian, ByteOrder};
-
-use core_io::{Write, Error};
-use libboard_zynq::smoltcp;
-use libasync::smoltcp::TcpStream;
-use alloc::boxed::Box;
-use async_recursion::async_recursion;
-
-use io::proto::ProtoWrite;
-use crate::proto_async;
-use self::tag::{Tag, TagIterator, split_tag};
-
-#[inline]
-fn alignment_offset(alignment: isize, ptr: isize) -> isize {
-    (alignment - ptr % alignment) % alignment
-}
-
-unsafe fn align_ptr<T>(ptr: *const ()) -> *const T {
-    let alignment = core::mem::align_of::<T>() as isize;
-    let fix = alignment_offset(alignment, ptr as isize);
-    ((ptr as isize) + fix) as *const T
-}
-
-unsafe fn align_ptr_mut<T>(ptr: *mut ()) -> *mut T {
-    let alignment = core::mem::align_of::<T>() as isize;
-    let fix = alignment_offset(alignment, ptr as isize);
-    ((ptr as isize) + fix) as *mut T
-}
-
-#[async_recursion(?Send)]
-async unsafe fn recv_value<F>(stream: &TcpStream, tag: Tag<'async_recursion>, data: &mut *mut (),
-                              alloc: &(impl Fn(usize) -> F + 'async_recursion))
-                             -> Result<(), smoltcp::Error>
-                          where F: Future<Output=*mut ()>
-{
-    macro_rules! consume_value {
-        ($ty:ty, |$ptr:ident| $map:expr) => ({
-            let $ptr = align_ptr_mut::<$ty>(*data);
-            *data = $ptr.offset(1) as *mut ();
-            $map
-        })
-    }
-
-    match tag {
-        Tag::None => Ok(()),
-        Tag::Bool =>
-            consume_value!(i8, |ptr| {
-                *ptr = proto_async::read_i8(stream).await?;
-                Ok(())
-            }),
-        Tag::Int32 =>
-            consume_value!(i32, |ptr| {
-                *ptr = proto_async::read_i32(stream).await?;
-                Ok(())
-            }),
-        Tag::Int64 | Tag::Float64 =>
-            consume_value!(i64, |ptr| {
-                *ptr = proto_async::read_i64(stream).await?;
-                Ok(())
-            }),
-        Tag::String | Tag::Bytes | Tag::ByteArray => {
-            consume_value!(CMutSlice<u8>, |ptr| {
-                let length = proto_async::read_i32(stream).await? as usize;
-                *ptr = CMutSlice::new(alloc(length).await as *mut u8, length);
-                proto_async::read_chunk(stream, (*ptr).as_mut()).await?;
-                Ok(())
-            })
-        }
-        Tag::Tuple(it, arity) => {
-            *data = (*data).offset(alignment_offset(tag.alignment() as isize, *data as isize));
-            let mut it = it.clone();
-            for _ in 0..arity {
-                let tag = it.next().expect("truncated tag");
-                recv_value(stream, tag, data, alloc).await?;
-            }
-            Ok(())
-        }
-        Tag::List(it) => {
-            #[repr(C)]
-            struct List { elements: *mut (), length: u32 }
-            consume_value!(*mut List, |ptr| {
-                let length = proto_async::read_i32(stream).await? as usize;
-                let tag = it.clone().next().expect("truncated tag");
-                let data_size = tag.size() * length as usize +
-                    match tag {
-                        Tag::Int64 | Tag::Float64 => 4,
-                        _ => 0
-                    };
-                let data = alloc(data_size + 8).await as *mut u8;
-                *ptr = data as *mut List;
-                let ptr = data as *mut List;
-                let data = data.offset(8);
-
-                let alignment = tag.alignment();
-                let mut data = data.offset(alignment_offset(alignment as isize, data as isize)) as *mut ();
-                (*ptr).length  = length as u32;
-                (*ptr).elements = data;
-                match tag {
-                    Tag::Bool => {
-                        let ptr = data as *mut u8;
-                        let dest = core::slice::from_raw_parts_mut(ptr, length);
-                        proto_async::read_chunk(stream, dest).await?;
-                    },
-                    Tag::Int32 => {
-                        let ptr = data as *mut u32;
-                        // reading as raw bytes and do endianness conversion later
-                        let dest = core::slice::from_raw_parts_mut(ptr as *mut u8, length * 4);
-                        proto_async::read_chunk(stream, dest).await?;
-                        drop(dest);
-                        let dest = core::slice::from_raw_parts_mut(ptr, length);
-                        NativeEndian::from_slice_u32(dest);
-                    },
-                    Tag::Int64 | Tag::Float64 => {
-                        let ptr = data as *mut u64;
-                        let dest = core::slice::from_raw_parts_mut(ptr as *mut u8, length * 8);
-                        proto_async::read_chunk(stream, dest).await?;
-                        drop(dest);
-                        let dest = core::slice::from_raw_parts_mut(ptr, length);
-                        NativeEndian::from_slice_u64(dest);
-                    },
-                    _ => {
-                        for _ in 0..(*ptr).length as usize {
-                            recv_value(stream, tag, &mut data, alloc).await?
-                        }
-                    }
-                }
-                Ok(())
-            })
-        }
-        Tag::Array(it, num_dims) => {
-            consume_value!(*mut (), |buffer| {
-                let mut total_len: u32 = 1;
-                for _ in 0..num_dims {
-                    let len = proto_async::read_i32(stream).await? as u32;
-                    total_len *= len;
-                    consume_value!(u32, |ptr| *ptr = len )
-                }
-
-                let elt_tag = it.clone().next().expect("truncated tag");
-                let data_size = elt_tag.size() * total_len as usize +
-                    match elt_tag {
-                        Tag::Int64 | Tag::Float64 => 4,
-                        _ => 0
-                    };
-                let mut data = alloc(data_size).await;
-
-                let alignment = tag.alignment();
-                data = data.offset(alignment_offset(alignment as isize, data as isize));
-                *buffer = data;
-                let length = total_len as usize;
-                match elt_tag {
-                    Tag::Bool => {
-                        let ptr = data as *mut u8;
-                        let dest = core::slice::from_raw_parts_mut(ptr, length);
-                        proto_async::read_chunk(stream, dest).await?;
-                    },
-                    Tag::Int32 => {
-                        let ptr = data as *mut u32;
-                        let dest = core::slice::from_raw_parts_mut(ptr as *mut u8, length * 4);
-                        proto_async::read_chunk(stream, dest).await?;
-                        drop(dest);
-                        let dest = core::slice::from_raw_parts_mut(ptr, length);
-                        NativeEndian::from_slice_u32(dest);
-                    },
-                    Tag::Int64 | Tag::Float64 => {
-                        let ptr = data as *mut u64;
-                        let dest = core::slice::from_raw_parts_mut(ptr as *mut u8, length * 8);
-                        proto_async::read_chunk(stream, dest).await?;
-                        drop(dest);
-                        let dest = core::slice::from_raw_parts_mut(ptr, length);
-                        NativeEndian::from_slice_u64(dest);
-                    },
-                    _ => {
-                        for _ in 0..length {
-                            recv_value(stream, elt_tag, &mut data, alloc).await?
-                        }
-                    }
-                }
-                Ok(())
-            })
-        }
-        Tag::Range(it) => {
-            *data = (*data).offset(alignment_offset(tag.alignment() as isize, *data as isize));
-            let tag = it.clone().next().expect("truncated tag");
-            recv_value(stream, tag, data, alloc).await?;
-            recv_value(stream, tag, data, alloc).await?;
-            recv_value(stream, tag, data, alloc).await?;
-            Ok(())
-        }
-        Tag::Keyword(_) => unreachable!(),
-        Tag::Object => unreachable!()
-    }
-}
-
-pub async fn recv_return<F>(stream: &TcpStream, tag_bytes: &[u8], data: *mut (),
-                            alloc: &impl Fn(usize) -> F)
-                           -> Result<(), smoltcp::Error>
-                        where F: Future<Output=*mut ()>
-{
-    let mut it = TagIterator::new(tag_bytes);
-    trace!("recv ...->{}", it);
-
-    let tag = it.next().expect("truncated tag");
-    let mut data = data;
-    unsafe { recv_value(stream, tag, &mut data, alloc).await? };
-
-    Ok(())
-}
-
-unsafe fn send_value<W>(writer: &mut W, tag: Tag, data: &mut *const ())
-                       -> Result<(), Error>
-    where W: Write + ?Sized
-{
-    macro_rules! consume_value {
-        ($ty:ty, |$ptr:ident| $map:expr) => ({
-            let $ptr = align_ptr::<$ty>(*data);
-            *data = $ptr.offset(1) as *const ();
-            $map
-        })
-    }
-
-    writer.write_u8(tag.as_u8())?;
-    match tag {
-        Tag::None => Ok(()),
-        Tag::Bool =>
-            consume_value!(u8, |ptr|
-                writer.write_u8(*ptr)),
-        Tag::Int32 =>
-            consume_value!(u32, |ptr|
-                writer.write_u32(*ptr)),
-        Tag::Int64 | Tag::Float64 =>
-            consume_value!(u64, |ptr|
-                writer.write_u64(*ptr)),
-        Tag::String =>
-            consume_value!(CSlice<u8>, |ptr|
-                writer.write_string(str::from_utf8((*ptr).as_ref()).unwrap())),
-        Tag::Bytes | Tag::ByteArray =>
-            consume_value!(CSlice<u8>, |ptr|
-                writer.write_bytes((*ptr).as_ref())),
-        Tag::Tuple(it, arity) => {
-            let mut it = it.clone();
-            writer.write_u8(arity)?;
-            for _ in 0..arity {
-                let tag = it.next().expect("truncated tag");
-                send_value(writer, tag, data)?
-            }
-            Ok(())
-        }
-        Tag::List(it) => {
-            #[repr(C)]
-            struct List { elements: *const (), length: u32 }
-            consume_value!(&List, |ptr| {
-                let length = (**ptr).length as isize;
-                writer.write_u32((*ptr).length)?;
-                let tag = it.clone().next().expect("truncated tag");
-                let mut data = (**ptr).elements;
-                writer.write_u8(tag.as_u8())?;
-                match tag {
-                    Tag::Bool => {
-                        // we can pretend this is u8...
-                        let ptr1 = align_ptr::<u8>(data);
-                        let slice = core::slice::from_raw_parts(ptr1, length as usize);
-                        writer.write_all(slice)?;
-                    },
-                    Tag::Int32 => {
-                        let ptr1 = align_ptr::<i32>(data);
-                        let slice = core::slice::from_raw_parts(ptr1 as *const u8, length as usize * 4);
-                        writer.write_all(slice)?;
-                    },
-                    Tag::Int64 | Tag::Float64 => {
-                        let ptr1 = align_ptr::<i64>(data);
-                        let slice = core::slice::from_raw_parts(ptr1 as *const u8, length as usize * 8);
-                        writer.write_all(slice)?;
-                    },
-                    // non-primitive types, not sure if this would happen but we can handle it...
-                    _ => {
-                        for _ in 0..length {
-                            send_value(writer, tag, &mut data)?;
-                        }
-                    }
-                };
-                Ok(())
-            })
-        }
-        Tag::Array(it, num_dims) => {
-            writer.write_u8(num_dims)?;
-            consume_value!(*const(), |buffer| {
-                let elt_tag = it.clone().next().expect("truncated tag");
-
-                let mut total_len = 1;
-                for _ in 0..num_dims {
-                    consume_value!(u32, |len| {
-                        writer.write_u32(*len)?;
-                        total_len *= *len;
-                    })
-                }
-                let mut data = *buffer;
-                let length = total_len as isize;
-                writer.write_u8(elt_tag.as_u8())?;
-                match elt_tag {
-                    Tag::Bool => {
-                        let ptr1 = align_ptr::<u8>(data);
-                        let slice = core::slice::from_raw_parts(ptr1, length as usize);
-                        writer.write_all(slice)?;
-                    },
-                    Tag::Int32 => {
-                        let ptr1 = align_ptr::<i32>(data);
-                        let slice = core::slice::from_raw_parts(ptr1 as *const u8, length as usize * 4);
-                        writer.write_all(slice)?;
-                    },
-                    Tag::Int64 | Tag::Float64 => {
-                        let ptr1 = align_ptr::<i64>(data);
-                        let slice = core::slice::from_raw_parts(ptr1 as *const u8, length as usize * 8);
-                        writer.write_all(slice)?;
-                    },
-                    // non-primitive types, not sure if this would happen but we can handle it...
-                    _ => {
-                        for _ in 0..length {
-                            send_value(writer, elt_tag, &mut data)?;
-                        }
-                    }
-                };
-                Ok(())
-            })
-        }
-        Tag::Range(it) => {
-            let tag = it.clone().next().expect("truncated tag");
-            send_value(writer, tag, data)?;
-            send_value(writer, tag, data)?;
-            send_value(writer, tag, data)?;
-            Ok(())
-        }
-        Tag::Keyword(it) => {
-            #[repr(C)]
-            struct Keyword<'a> { name: CSlice<'a, u8> }
-            consume_value!(Keyword, |ptr| {
-                writer.write_string(str::from_utf8((*ptr).name.as_ref()).unwrap())?;
-                let tag = it.clone().next().expect("truncated tag");
-                let mut data = ptr.offset(1) as *const ();
-                send_value(writer, tag, &mut data)
-            })
-            // Tag::Keyword never appears in composite types, so we don't have
-            // to accurately advance data.
-        }
-        Tag::Object => {
-            #[repr(C)]
-            struct Object { id: u32 }
-            consume_value!(*const Object, |ptr|
-                writer.write_u32((**ptr).id))
-        }
-    }
-}
-
-pub fn send_args<W>(writer: &mut W, service: u32, tag_bytes: &[u8], data: *const *const ())
-                   -> Result<(), Error>
-    where W: Write + ?Sized
-{
-    let (arg_tags_bytes, return_tag_bytes) = split_tag(tag_bytes);
-
-    let mut args_it = TagIterator::new(arg_tags_bytes);
-    let return_it = TagIterator::new(return_tag_bytes);
-    trace!("send<{}>({})->{}", service, args_it, return_it);
-
-    writer.write_u32(service)?;
-    for index in 0.. {
-        if let Some(arg_tag) = args_it.next() {
-            let mut data = unsafe { *data.offset(index) };
-            unsafe { send_value(writer, arg_tag, &mut data)? };
-        } else {
-            break
-        }
-    }
-    writer.write_u8(0)?;
-    writer.write_bytes(return_tag_bytes)?;
-
-    Ok(())
-}
-
-mod tag {
-    use core::fmt;
-
-    pub fn split_tag(tag_bytes: &[u8]) -> (&[u8], &[u8]) {
-        let tag_separator =
-            tag_bytes.iter()
-                     .position(|&b| b == b':')
-                     .expect("tag without a return separator");
-        let (arg_tags_bytes, rest) = tag_bytes.split_at(tag_separator);
-        let return_tag_bytes = &rest[1..];
-        (arg_tags_bytes, return_tag_bytes)
-    }
-
-    #[derive(Debug, Clone, Copy)]
-    pub enum Tag<'a> {
-        None,
-        Bool,
-        Int32,
-        Int64,
-        Float64,
-        String,
-        Bytes,
-        ByteArray,
-        Tuple(TagIterator<'a>, u8),
-        List(TagIterator<'a>),
-        Array(TagIterator<'a>, u8),
-        Range(TagIterator<'a>),
-        Keyword(TagIterator<'a>),
-        Object
-    }
-
-    impl<'a> Tag<'a> {
-        pub fn as_u8(self) -> u8 {
-            match self {
-                Tag::None => b'n',
-                Tag::Bool => b'b',
-                Tag::Int32 => b'i',
-                Tag::Int64 => b'I',
-                Tag::Float64 => b'f',
-                Tag::String => b's',
-                Tag::Bytes => b'B',
-                Tag::ByteArray => b'A',
-                Tag::Tuple(_, _) => b't',
-                Tag::List(_) => b'l',
-                Tag::Array(_, _) => b'a',
-                Tag::Range(_) => b'r',
-                Tag::Keyword(_) => b'k',
-                Tag::Object => b'O',
-            }
-        }
-
-        pub fn alignment(self) -> usize {
-            use cslice::CSlice;
-            match self {
-                Tag::None => 1,
-                Tag::Bool => core::mem::align_of::<u8>(),
-                Tag::Int32 => core::mem::align_of::<i32>(),
-                Tag::Int64 => core::mem::align_of::<i64>(),
-                Tag::Float64 => core::mem::align_of::<f64>(),
-                // struct type: align to largest element
-                Tag::Tuple(it, arity) => {
-                    let it = it.clone();
-                    it.take(arity.into()).map(|t| t.alignment()).max().unwrap()
-                },
-                Tag::Range(it) => {
-                    let it = it.clone();
-                    it.take(3).map(|t| t.alignment()).max().unwrap()
-                }
-                // CSlice basically
-                Tag::Bytes | Tag::String | Tag::ByteArray =>
-                    core::mem::align_of::<CSlice<()>>(),
-                // array buffer is allocated, so no need for alignment first
-                Tag::List(_) | Tag::Array(_, _) => 1,
-                // will not be sent from the host
-                _ => unreachable!("unexpected tag from host")
-            }
-        }
-
-        pub fn size(self) -> usize {
-            use super::alignment_offset;
-            match self {
-                Tag::None => 0,
-                Tag::Bool => 1,
-                Tag::Int32 => 4,
-                Tag::Int64 => 8,
-                Tag::Float64 => 8,
-                Tag::String => 8,
-                Tag::Bytes => 8,
-                Tag::ByteArray => 8,
-                Tag::Tuple(it, arity) => {
-                    let mut size = 0;
-                    let mut it = it.clone();
-                    for _ in 0..arity {
-                        let tag = it.next().expect("truncated tag");
-                        size += tag.size();
-                        // includes padding
-                        size += alignment_offset(tag.alignment() as isize, size as isize) as usize;
-                    }
-                    size
-                }
-                Tag::List(_) => 4,
-                Tag::Array(_, num_dims) => 4 * (1 + num_dims as usize),
-                Tag::Range(it) => {
-                    let tag = it.clone().next().expect("truncated tag");
-                    tag.size() * 3
-                }
-                Tag::Keyword(_) => unreachable!(),
-                Tag::Object => unreachable!(),
-            }
-        }
-    }
-
-    #[derive(Debug, Clone, Copy)]
-    pub struct TagIterator<'a> {
-        data: &'a [u8]
-    }
-
-    impl<'a> TagIterator<'a> {
-        pub fn new(data: &'a [u8]) -> TagIterator<'a> {
-            TagIterator { data }
-        }
-
-
-        fn sub(&mut self, count: u8) -> TagIterator<'a> {
-            let data = self.data;
-            for _ in 0..count {
-                self.next().expect("truncated tag");
-            }
-            TagIterator { data: &data[..(data.len() - self.data.len())] }
-        }
-    }
-
-    impl<'a> core::iter::Iterator for TagIterator<'a> {
-        type Item = Tag<'a>;
-
-        fn next(&mut self) -> Option<Tag<'a>> {
-            if self.data.len() == 0 {
-                return None
-            }
-
-            let tag_byte = self.data[0];
-            self.data = &self.data[1..];
-            Some(match tag_byte {
-                b'n' => Tag::None,
-                b'b' => Tag::Bool,
-                b'i' => Tag::Int32,
-                b'I' => Tag::Int64,
-                b'f' => Tag::Float64,
-                b's' => Tag::String,
-                b'B' => Tag::Bytes,
-                b'A' => Tag::ByteArray,
-                b't' => {
-                    let count = self.data[0];
-                    self.data = &self.data[1..];
-                    Tag::Tuple(self.sub(count), count)
-                }
-                b'l' => Tag::List(self.sub(1)),
-                b'a' => {
-                    let count = self.data[0];
-                    self.data = &self.data[1..];
-                    Tag::Array(self.sub(1), count)
-                }
-                b'r' => Tag::Range(self.sub(1)),
-                b'k' => Tag::Keyword(self.sub(1)),
-                b'O' => Tag::Object,
-                _    => unreachable!()
-            })
-        }
-    }
-
-    impl<'a> fmt::Display for TagIterator<'a> {
-        fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
-            let mut it = self.clone();
-            let mut first = true;
-            while let Some(tag) = it.next() {
-                if first {
-                    first = false
-                } else {
-                    write!(f, ", ")?
-                }
-
-                match tag {
-                    Tag::None =>
-                        write!(f, "None")?,
-                    Tag::Bool =>
-                        write!(f, "Bool")?,
-                    Tag::Int32 =>
-                        write!(f, "Int32")?,
-                    Tag::Int64 =>
-                        write!(f, "Int64")?,
-                    Tag::Float64 =>
-                        write!(f, "Float64")?,
-                    Tag::String =>
-                        write!(f, "String")?,
-                    Tag::Bytes =>
-                        write!(f, "Bytes")?,
-                    Tag::ByteArray =>
-                        write!(f, "ByteArray")?,
-                    Tag::Tuple(it, _) => {
-                        write!(f, "Tuple(")?;
-                        it.fmt(f)?;
-                        write!(f, ")")?;
-                    }
-                    Tag::List(it) => {
-                        write!(f, "List(")?;
-                        it.fmt(f)?;
-                        write!(f, ")")?;
-                    }
-                    Tag::Array(it, num_dims) => {
-                        write!(f, "Array(")?;
-                        it.fmt(f)?;
-                        write!(f, ", {})", num_dims)?;
-                    }
-                    Tag::Range(it) => {
-                        write!(f, "Range(")?;
-                        it.fmt(f)?;
-                        write!(f, ")")?;
-                    }
-                    Tag::Keyword(it) => {
-                        write!(f, "Keyword(")?;
-                        it.fmt(f)?;
-                        write!(f, ")")?;
-                    }
-                    Tag::Object =>
-                        write!(f, "Object")?,
-                }
-            }
-
-            Ok(())
-        }
-    }
-}

src/runtime/src/rpc_async.rs

@@ -0,0 +1,197 @@
+use alloc::boxed::Box;
+use core::future::Future;
+
+use async_recursion::async_recursion;
+use byteorder::{ByteOrder, NativeEndian};
+use cslice::CMutSlice;
+use ksupport::rpc::{tag::{Tag, TagIterator},
+                    *};
+use libasync::smoltcp::TcpStream;
+use libboard_zynq::smoltcp;
+use log::trace;
+
+use crate::proto_async;
+
+/// Reads (deserializes) `length` array or list elements of type `tag` from `stream`,
+/// writing them into the buffer given by `storage`.
+///
+/// `alloc` is used for nested allocations (if elements themselves contain
+/// lists/arrays), see [recv_value].
+#[async_recursion(?Send)]
+async unsafe fn recv_elements<F>(
+    stream: &TcpStream,
+    elt_tag: Tag<'async_recursion>,
+    length: usize,
+    storage: *mut (),
+    alloc: &(impl Fn(usize) -> F + 'async_recursion),
+) -> Result<(), smoltcp::Error>
+where
+    F: Future<Output = *mut ()>,
+{
+    // List of simple types are special-cased in the protocol for performance.
+    match elt_tag {
+        Tag::Bool => {
+            let dest = core::slice::from_raw_parts_mut(storage as *mut u8, length);
+            proto_async::read_chunk(stream, dest).await?;
+        }
+        Tag::Int32 => {
+            let ptr = storage as *mut u32;
+            let dest = core::slice::from_raw_parts_mut(ptr as *mut u8, length * 4);
+            proto_async::read_chunk(stream, dest).await?;
+            drop(dest);
+            let dest = core::slice::from_raw_parts_mut(ptr, length);
+            NativeEndian::from_slice_u32(dest);
+        }
+        Tag::Int64 | Tag::Float64 => {
+            let ptr = storage as *mut u64;
+            let dest = core::slice::from_raw_parts_mut(ptr as *mut u8, length * 8);
+            proto_async::read_chunk(stream, dest).await?;
+            drop(dest);
+            let dest = core::slice::from_raw_parts_mut(ptr, length);
+            NativeEndian::from_slice_u64(dest);
+        }
+        _ => {
+            let mut data = storage;
+            for _ in 0..length {
+                recv_value(stream, elt_tag, &mut data, alloc).await?
+            }
+        }
+    }
+    Ok(())
+}
+
+/// Reads (deserializes) a value of type `tag` from `stream`, writing the results to
+/// the kernel-side buffer `data` (the passed pointer to which is incremented to point
+/// past the just-received data). For nested allocations (lists/arrays), `alloc` is
+/// invoked any number of times with the size of the required allocation as a parameter
+/// (which is assumed to be correctly aligned for all payload types).
+#[async_recursion(?Send)]
+async unsafe fn recv_value<F>(
+    stream: &TcpStream,
+    tag: Tag<'async_recursion>,
+    data: &mut *mut (),
+    alloc: &(impl Fn(usize) -> F + 'async_recursion),
+) -> Result<(), smoltcp::Error>
+where
+    F: Future<Output = *mut ()>,
+{
+    macro_rules! consume_value {
+        ($ty:ty, | $ptr:ident | $map:expr) => {{
+            let $ptr = align_ptr_mut::<$ty>(*data);
+            *data = $ptr.offset(1) as *mut ();
+            $map
+        }};
+    }
+
+    match tag {
+        Tag::None => Ok(()),
+        Tag::Bool => consume_value!(i8, |ptr| {
+            *ptr = proto_async::read_i8(stream).await?;
+            Ok(())
+        }),
+        Tag::Int32 => consume_value!(i32, |ptr| {
+            *ptr = proto_async::read_i32(stream).await?;
+            Ok(())
+        }),
+        Tag::Int64 | Tag::Float64 => consume_value!(i64, |ptr| {
+            *ptr = proto_async::read_i64(stream).await?;
+            Ok(())
+        }),
+        Tag::String | Tag::Bytes | Tag::ByteArray => {
+            consume_value!(CMutSlice<u8>, |ptr| {
+                let length = proto_async::read_i32(stream).await? as usize;
+                *ptr = CMutSlice::new(alloc(length).await as *mut u8, length);
+                proto_async::read_chunk(stream, (*ptr).as_mut()).await?;
+                Ok(())
+            })
+        }
+        Tag::Tuple(it, arity) => {
+            let alignment = tag.alignment();
+            *data = round_up_mut(*data, alignment);
+            let mut it = it.clone();
+            for _ in 0..arity {
+                let tag = it.next().expect("truncated tag");
+                recv_value(stream, tag, data, alloc).await?
+            }
+            // Take into account any tail padding (if element(s) with largest alignment
+            // are not at the end).
+            *data = round_up_mut(*data, alignment);
+            Ok(())
+        }
+        Tag::List(it) => {
+            #[repr(C)]
+            struct List {
+                elements: *mut (),
+                length: usize,
+            }
+            consume_value!(*mut List, |ptr_to_list| {
+                let tag = it.clone().next().expect("truncated tag");
+                let length = proto_async::read_i32(stream).await? as usize;
+
+                // To avoid multiple kernel CPU roundtrips, use a single allocation for
+                // both the pointer/length List (slice) and the backing storage for the
+                // elements. We can assume that alloc() is aligned suitably, so just
+                // need to take into account any extra padding required.
+                // (Note: At the time of writing, there will never actually be any types
+                // with alignment larger than 8 bytes, so storage_offset == 0 always.)
+                let list_size = 4 + 4;
+                let storage_offset = round_up(list_size, tag.alignment());
+                let storage_size = tag.size() * length;
+
+                let allocation = alloc(storage_offset + storage_size).await as *mut u8;
+                *ptr_to_list = allocation as *mut List;
+                let storage = allocation.offset(storage_offset as isize) as *mut ();
+
+                (**ptr_to_list).length = length;
+                (**ptr_to_list).elements = storage;
+                recv_elements(stream, tag, length, storage, alloc).await
+            })
+        }
+        Tag::Array(it, num_dims) => {
+            consume_value!(*mut (), |buffer| {
+                // Deserialize length along each dimension and compute total number of
+                // elements.
+                let mut total_len: usize = 1;
+                for _ in 0..num_dims {
+                    let len = proto_async::read_i32(stream).await? as usize;
+                    total_len *= len;
+                    consume_value!(usize, |ptr| *ptr = len)
+                }
+
+                // Allocate backing storage for elements; deserialize them.
+                let elt_tag = it.clone().next().expect("truncated tag");
+                *buffer = alloc(elt_tag.size() * total_len).await;
+                recv_elements(stream, elt_tag, total_len, *buffer, alloc).await
+            })
+        }
+        Tag::Range(it) => {
+            *data = round_up_mut(*data, tag.alignment());
+            let tag = it.clone().next().expect("truncated tag");
+            recv_value(stream, tag, data, alloc).await?;
+            recv_value(stream, tag, data, alloc).await?;
+            recv_value(stream, tag, data, alloc).await?;
+            Ok(())
+        }
+        Tag::Keyword(_) => unreachable!(),
+        Tag::Object => unreachable!(),
+    }
+}
+
+pub async fn recv_return<F>(
+    stream: &TcpStream,
+    tag_bytes: &[u8],
+    data: *mut (),
+    alloc: &impl Fn(usize) -> F,
+) -> Result<(), smoltcp::Error>
+where
+    F: Future<Output = *mut ()>,
+{
+    let mut it = TagIterator::new(tag_bytes);
+    trace!("recv ...->{}", it);
+
+    let tag = it.next().expect("truncated tag");
+    let mut data = data;
+    unsafe { recv_value(stream, tag, &mut data, alloc).await? };
+
+    Ok(())
+}

src/runtime/src/rtio_clocking.rs

@@ -1,14 +1,14 @@
-use log::{info, warn, error};
-use libboard_zynq::timer::GlobalTimer;
 use embedded_hal::blocking::delay::DelayMs;
-use libconfig::Config;
+#[cfg(has_si5324)]
+use ksupport::i2c;
 use libboard_artiq::pl;
 #[cfg(has_si5324)]
-use libboard_zynq::i2c::I2c;
-#[cfg(has_si5324)]
-use crate::i2c;
-#[cfg(has_si5324)]
 use libboard_artiq::si5324;
+#[cfg(has_si5324)]
+use libboard_zynq::i2c::I2c;
+use libboard_zynq::timer::GlobalTimer;
+use libconfig::Config;
+use log::{info, warn};
 
 #[derive(Debug, PartialEq, Copy, Clone)]
 #[allow(non_camel_case_types)]
@@ -19,6 +19,7 @@ pub enum RtioClock {
     Int_150,
     Ext0_Bypass,
     Ext0_Synth0_10to125,
+    Ext0_Synth0_80to125,
     Ext0_Synth0_100to125,
     Ext0_Synth0_125to125,
 }
@@ -35,6 +36,7 @@ fn get_rtio_clock_cfg(cfg: &Config) -> RtioClock {
             "ext0_bypass_125" => RtioClock::Ext0_Bypass,
             "ext0_bypass_100" => RtioClock::Ext0_Bypass,
             "ext0_synth0_10to125" => RtioClock::Ext0_Synth0_10to125,
+            "ext0_synth0_80to125" => RtioClock::Ext0_Synth0_80to125,
             "ext0_synth0_100to125" => RtioClock::Ext0_Synth0_100to125,
             "ext0_synth0_125to125" => RtioClock::Ext0_Synth0_125to125,
             _ => {
@@ -42,17 +44,16 @@ fn get_rtio_clock_cfg(cfg: &Config) -> RtioClock {
                 RtioClock::Default
             }
         };
-    }
-    else {
+    } else {
         warn!("error reading configuration. Falling back to default.");
     }
     if res == RtioClock::Default {
-        #[cfg(rtio_frequency="100.0")]
+        #[cfg(rtio_frequency = "100.0")]
         {
             warn!("Using default configuration - internal 100MHz RTIO clock.");
             return RtioClock::Int_100;
         }
-        #[cfg(rtio_frequency="125.0")]
+        #[cfg(rtio_frequency = "125.0")]
         {
             warn!("Using default configuration - internal 125MHz RTIO clock.");
             return RtioClock::Int_125;
@@ -66,168 +67,193 @@ fn get_rtio_clock_cfg(cfg: &Config) -> RtioClock {
     res
 }
 
-
-fn init_rtio(timer: &mut GlobalTimer, _clk: RtioClock) {
-    #[cfg(has_rtio_crg_clock_sel)]
-    let clock_sel = match _clk {
-        RtioClock::Ext0_Bypass => { 
-            info!("Using bypassed external clock");
-            1
-        },
-        RtioClock::Int_125 => {
-            info!("Using internal RTIO clock");
-            0
-        },
-        _ => {
-            warn!("rtio_clock setting '{:?}' is not supported. Using default internal RTIO clock instead", _clk);
-            0
-        }
-    };
-
+#[cfg(not(has_drtio))]
+fn init_rtio(timer: &mut GlobalTimer) {
+    info!("Switching SYS clocks...");
     unsafe {
-        pl::csr::rtio_crg::pll_reset_write(1);
-        #[cfg(has_rtio_crg_clock_sel)]
-        pl::csr::rtio_crg::clock_sel_write(clock_sel);
-        pl::csr::rtio_crg::pll_reset_write(0);
-    }
-    timer.delay_ms(1);
-    let locked = unsafe { pl::csr::rtio_crg::pll_locked_read() != 0 };
-    if locked {
-        info!("RTIO PLL locked");
-    } else {
-        error!("RTIO PLL failed to lock");
+        pl::csr::sys_crg::clock_switch_write(1);
     }
+    // if it's not locked, it will hang at the CSR.
 
+    timer.delay_ms(50); // wait for CPLL/QPLL/SYS PLL lock
+    let clk = unsafe { pl::csr::sys_crg::current_clock_read() };
+    if clk == 1 {
+        info!("SYS CLK switched successfully");
+    } else {
+        panic!("SYS CLK did not switch");
+    }
     unsafe {
         pl::csr::rtio_core::reset_phy_write(1);
     }
+    info!("SYS PLL locked");
 }
 
 #[cfg(has_drtio)]
-fn init_drtio(timer: &mut GlobalTimer)
-{
+fn init_drtio(timer: &mut GlobalTimer) {
     unsafe {
-        pl::csr::drtio_transceiver::stable_clkin_write(1);
+        pl::csr::gt_drtio::stable_clkin_write(1);
+    }
+
+    timer.delay_ms(50); // wait for CPLL/QPLL/SYS PLL lock
+    let clk = unsafe { pl::csr::sys_crg::current_clock_read() };
+    if clk == 1 {
+        info!("SYS CLK switched successfully");
+    } else {
+        panic!("SYS CLK did not switch");
     }
-    timer.delay_ms(2); // wait for CPLL/QPLL lock
     unsafe {
-        pl::csr::drtio_transceiver::txenable_write(0xffffffffu32 as _);
+        pl::csr::rtio_core::reset_phy_write(1);
+        pl::csr::gt_drtio::txenable_write(0xffffffffu32 as _);
+
+        #[cfg(has_drtio_eem)]
+        pl::csr::eem_transceiver::txenable_write(0xffffffffu32 as _);
     }
 }
 
+// Si5324 input to select for locking to an external clock.
+#[cfg(has_si5324)]
+const SI5324_EXT_INPUT: si5324::Input = si5324::Input::Ckin1;
+
 #[cfg(has_si5324)]
 fn setup_si5324(i2c: &mut I2c, timer: &mut GlobalTimer, clk: RtioClock) {
     let (si5324_settings, si5324_ref_input) = match clk {
-        RtioClock::Ext0_Synth0_10to125 => { // 125 MHz output from 10 MHz CLKINx reference, 504 Hz BW
+        RtioClock::Ext0_Synth0_10to125 => {
+            // 125 MHz output from 10 MHz CLKINx reference, 504 Hz BW
             info!("using 10MHz reference to make 125MHz RTIO clock with PLL");
             (
                 si5324::FrequencySettings {
-                    n1_hs  : 10,
-                    nc1_ls : 4,
-                    n2_hs  : 10,
-                    n2_ls  : 300,
-                    n31    : 6,
-                    n32    : 6,
-                    bwsel  : 4,
-                    crystal_ref: false
+                    n1_hs: 10,
+                    nc1_ls: 4,
+                    n2_hs: 10,
+                    n2_ls: 300,
+                    n31: 6,
+                    n32: 6,
+                    bwsel: 4,
+                    crystal_as_ckin2: false,
                 },
-                si5324::Input::Ckin1
+                SI5324_EXT_INPUT,
             )
-        },
-        RtioClock::Ext0_Synth0_100to125 => { // 125MHz output, from 100MHz CLKINx reference, 586 Hz loop bandwidth
+        }
+        RtioClock::Ext0_Synth0_80to125 => {
+            // 125 MHz output from 80 MHz CLKINx reference, 611 Hz BW
+            info!("using 80MHz reference to make 125MHz RTIO clock with PLL");
+            (
+                si5324::FrequencySettings {
+                    n1_hs: 4,
+                    nc1_ls: 10,
+                    n2_hs: 10,
+                    n2_ls: 250,
+                    n31: 40,
+                    n32: 40,
+                    bwsel: 4,
+                    crystal_as_ckin2: false,
+                },
+                SI5324_EXT_INPUT,
+            )
+        }
+        RtioClock::Ext0_Synth0_100to125 => {
+            // 125MHz output, from 100MHz CLKINx reference, 586 Hz loop bandwidth
             info!("using 100MHz reference to make 125MHz RTIO clock with PLL");
             (
                 si5324::FrequencySettings {
-                    n1_hs  : 10,
-                    nc1_ls : 4,
-                    n2_hs  : 10,
-                    n2_ls  : 260,
-                    n31    : 52,
-                    n32    : 52,
-                    bwsel  : 4,
-                    crystal_ref: false
+                    n1_hs: 10,
+                    nc1_ls: 4,
+                    n2_hs: 10,
+                    n2_ls: 260,
+                    n31: 52,
+                    n32: 52,
+                    bwsel: 4,
+                    crystal_as_ckin2: false,
                 },
-                si5324::Input::Ckin1
+                SI5324_EXT_INPUT,
             )
-        },
-        RtioClock::Ext0_Synth0_125to125 => { // 125MHz output, from 125MHz CLKINx reference, 606 Hz loop bandwidth
+        }
+        RtioClock::Ext0_Synth0_125to125 => {
+            // 125MHz output, from 125MHz CLKINx reference, 606 Hz loop bandwidth
             info!("using 125MHz reference to make 125MHz RTIO clock with PLL");
             (
                 si5324::FrequencySettings {
-                    n1_hs  : 5,
-                    nc1_ls : 8,
-                    n2_hs  : 7,
-                    n2_ls  : 360,
-                    n31    : 63,
-                    n32    : 63,
-                    bwsel  : 4,
-                    crystal_ref: false
+                    n1_hs: 5,
+                    nc1_ls: 8,
+                    n2_hs: 7,
+                    n2_ls: 360,
+                    n31: 63,
+                    n32: 63,
+                    bwsel: 4,
+                    crystal_as_ckin2: false,
                 },
-                si5324::Input::Ckin1
+                SI5324_EXT_INPUT,
             )
-        },
-        RtioClock::Int_150 => { // 150MHz output, from crystal
+        }
+        RtioClock::Int_150 => {
+            // 150MHz output, from crystal
             info!("using internal 150MHz RTIO clock");
             (
                 si5324::FrequencySettings {
-                    n1_hs  : 9,
-                    nc1_ls : 4,
-                    n2_hs  : 10,
-                    n2_ls  : 33732,
-                    n31    : 7139,
-                    n32    : 7139,
-                    bwsel  : 3,
-                    crystal_ref: true
+                    n1_hs: 9,
+                    nc1_ls: 4,
+                    n2_hs: 10,
+                    n2_ls: 33732,
+                    n31: 7139,
+                    n32: 7139,
+                    bwsel: 3,
+                    crystal_as_ckin2: true,
                 },
-                si5324::Input::Ckin2
+                si5324::Input::Ckin2,
             )
-        },
-        RtioClock::Int_100 => { // 100MHz output, from crystal.
+        }
+        RtioClock::Int_100 => {
+            // 100MHz output, from crystal
             info!("using internal 100MHz RTIO clock");
             (
                 si5324::FrequencySettings {
-                    n1_hs  : 9,
-                    nc1_ls : 6,
-                    n2_hs  : 10,
-                    n2_ls  : 33732,
-                    n31    : 7139,
-                    n32    : 7139,
-                    bwsel  : 3,
-                    crystal_ref: true
+                    n1_hs: 9,
+                    nc1_ls: 6,
+                    n2_hs: 10,
+                    n2_ls: 33732,
+                    n31: 7139,
+                    n32: 7139,
+                    bwsel: 3,
+                    crystal_as_ckin2: true,
                 },
-                si5324::Input::Ckin2
+                si5324::Input::Ckin2,
             )
-        },
-        RtioClock::Int_125 => { // 125MHz output, from crystal, 7 Hz
+        }
+        RtioClock::Int_125 => {
+            // 125MHz output, from crystal, 7 Hz
             info!("using internal 125MHz RTIO clock");
             (
                 si5324::FrequencySettings {
-                    n1_hs  : 10,
-                    nc1_ls : 4,
-                    n2_hs  : 10,
-                    n2_ls  : 19972,
-                    n31    : 4565,
-                    n32    : 4565,
-                    bwsel  : 4,
-                    crystal_ref: true
+                    n1_hs: 10,
+                    nc1_ls: 4,
+                    n2_hs: 10,
+                    n2_ls: 19972,
+                    n31: 4565,
+                    n32: 4565,
+                    bwsel: 4,
+                    crystal_as_ckin2: true,
                 },
-                si5324::Input::Ckin2
+                si5324::Input::Ckin2,
             )
-        },
-        _ => { // same setting  as Int_125, but fallback to default
-            warn!("rtio_clock setting '{:?}' is unsupported. Falling back to default internal 125MHz RTIO clock.", clk);
+        }
+        _ => {
+            // same setting  as Int_125, but fallback to default
+            warn!(
+                "rtio_clock setting '{:?}' is unsupported. Falling back to default internal 125MHz RTIO clock.",
+                clk
+            );
             (
                 si5324::FrequencySettings {
-                    n1_hs  : 10,
-                    nc1_ls : 4,
-                    n2_hs  : 10,
-                    n2_ls  : 19972,
-                    n31    : 4565,
-                    n32    : 4565,
-                    bwsel  : 4,
-                    crystal_ref: true
+                    n1_hs: 10,
+                    nc1_ls: 4,
+                    n2_hs: 10,
+                    n2_ls: 19972,
+                    n31: 4565,
+                    n32: 4565,
+                    bwsel: 4,
+                    crystal_as_ckin2: true,
                 },
-                si5324::Input::Ckin2
+                si5324::Input::Ckin2,
             )
         }
     };
@@ -235,20 +261,22 @@ fn setup_si5324(i2c: &mut I2c, timer: &mut GlobalTimer, clk: RtioClock) {
 }
 
 pub fn init(timer: &mut GlobalTimer, cfg: &Config) {
-
     let clk = get_rtio_clock_cfg(cfg);
     #[cfg(has_si5324)]
     {
         let i2c = unsafe { (&mut i2c::I2C_BUS).as_mut().unwrap() };
-        let si5324_ext_input = si5324::Input::Ckin1;
         match clk {
-            RtioClock::Ext0_Bypass => si5324::bypass(i2c, si5324_ext_input, timer).expect("cannot bypass Si5324"),
+            RtioClock::Ext0_Bypass => {
+                info!("bypassing the PLL for RTIO clock");
+                si5324::bypass(i2c, SI5324_EXT_INPUT, timer).expect("cannot bypass Si5324")
+            }
             _ => setup_si5324(i2c, timer, clk),
         }
     }
+
     #[cfg(has_drtio)]
     init_drtio(timer);
 
-    init_rtio(timer, clk);
-
+    #[cfg(not(has_drtio))]
+    init_rtio(timer);
 }

src/runtime/src/rtio_csr.rs

@@ -1,213 +0,0 @@
-use core::ptr::{read_volatile, write_volatile};
-use cslice::CSlice;
-
-use crate::artiq_raise;
-
-use crate::pl::csr;
-
-pub const RTIO_O_STATUS_WAIT:                      u8 = 1;
-pub const RTIO_O_STATUS_UNDERFLOW:                 u8 = 2;
-pub const RTIO_O_STATUS_DESTINATION_UNREACHABLE:   u8 = 4;
-pub const RTIO_I_STATUS_WAIT_EVENT:                u8 = 1;
-pub const RTIO_I_STATUS_OVERFLOW:                  u8 = 2;
-pub const RTIO_I_STATUS_WAIT_STATUS:               u8 = 4;
-pub const RTIO_I_STATUS_DESTINATION_UNREACHABLE:   u8 = 8;
-
-#[repr(C)]
-pub struct TimestampedData {
-    timestamp: i64,
-    data: i32,
-}
-
-pub extern fn init() {
-    unsafe {
-        csr::rtio_core::reset_write(1);
-    }
-}
-
-pub extern fn get_counter() -> i64 {
-    unsafe {
-        csr::rtio::counter_update_write(1);
-        csr::rtio::counter_read() as i64
-    }
-}
-
-pub extern fn now_mu() -> i64 {
-    unsafe {
-        csr::rtio::now_read() as i64
-    }
-}
-
-pub extern fn at_mu(t: i64) {
-    unsafe {
-        csr::rtio::now_write(t as u64);
-    }
-}
-
-pub extern fn delay_mu(dt: i64) {
-    unsafe {
-        csr::rtio::now_write(csr::rtio::now_read() + dt as u64);
-    }
-}
-
-// writing the LSB of o_data (offset=0) triggers the RTIO write
-#[inline(always)]
-pub unsafe fn rtio_o_data_write(offset: usize, data: u32) {
-    write_volatile(
-        csr::rtio::O_DATA_ADDR.offset((csr::rtio::O_DATA_SIZE - 1 - offset) as isize),
-        data);
-}
-
-#[inline(always)]
-pub unsafe fn rtio_i_data_read(offset: usize) -> u32 {
-    read_volatile(
-        csr::rtio::I_DATA_ADDR.offset((csr::rtio::I_DATA_SIZE - 1 - offset) as isize))
-}
-
-#[inline(never)]
-unsafe fn process_exceptional_status(channel: i32, status: u8) {
-    let timestamp = csr::rtio::now_read() as i64;
-    if status & RTIO_O_STATUS_WAIT != 0 {
-        while csr::rtio::o_status_read() & RTIO_O_STATUS_WAIT != 0 {}
-    }
-    if status & RTIO_O_STATUS_UNDERFLOW != 0 {
-        artiq_raise!("RTIOUnderflow",
-            "RTIO underflow at {0} mu, channel {1}, slack {2} mu",
-            timestamp, channel as i64, timestamp - get_counter());
-    }
-    if status & RTIO_O_STATUS_DESTINATION_UNREACHABLE != 0 {
-        artiq_raise!("RTIODestinationUnreachable",
-            "RTIO destination unreachable, output, at {0} mu, channel {1}",
-            timestamp, channel as i64, 0);
-    }
-}
-
-pub extern fn output(target: i32, data: i32) {
-    unsafe {
-        csr::rtio::target_write(target as u32);
-        // writing target clears o_data
-        rtio_o_data_write(0, data as _);
-        let status = csr::rtio::o_status_read();
-        if status != 0 {
-            process_exceptional_status(target >> 8, status);
-        }
-    }
-}
-
-pub extern fn output_wide(target: i32, data: CSlice<i32>) {
-    unsafe {
-        csr::rtio::target_write(target as u32);
-        // writing target clears o_data
-        for i in (0..data.len()).rev() {
-            rtio_o_data_write(i, data[i] as _)
-        }
-        let status = csr::rtio::o_status_read();
-        if status != 0 {
-            process_exceptional_status(target >> 8, status);
-        }
-    }
-}
-
-pub extern fn input_timestamp(timeout: i64, channel: i32) -> i64 {
-    unsafe {
-        csr::rtio::target_write((channel as u32) << 8);
-        csr::rtio::i_timeout_write(timeout as u64);
-
-        let mut status = RTIO_I_STATUS_WAIT_STATUS;
-        while status & RTIO_I_STATUS_WAIT_STATUS != 0 {
-            status = csr::rtio::i_status_read();
-        }
-
-        if status & RTIO_I_STATUS_OVERFLOW != 0 {
-            artiq_raise!("RTIOOverflow",
-                         "RTIO input overflow on channel {0}",
-                         channel as i64, 0, 0);
-        }
-        if status & RTIO_I_STATUS_WAIT_EVENT != 0 {
-            return -1
-        }
-        if status & RTIO_I_STATUS_DESTINATION_UNREACHABLE != 0 {
-            artiq_raise!("RTIODestinationUnreachable",
-                         "RTIO destination unreachable, input, on channel {0}",
-                         channel as i64, 0, 0);
-        }
-
-        csr::rtio::i_timestamp_read() as i64
-    }
-}
-
-pub extern fn input_data(channel: i32) -> i32 {
-    unsafe {
-        csr::rtio::target_write((channel as u32) << 8);
-        csr::rtio::i_timeout_write(0xffffffff_ffffffff);
-
-        let mut status = RTIO_I_STATUS_WAIT_STATUS;
-        while status & RTIO_I_STATUS_WAIT_STATUS != 0 {
-            status = csr::rtio::i_status_read();
-        }
-
-        if status & RTIO_I_STATUS_OVERFLOW != 0 {
-            artiq_raise!("RTIOOverflow",
-                         "RTIO input overflow on channel {0}",
-                         channel as i64, 0, 0);
-        }
-        if status & RTIO_I_STATUS_DESTINATION_UNREACHABLE != 0 {
-            artiq_raise!("RTIODestinationUnreachable",
-                         "RTIO destination unreachable, input, on channel {0}",
-                         channel as i64, 0, 0);
-        }
-
-        rtio_i_data_read(0) as i32
-    }
-}
-
-pub extern fn input_timestamped_data(timeout: i64, channel: i32) -> TimestampedData {
-    unsafe {
-        csr::rtio::target_write((channel as u32) << 8);
-        csr::rtio::i_timeout_write(timeout as u64);
-
-        let mut status = RTIO_I_STATUS_WAIT_STATUS;
-        while status & RTIO_I_STATUS_WAIT_STATUS != 0 {
-            status = csr::rtio::i_status_read();
-        }
-
-        if status & RTIO_I_STATUS_OVERFLOW != 0 {
-            artiq_raise!("RTIOOverflow",
-                         "RTIO input overflow on channel {0}",
-                         channel as i64, 0, 0);
-        }
-        if status & RTIO_I_STATUS_WAIT_EVENT != 0 {
-            return TimestampedData { timestamp: -1, data: 0 }
-        }
-        if status & RTIO_I_STATUS_DESTINATION_UNREACHABLE != 0 {
-            artiq_raise!("RTIODestinationUnreachable",
-                         "RTIO destination unreachable, input, on channel {0}",
-                         channel as i64, 0, 0);
-        }
-
-        TimestampedData {
-            timestamp: csr::rtio::i_timestamp_read() as i64,
-            data: rtio_i_data_read(0) as i32
-        }
-    }
-}
-
-pub fn write_log(data: &[i8]) {
-    unsafe {
-        csr::rtio::target_write(csr::CONFIG_RTIO_LOG_CHANNEL << 8);
-
-        let mut word: u32 = 0;
-        for i in 0..data.len() {
-            word <<= 8;
-            word |= data[i] as u32;
-            if i % 4 == 3 {
-                rtio_o_data_write(0, word);
-                word = 0;
-            }
-        }
-
-        if word != 0 {
-            rtio_o_data_write(0, word);
-        }
-    }
-}

src/runtime/src/rtio_dma.rs

@@ -0,0 +1,361 @@
+use alloc::{collections::BTreeMap, rc::Rc, string::String, vec::Vec};
+#[cfg(has_drtio)]
+use core::mem;
+
+use ksupport::kernel::DmaRecorder;
+#[cfg(has_drtio)]
+use libasync::task;
+use libboard_artiq::drtio_routing::RoutingTable;
+use libboard_zynq::timer::GlobalTimer;
+use libcortex_a9::{cache::dcci_slice, mutex::Mutex};
+
+const ALIGNMENT: usize = 16 * 8;
+
+static DMA_RECORD_STORE: Mutex<BTreeMap<String, (u32, Vec<u8>, i64)>> = Mutex::new(BTreeMap::new());
+
+#[cfg(has_drtio)]
+pub mod remote_dma {
+    use libboard_zynq::time::Milliseconds;
+    use log::error;
+
+    use super::*;
+    use crate::rtio_mgt::drtio;
+
+    #[derive(Debug, PartialEq, Clone)]
+    pub enum RemoteState {
+        NotLoaded,
+        Loaded,
+        PlaybackEnded { error: u8, channel: u32, timestamp: u64 },
+    }
+    #[derive(Debug, Clone)]
+    struct RemoteTrace {
+        trace: Vec<u8>,
+        pub state: RemoteState,
+    }
+
+    impl From<Vec<u8>> for RemoteTrace {
+        fn from(trace: Vec<u8>) -> Self {
+            RemoteTrace {
+                trace: trace,
+                state: RemoteState::NotLoaded,
+            }
+        }
+    }
+
+    impl RemoteTrace {
+        pub fn get_trace(&self) -> &Vec<u8> {
+            &self.trace
+        }
+    }
+
+    // represents all traces for a given ID
+    struct TraceSet {
+        id: u32,
+        done_count: Mutex<usize>,
+        traces: Mutex<BTreeMap<u8, RemoteTrace>>,
+    }
+
+    impl TraceSet {
+        pub fn new(id: u32, traces: BTreeMap<u8, Vec<u8>>) -> TraceSet {
+            let mut trace_map: BTreeMap<u8, RemoteTrace> = BTreeMap::new();
+            for (destination, trace) in traces {
+                trace_map.insert(destination, trace.into());
+            }
+            TraceSet {
+                id: id,
+                done_count: Mutex::new(0),
+                traces: Mutex::new(trace_map),
+            }
+        }
+
+        pub async fn await_done(&self, timeout: Option<u64>, timer: GlobalTimer) -> Result<RemoteState, &'static str> {
+            let timeout_ms = Milliseconds(timeout.unwrap_or(10_000));
+            let limit = timer.get_time() + timeout_ms;
+            while (timer.get_time() < limit)
+                & (*(self.done_count.async_lock().await) < self.traces.async_lock().await.len())
+            {
+                task::r#yield().await;
+            }
+            if timer.get_time() >= limit {
+                error!("Remote DMA await done timed out");
+                return Err("Timed out waiting for results.");
+            }
+            let mut playback_state: RemoteState = RemoteState::PlaybackEnded {
+                error: 0,
+                channel: 0,
+                timestamp: 0,
+            };
+            let mut lock = self.traces.async_lock().await;
+            let trace_iter = lock.iter_mut();
+            for (_dest, trace) in trace_iter {
+                match trace.state {
+                    RemoteState::PlaybackEnded {
+                        error: e,
+                        channel: _c,
+                        timestamp: _ts,
+                    } => {
+                        if e != 0 {
+                            playback_state = trace.state.clone();
+                        }
+                    }
+                    _ => (),
+                }
+                trace.state = RemoteState::Loaded;
+            }
+            Ok(playback_state)
+        }
+
+        pub async fn upload_traces(
+            &mut self,
+            aux_mutex: &Rc<Mutex<bool>>,
+            routing_table: &RoutingTable,
+            timer: GlobalTimer,
+        ) {
+            let mut lock = self.traces.async_lock().await;
+            let trace_iter = lock.iter_mut();
+            for (destination, trace) in trace_iter {
+                match drtio::ddma_upload_trace(
+                    aux_mutex,
+                    routing_table,
+                    timer,
+                    self.id,
+                    *destination,
+                    trace.get_trace(),
+                )
+                .await
+                {
+                    Ok(_) => trace.state = RemoteState::Loaded,
+                    Err(e) => error!("Error adding DMA trace on destination {}: {}", destination, e),
+                }
+            }
+            *(self.done_count.async_lock().await) = 0;
+        }
+
+        pub async fn erase(&mut self, aux_mutex: &Rc<Mutex<bool>>, routing_table: &RoutingTable, timer: GlobalTimer) {
+            let lock = self.traces.async_lock().await;
+            let trace_iter = lock.keys();
+            for destination in trace_iter {
+                match drtio::ddma_send_erase(aux_mutex, routing_table, timer, self.id, *destination).await {
+                    Ok(_) => (),
+                    Err(e) => error!("Error adding DMA trace on destination {}: {}", destination, e),
+                }
+            }
+        }
+
+        pub async fn playback_done(&mut self, destination: u8, error: u8, channel: u32, timestamp: u64) {
+            let mut traces_locked = self.traces.async_lock().await;
+            let mut trace = traces_locked.get_mut(&destination).unwrap();
+            trace.state = RemoteState::PlaybackEnded {
+                error: error,
+                channel: channel,
+                timestamp: timestamp,
+            };
+            *(self.done_count.async_lock().await) += 1;
+        }
+
+        pub async fn playback(
+            &self,
+            aux_mutex: &Rc<Mutex<bool>>,
+            routing_table: &RoutingTable,
+            timer: GlobalTimer,
+            timestamp: u64,
+        ) {
+            let mut dest_list: Vec<u8> = Vec::new();
+            {
+                let lock = self.traces.async_lock().await;
+                let trace_iter = lock.iter();
+                for (dest, trace) in trace_iter {
+                    if trace.state != RemoteState::Loaded {
+                        error!("Destination {} not ready for DMA, state: {:?}", dest, trace.state);
+                        continue;
+                    }
+                    dest_list.push(dest.clone());
+                }
+            }
+            // mutex lock must be dropped before sending a playback request to avoid a deadlock,
+            // if PlaybackStatus is sent from another satellite and the state must be updated.
+            for destination in dest_list {
+                match drtio::ddma_send_playback(aux_mutex, routing_table, timer, self.id, destination, timestamp).await
+                {
+                    Ok(_) => (),
+                    Err(e) => error!("Error during remote DMA playback: {}", e),
+                }
+            }
+        }
+
+        pub async fn destination_changed(
+            &mut self,
+            aux_mutex: &Rc<Mutex<bool>>,
+            routing_table: &RoutingTable,
+            timer: GlobalTimer,
+            destination: u8,
+            up: bool,
+        ) {
+            // update state of the destination, resend traces if it's up
+            if let Some(trace) = self.traces.async_lock().await.get_mut(&destination) {
+                if up {
+                    match drtio::ddma_upload_trace(
+                        aux_mutex,
+                        routing_table,
+                        timer,
+                        self.id,
+                        destination,
+                        trace.get_trace(),
+                    )
+                    .await
+                    {
+                        Ok(_) => trace.state = RemoteState::Loaded,
+                        Err(e) => error!("Error adding DMA trace on destination {}: {}", destination, e),
+                    }
+                } else {
+                    trace.state = RemoteState::NotLoaded;
+                }
+            }
+        }
+
+        pub async fn is_empty(&self) -> bool {
+            self.traces.async_lock().await.is_empty()
+        }
+    }
+
+    static mut TRACES: BTreeMap<u32, TraceSet> = BTreeMap::new();
+
+    pub fn add_traces(id: u32, traces: BTreeMap<u8, Vec<u8>>) {
+        unsafe { TRACES.insert(id, TraceSet::new(id, traces)) };
+    }
+
+    pub async fn await_done(id: u32, timeout: Option<u64>, timer: GlobalTimer) -> Result<RemoteState, &'static str> {
+        let trace_set = unsafe { TRACES.get_mut(&id).unwrap() };
+        trace_set.await_done(timeout, timer).await
+    }
+
+    pub async fn erase(aux_mutex: &Rc<Mutex<bool>>, routing_table: &RoutingTable, timer: GlobalTimer, id: u32) {
+        let trace_set = unsafe { TRACES.get_mut(&id).unwrap() };
+        trace_set.erase(aux_mutex, routing_table, timer).await;
+        unsafe {
+            TRACES.remove(&id);
+        }
+    }
+
+    pub async fn upload_traces(aux_mutex: &Rc<Mutex<bool>>, routing_table: &RoutingTable, timer: GlobalTimer, id: u32) {
+        let trace_set = unsafe { TRACES.get_mut(&id).unwrap() };
+        trace_set.upload_traces(aux_mutex, routing_table, timer).await;
+    }
+
+    pub async fn playback(
+        aux_mutex: &Rc<Mutex<bool>>,
+        routing_table: &RoutingTable,
+        timer: GlobalTimer,
+        id: u32,
+        timestamp: u64,
+    ) {
+        let trace_set = unsafe { TRACES.get_mut(&id).unwrap() };
+        trace_set.playback(aux_mutex, routing_table, timer, timestamp).await;
+    }
+
+    pub async fn playback_done(id: u32, destination: u8, error: u8, channel: u32, timestamp: u64) {
+        let trace_set = unsafe { TRACES.get_mut(&id).unwrap() };
+        trace_set.playback_done(destination, error, channel, timestamp).await;
+    }
+
+    pub async fn destination_changed(
+        aux_mutex: &Rc<Mutex<bool>>,
+        routing_table: &RoutingTable,
+        timer: GlobalTimer,
+        destination: u8,
+        up: bool,
+    ) {
+        let trace_iter = unsafe { TRACES.values_mut() };
+        for trace_set in trace_iter {
+            trace_set
+                .destination_changed(aux_mutex, routing_table, timer, destination, up)
+                .await;
+        }
+    }
+
+    pub async fn has_remote_traces(id: u32) -> bool {
+        let trace_set = unsafe { TRACES.get_mut(&id).unwrap() };
+        !(trace_set.is_empty().await)
+    }
+}
+
+pub async fn put_record(
+    _aux_mutex: &Rc<Mutex<bool>>,
+    _routing_table: &RoutingTable,
+    _timer: GlobalTimer,
+    mut recorder: DmaRecorder,
+) -> u32 {
+    #[cfg(has_drtio)]
+    let mut remote_traces: BTreeMap<u8, Vec<u8>> = BTreeMap::new();
+
+    #[cfg(has_drtio)]
+    if recorder.enable_ddma {
+        let mut local_trace: Vec<u8> = Vec::new();
+        // analyze each entry and put in proper buckets, as the kernel core
+        // sends whole chunks, to limit comms/kernel CPU communication,
+        // and as only comms core has access to varios DMA buffers.
+        let mut ptr = 0;
+        recorder.buffer.push(0);
+        while recorder.buffer[ptr] != 0 {
+            // ptr + 3 = tgt >> 24 (destination)
+            let len = recorder.buffer[ptr] as usize;
+            let destination = recorder.buffer[ptr + 3];
+            if destination == 0 {
+                local_trace.extend(&recorder.buffer[ptr..ptr + len]);
+            } else {
+                if let Some(remote_trace) = remote_traces.get_mut(&destination) {
+                    remote_trace.extend(&recorder.buffer[ptr..ptr + len]);
+                } else {
+                    remote_traces.insert(destination, recorder.buffer[ptr..ptr + len].to_vec());
+                }
+            }
+            // and jump to the next event
+            ptr += len;
+        }
+        mem::swap(&mut recorder.buffer, &mut local_trace);
+    }
+    // trailing zero to indicate end of buffer
+    recorder.buffer.push(0);
+    recorder.buffer.reserve(ALIGNMENT - 1);
+    let original_length = recorder.buffer.len();
+    let padding = ALIGNMENT - recorder.buffer.as_ptr() as usize % ALIGNMENT;
+    let padding = if padding == ALIGNMENT { 0 } else { padding };
+    for _ in 0..padding {
+        recorder.buffer.push(0);
+    }
+    recorder.buffer.copy_within(0..original_length, padding);
+    dcci_slice(&recorder.buffer);
+
+    let ptr = recorder.buffer[padding..].as_ptr() as u32;
+
+    let _old_record = DMA_RECORD_STORE
+        .lock()
+        .insert(recorder.name, (ptr, recorder.buffer, recorder.duration));
+
+    #[cfg(has_drtio)]
+    {
+        if let Some((old_id, _v, _d)) = _old_record {
+            remote_dma::erase(_aux_mutex, _routing_table, _timer, old_id).await;
+        }
+        remote_dma::add_traces(ptr, remote_traces);
+    }
+
+    ptr
+}
+
+pub async fn erase(name: String, _aux_mutex: &Rc<Mutex<bool>>, _routing_table: &RoutingTable, _timer: GlobalTimer) {
+    let _entry = DMA_RECORD_STORE.lock().remove(&name);
+    #[cfg(has_drtio)]
+    if let Some((id, _v, _d)) = _entry {
+        remote_dma::erase(_aux_mutex, _routing_table, _timer, id).await;
+    }
+}
+
+pub async fn retrieve(name: String) -> Option<(i32, i64, bool)> {
+    let (ptr, _v, duration) = DMA_RECORD_STORE.lock().get(&name)?.clone();
+    #[cfg(has_drtio)]
+    let uses_ddma = remote_dma::has_remote_traces(ptr).await;
+    #[cfg(not(has_drtio))]
+    let uses_ddma = false;
+    Some((ptr as i32, duration, uses_ddma))
+}

src/runtime/src/rtio_mgt.rs

@@ -1,26 +1,34 @@
-use core::cell::RefCell;
 use alloc::rc::Rc;
-use libboard_zynq::timer::GlobalTimer;
-use libboard_artiq::{pl::csr, drtio_routing};
-use libcortex_a9::mutex::Mutex;
+use core::cell::RefCell;
 
+use libboard_artiq::{drtio_routing, pl::csr};
+use libboard_zynq::timer::GlobalTimer;
+use libcortex_a9::mutex::Mutex;
 
 #[cfg(has_drtio)]
 pub mod drtio {
-    use super::*;
-    use crate::{SEEN_ASYNC_ERRORS, ASYNC_ERROR_BUSY, ASYNC_ERROR_SEQUENCE_ERROR, ASYNC_ERROR_COLLISION};
-    use libboard_artiq::drtioaux_async;
-    use libboard_artiq::drtioaux_async::Packet;
-    use libboard_artiq::drtioaux::Error;
-    use log::{warn, error, info};
-    use embedded_hal::blocking::delay::DelayMs;
-    use libasync::{task, delay};
-    use libboard_zynq::time::Milliseconds;
+    use alloc::vec::Vec;
 
-    pub fn startup(aux_mutex: &Rc<Mutex<bool>>,
-            routing_table: &Rc<RefCell<drtio_routing::RoutingTable>>,
-            up_destinations: &Rc<RefCell<[bool; drtio_routing::DEST_COUNT]>>,
-            timer: GlobalTimer) {
+    use embedded_hal::blocking::delay::DelayMs;
+    use ksupport::{resolve_channel_name, ASYNC_ERROR_BUSY, ASYNC_ERROR_COLLISION, ASYNC_ERROR_SEQUENCE_ERROR,
+                   SEEN_ASYNC_ERRORS};
+    use libasync::{delay, task};
+    use libboard_artiq::{drtioaux::Error,
+                         drtioaux_async,
+                         drtioaux_async::Packet,
+                         drtioaux_proto::{PayloadStatus, MASTER_PAYLOAD_MAX_SIZE}};
+    use libboard_zynq::time::Milliseconds;
+    use log::{error, info, warn};
+
+    use super::*;
+    use crate::{analyzer::remote_analyzer::RemoteBuffer, rtio_dma::remote_dma, subkernel};
+
+    pub fn startup(
+        aux_mutex: &Rc<Mutex<bool>>,
+        routing_table: &Rc<RefCell<drtio_routing::RoutingTable>>,
+        up_destinations: &Rc<RefCell<[bool; drtio_routing::DEST_COUNT]>>,
+        timer: GlobalTimer,
+    ) {
         let aux_mutex = aux_mutex.clone();
         let routing_table = routing_table.clone();
         let up_destinations = up_destinations.clone();
@@ -32,8 +40,42 @@ pub mod drtio {
 
     async fn link_rx_up(linkno: u8) -> bool {
         let linkno = linkno as usize;
-        unsafe {
-            (csr::DRTIO[linkno].rx_up_read)() == 1
+        unsafe { (csr::DRTIO[linkno].rx_up_read)() == 1 }
+    }
+
+    async fn process_async_packets(aux_mutex: &Mutex<bool>, linkno: u8, packet: Packet) -> Option<Packet> {
+        // returns None if an async packet has been consumed
+        match packet {
+            Packet::DmaPlaybackStatus {
+                id,
+                destination,
+                error,
+                channel,
+                timestamp,
+            } => {
+                remote_dma::playback_done(id, destination, error, channel, timestamp).await;
+                None
+            }
+            Packet::SubkernelFinished { id, with_exception } => {
+                subkernel::subkernel_finished(id, with_exception).await;
+                None
+            }
+            Packet::SubkernelMessage {
+                id,
+                destination: from,
+                status,
+                length,
+                data,
+            } => {
+                subkernel::message_handle_incoming(id, status, length as usize, &data).await;
+                // acknowledge receiving part of the message
+                let _lock = aux_mutex.async_lock().await;
+                drtioaux_async::send(linkno, &Packet::SubkernelMessageAck { destination: from })
+                    .await
+                    .unwrap();
+                None
+            }
+            other => Some(other),
         }
     }
 
@@ -48,14 +90,18 @@ pub mod drtio {
         }
     }
 
-    pub async fn aux_transact(aux_mutex: &Mutex<bool>, linkno: u8, request: &Packet,
-            timer: GlobalTimer) -> Result<Packet, &'static str> {
+    pub async fn aux_transact(
+        aux_mutex: &Mutex<bool>,
+        linkno: u8,
+        request: &Packet,
+        timer: GlobalTimer,
+    ) -> Result<Packet, &'static str> {
         if !link_rx_up(linkno).await {
             return Err("link went down");
         }
         let _lock = aux_mutex.async_lock().await;
         drtioaux_async::send(linkno, request).await.unwrap();
-        recv_aux_timeout(linkno, 200, timer).await
+        Ok(recv_aux_timeout(linkno, 200, timer).await?)
     }
 
     async fn drain_buffer(linkno: u8, draining_time: Milliseconds, timer: GlobalTimer) {
@@ -72,7 +118,7 @@ pub mod drtio {
         let mut count = 0;
         loop {
             if !link_rx_up(linkno).await {
-                return 0
+                return 0;
             }
             count += 1;
             if count > 100 {
@@ -108,13 +154,23 @@ pub mod drtio {
         }
     }
 
-    async fn load_routing_table(aux_mutex: &Rc<Mutex<bool>>, linkno: u8, routing_table: &drtio_routing::RoutingTable,
-        timer: GlobalTimer) -> Result<(), &'static str> {
+    async fn load_routing_table(
+        aux_mutex: &Rc<Mutex<bool>>,
+        linkno: u8,
+        routing_table: &drtio_routing::RoutingTable,
+        timer: GlobalTimer,
+    ) -> Result<(), &'static str> {
         for i in 0..drtio_routing::DEST_COUNT {
-            let reply = aux_transact(aux_mutex, linkno, &Packet::RoutingSetPath {
-                destination: i as u8,
-                hops: routing_table.0[i]
-            }, timer).await?;
+            let reply = aux_transact(
+                aux_mutex,
+                linkno,
+                &Packet::RoutingSetPath {
+                    destination: i as u8,
+                    hops: routing_table.0[i],
+                },
+                timer,
+            )
+            .await?;
             if reply != Packet::RoutingAck {
                 return Err("unexpected reply");
             }
@@ -122,10 +178,13 @@ pub mod drtio {
         Ok(())
     }
 
-    async fn set_rank(aux_mutex: &Rc<Mutex<bool>>, linkno: u8, rank: u8, timer: GlobalTimer) -> Result<(), &'static str> {
-        let reply = aux_transact(aux_mutex, linkno, &Packet::RoutingSetRank {
-            rank: rank
-        }, timer).await?;
+    async fn set_rank(
+        aux_mutex: &Rc<Mutex<bool>>,
+        linkno: u8,
+        rank: u8,
+        timer: GlobalTimer,
+    ) -> Result<(), &'static str> {
+        let reply = aux_transact(aux_mutex, linkno, &Packet::RoutingSetRank { rank: rank }, timer).await?;
         if reply != Packet::RoutingAck {
             return Err("unexpected reply");
         }
@@ -139,8 +198,11 @@ pub mod drtio {
             (csr::DRTIO[linkno].force_destination_write)(1);
             (csr::DRTIO[linkno].o_get_buffer_space_write)(1);
             while (csr::DRTIO[linkno].o_wait_read)() == 1 {}
-            info!("[DEST#{}] buffer space is {}",
-                destination, (csr::DRTIO[linkno].o_dbg_buffer_space_read)());
+            info!(
+                "[DEST#{}] buffer space is {}",
+                destination,
+                (csr::DRTIO[linkno].o_dbg_buffer_space_read)()
+            );
             (csr::DRTIO[linkno].force_destination_write)(0);
         }
     }
@@ -148,9 +210,13 @@ pub mod drtio {
     async fn process_unsolicited_aux(aux_mutex: &Rc<Mutex<bool>>, linkno: u8) {
         let _lock = aux_mutex.async_lock().await;
         match drtioaux_async::recv(linkno).await {
-            Ok(Some(packet)) => warn!("[LINK#{}] unsolicited aux packet: {:?}", linkno, packet),
+            Ok(Some(packet)) => {
+                if let Some(packet) = process_async_packets(aux_mutex, linkno, packet).await {
+                    warn!("[LINK#{}] unsolicited aux packet: {:?}", linkno, packet);
+                }
+            }
             Ok(None) => (),
-            Err(_) => warn!("[LINK#{}] aux packet error", linkno)
+            Err(_) => warn!("[LINK#{}] aux packet error", linkno),
         }
     }
 
@@ -175,9 +241,12 @@ pub mod drtio {
         }
     }
 
-    async fn destination_set_up(routing_table: &drtio_routing::RoutingTable,
-            up_destinations: &Rc<RefCell<[bool; drtio_routing::DEST_COUNT]>>,
-            destination: u8, up: bool) {
+    async fn destination_set_up(
+        routing_table: &drtio_routing::RoutingTable,
+        up_destinations: &Rc<RefCell<[bool; drtio_routing::DEST_COUNT]>>,
+        destination: u8,
+        up: bool,
+    ) {
         let mut up_destinations = up_destinations.borrow_mut();
         up_destinations[destination as usize] = up;
         if up {
@@ -194,10 +263,13 @@ pub mod drtio {
         up_destinations[destination as usize]
     }
 
-    async fn destination_survey(aux_mutex: &Rc<Mutex<bool>>, routing_table: &drtio_routing::RoutingTable,
-            up_links: &[bool],
-            up_destinations: &Rc<RefCell<[bool; drtio_routing::DEST_COUNT]>>,
-            timer: GlobalTimer) {
+    async fn destination_survey(
+        aux_mutex: &Rc<Mutex<bool>>,
+        routing_table: &drtio_routing::RoutingTable,
+        up_links: &[bool],
+        up_destinations: &Rc<RefCell<[bool; drtio_routing::DEST_COUNT]>>,
+        timer: GlobalTimer,
+    ) {
         for destination in 0..drtio_routing::DEST_COUNT {
             let hop = routing_table.0[destination][0];
             let destination = destination as u8;
@@ -211,44 +283,96 @@ pub mod drtio {
                 let linkno = hop - 1;
                 if destination_up(up_destinations, destination).await {
                     if up_links[linkno as usize] {
-                        let reply = aux_transact(aux_mutex, linkno, &Packet::DestinationStatusRequest {
-                            destination: destination
-                        }, timer).await;
-                        match reply {
-                            Ok(Packet::DestinationDownReply) =>
-                                destination_set_up(routing_table, up_destinations, destination, false).await,
-                            Ok(Packet::DestinationOkReply) => (),
-                            Ok(Packet::DestinationSequenceErrorReply { channel }) =>{
-                                error!("[DEST#{}] RTIO sequence error involving channel 0x{:04x}", destination, channel);
-                                unsafe { SEEN_ASYNC_ERRORS |= ASYNC_ERROR_SEQUENCE_ERROR };
+                        loop {
+                            let reply = aux_transact(
+                                aux_mutex,
+                                linkno,
+                                &Packet::DestinationStatusRequest {
+                                    destination: destination,
+                                },
+                                timer,
+                            )
+                            .await;
+                            match reply {
+                                Ok(Packet::DestinationDownReply) => {
+                                    destination_set_up(routing_table, up_destinations, destination, false).await;
+                                    remote_dma::destination_changed(
+                                        aux_mutex,
+                                        routing_table,
+                                        timer,
+                                        destination,
+                                        false,
+                                    )
+                                    .await;
+                                    subkernel::destination_changed(aux_mutex, routing_table, timer, destination, false)
+                                        .await;
+                                }
+                                Ok(Packet::DestinationOkReply) => (),
+                                Ok(Packet::DestinationSequenceErrorReply { channel }) => {
+                                    error!(
+                                        "[DEST#{}] RTIO sequence error involving channel 0x{:04x}:{}",
+                                        destination,
+                                        channel,
+                                        resolve_channel_name(channel as u32)
+                                    );
+                                    unsafe { SEEN_ASYNC_ERRORS |= ASYNC_ERROR_SEQUENCE_ERROR };
+                                }
+                                Ok(Packet::DestinationCollisionReply { channel }) => {
+                                    error!(
+                                        "[DEST#{}] RTIO collision involving channel 0x{:04x}:{}",
+                                        destination,
+                                        channel,
+                                        resolve_channel_name(channel as u32)
+                                    );
+                                    unsafe { SEEN_ASYNC_ERRORS |= ASYNC_ERROR_COLLISION };
+                                }
+                                Ok(Packet::DestinationBusyReply { channel }) => {
+                                    error!(
+                                        "[DEST#{}] RTIO busy error involving channel 0x{:04x}:{}",
+                                        destination,
+                                        channel,
+                                        resolve_channel_name(channel as u32)
+                                    );
+                                    unsafe { SEEN_ASYNC_ERRORS |= ASYNC_ERROR_BUSY };
+                                }
+                                Ok(packet) => match process_async_packets(aux_mutex, linkno, packet).await {
+                                    Some(packet) => {
+                                        error!("[DEST#{}] received unexpected aux packet: {:?}", destination, packet)
+                                    }
+                                    None => continue,
+                                },
+                                Err(e) => error!("[DEST#{}] communication failed ({})", destination, e),
                             }
-                            Ok(Packet::DestinationCollisionReply { channel }) =>{
-                                error!("[DEST#{}] RTIO collision involving channel 0x{:04x}", destination, channel);
-                                unsafe { SEEN_ASYNC_ERRORS |= ASYNC_ERROR_COLLISION };
-                            }
-                            Ok(Packet::DestinationBusyReply { channel }) =>{
-                                error!("[DEST#{}] RTIO busy error involving channel 0x{:04x}", destination, channel);
-                                unsafe { SEEN_ASYNC_ERRORS |= ASYNC_ERROR_BUSY };
-                            }
-                            Ok(packet) => error!("[DEST#{}] received unexpected aux packet: {:?}", destination, packet),
-                            Err(e) => error!("[DEST#{}] communication failed ({})", destination, e)
+                            break;
                         }
                     } else {
                         destination_set_up(routing_table, up_destinations, destination, false).await;
+                        remote_dma::destination_changed(aux_mutex, routing_table, timer, destination, false).await;
+                        subkernel::destination_changed(aux_mutex, routing_table, timer, destination, false).await;
                     }
                 } else {
                     if up_links[linkno as usize] {
-                        let reply = aux_transact(aux_mutex, linkno, &Packet::DestinationStatusRequest {
-                            destination: destination
-                        }, timer).await;
+                        let reply = aux_transact(
+                            aux_mutex,
+                            linkno,
+                            &Packet::DestinationStatusRequest {
+                                destination: destination,
+                            },
+                            timer,
+                        )
+                        .await;
                         match reply {
                             Ok(Packet::DestinationDownReply) => (),
                             Ok(Packet::DestinationOkReply) => {
                                 destination_set_up(routing_table, up_destinations, destination, true).await;
                                 init_buffer_space(destination as u8, linkno).await;
-                            },
+                                remote_dma::destination_changed(aux_mutex, routing_table, timer, destination, true)
+                                    .await;
+                                subkernel::destination_changed(aux_mutex, routing_table, timer, destination, true)
+                                    .await;
+                            }
                             Ok(packet) => error!("[DEST#{}] received unexpected aux packet: {:?}", destination, packet),
-                            Err(e) => error!("[DEST#{}] communication failed ({})", destination, e)
+                            Err(e) => error!("[DEST#{}] communication failed ({})", destination, e),
                         }
                     }
                 }
@@ -256,10 +380,12 @@ pub mod drtio {
         }
     }
 
-    pub async fn link_task(aux_mutex: &Rc<Mutex<bool>>,
-            routing_table: &drtio_routing::RoutingTable,
-            up_destinations: &Rc<RefCell<[bool; drtio_routing::DEST_COUNT]>>,
-            timer: GlobalTimer) {
+    pub async fn link_task(
+        aux_mutex: &Rc<Mutex<bool>>,
+        routing_table: &drtio_routing::RoutingTable,
+        up_destinations: &Rc<RefCell<[bool; drtio_routing::DEST_COUNT]>>,
+        timer: GlobalTimer,
+    ) {
         let mut up_links = [false; csr::DRTIO.len()];
         loop {
             for linkno in 0..csr::DRTIO.len() {
@@ -320,33 +446,344 @@ pub mod drtio {
         for linkno in 0..csr::DRTIO.len() {
             let linkno = linkno as u8;
             if task::block_on(link_rx_up(linkno)) {
-                let reply = task::block_on(aux_transact(&aux_mutex, linkno,
-                    &Packet::ResetRequest, timer));
+                let reply = task::block_on(aux_transact(&aux_mutex, linkno, &Packet::ResetRequest, timer));
                 match reply {
                     Ok(Packet::ResetAck) => (),
                     Ok(_) => error!("[LINK#{}] reset failed, received unexpected aux packet", linkno),
-                    Err(e) => error!("[LINK#{}] reset failed, aux packet error ({})", linkno, e)
+                    Err(e) => error!("[LINK#{}] reset failed, aux packet error ({})", linkno, e),
                 }
             }
         }
     }
+
+    async fn partition_data<PacketF, HandlerF>(
+        linkno: u8,
+        aux_mutex: &Rc<Mutex<bool>>,
+        timer: GlobalTimer,
+        data: &[u8],
+        packet_f: PacketF,
+        reply_handler_f: HandlerF,
+    ) -> Result<(), &'static str>
+    where
+        PacketF: Fn(&[u8; MASTER_PAYLOAD_MAX_SIZE], PayloadStatus, usize) -> Packet,
+        HandlerF: Fn(&Packet) -> Result<(), &'static str>,
+    {
+        let mut i = 0;
+        while i < data.len() {
+            let mut slice: [u8; MASTER_PAYLOAD_MAX_SIZE] = [0; MASTER_PAYLOAD_MAX_SIZE];
+            let len: usize = if i + MASTER_PAYLOAD_MAX_SIZE < data.len() {
+                MASTER_PAYLOAD_MAX_SIZE
+            } else {
+                data.len() - i
+            } as usize;
+            let first = i == 0;
+            let last = i + len == data.len();
+            slice[..len].clone_from_slice(&data[i..i + len]);
+            i += len;
+            let status = PayloadStatus::from_status(first, last);
+            let packet = packet_f(&slice, status, len);
+            let reply = aux_transact(aux_mutex, linkno, &packet, timer).await?;
+            reply_handler_f(&reply)?;
+        }
+        Ok(())
+    }
+
+    pub async fn ddma_upload_trace(
+        aux_mutex: &Rc<Mutex<bool>>,
+        routing_table: &drtio_routing::RoutingTable,
+        timer: GlobalTimer,
+        id: u32,
+        destination: u8,
+        trace: &Vec<u8>,
+    ) -> Result<(), &'static str> {
+        let linkno = routing_table.0[destination as usize][0] - 1;
+        partition_data(
+            linkno,
+            aux_mutex,
+            timer,
+            trace,
+            |slice, status, len| Packet::DmaAddTraceRequest {
+                id: id,
+                destination: destination,
+                status: status,
+                length: len as u16,
+                trace: *slice,
+            },
+            |reply| match reply {
+                Packet::DmaAddTraceReply { succeeded: true } => Ok(()),
+                Packet::DmaAddTraceReply { succeeded: false } => Err("error adding trace on satellite"),
+                _ => Err("adding DMA trace failed, unexpected aux packet"),
+            },
+        )
+        .await
+    }
+
+    pub async fn ddma_send_erase(
+        aux_mutex: &Rc<Mutex<bool>>,
+        routing_table: &drtio_routing::RoutingTable,
+        timer: GlobalTimer,
+        id: u32,
+        destination: u8,
+    ) -> Result<(), &'static str> {
+        let linkno = routing_table.0[destination as usize][0] - 1;
+        let reply = aux_transact(
+            aux_mutex,
+            linkno,
+            &Packet::DmaRemoveTraceRequest {
+                id: id,
+                destination: destination,
+            },
+            timer,
+        )
+        .await;
+        match reply {
+            Ok(Packet::DmaRemoveTraceReply { succeeded: true }) => Ok(()),
+            Ok(Packet::DmaRemoveTraceReply { succeeded: false }) => Err("satellite DMA erase error"),
+            Ok(_) => Err("adding trace failed, unexpected aux packet"),
+            Err(_) => Err("erasing trace failed, aux error"),
+        }
+    }
+
+    pub async fn ddma_send_playback(
+        aux_mutex: &Rc<Mutex<bool>>,
+        routing_table: &drtio_routing::RoutingTable,
+        timer: GlobalTimer,
+        id: u32,
+        destination: u8,
+        timestamp: u64,
+    ) -> Result<(), &'static str> {
+        let linkno = routing_table.0[destination as usize][0] - 1;
+        let reply = aux_transact(
+            aux_mutex,
+            linkno,
+            &Packet::DmaPlaybackRequest {
+                id: id,
+                destination: destination,
+                timestamp: timestamp,
+            },
+            timer,
+        )
+        .await;
+        match reply {
+            Ok(Packet::DmaPlaybackReply { succeeded: true }) => Ok(()),
+            Ok(Packet::DmaPlaybackReply { succeeded: false }) => Err("error on DMA playback request"),
+            Ok(_) => Err("received unexpected aux packet during DMA playback"),
+            Err(_) => Err("aux error on DMA playback"),
+        }
+    }
+
+    async fn analyzer_get_data(
+        aux_mutex: &Rc<Mutex<bool>>,
+        routing_table: &drtio_routing::RoutingTable,
+        timer: GlobalTimer,
+        destination: u8,
+    ) -> Result<RemoteBuffer, &'static str> {
+        let linkno = routing_table.0[destination as usize][0] - 1;
+        let reply = aux_transact(
+            aux_mutex,
+            linkno,
+            &Packet::AnalyzerHeaderRequest {
+                destination: destination,
+            },
+            timer,
+        )
+        .await;
+        let (sent, total, overflow) = match reply {
+            Ok(Packet::AnalyzerHeader {
+                sent_bytes,
+                total_byte_count,
+                overflow_occurred,
+            }) => (sent_bytes, total_byte_count, overflow_occurred),
+            Ok(_) => return Err("received unexpected aux packet during remote analyzer header request"),
+            Err(e) => return Err(e),
+        };
+
+        let mut remote_data: Vec<u8> = Vec::new();
+        if sent > 0 {
+            let mut last_packet = false;
+            while !last_packet {
+                let reply = aux_transact(
+                    aux_mutex,
+                    linkno,
+                    &Packet::AnalyzerDataRequest {
+                        destination: destination,
+                    },
+                    timer,
+                )
+                .await;
+                match reply {
+                    Ok(Packet::AnalyzerData { last, length, data }) => {
+                        last_packet = last;
+                        remote_data.extend(&data[0..length as usize]);
+                    }
+                    Ok(_) => return Err("received unexpected aux packet during remote analyzer data request"),
+                    Err(e) => return Err(e),
+                }
+            }
+        }
+
+        Ok(RemoteBuffer {
+            sent_bytes: sent,
+            total_byte_count: total,
+            error: overflow,
+            data: remote_data,
+        })
+    }
+
+    pub async fn analyzer_query(
+        aux_mutex: &Rc<Mutex<bool>>,
+        routing_table: &drtio_routing::RoutingTable,
+        up_destinations: &Rc<RefCell<[bool; drtio_routing::DEST_COUNT]>>,
+        timer: GlobalTimer,
+    ) -> Result<Vec<RemoteBuffer>, &'static str> {
+        let mut remote_buffers: Vec<RemoteBuffer> = Vec::new();
+        for i in 1..drtio_routing::DEST_COUNT {
+            if destination_up(up_destinations, i as u8).await {
+                remote_buffers.push(analyzer_get_data(aux_mutex, routing_table, timer, i as u8).await?);
+            }
+        }
+        Ok(remote_buffers)
+    }
+
+    pub async fn subkernel_upload(
+        aux_mutex: &Rc<Mutex<bool>>,
+        routing_table: &drtio_routing::RoutingTable,
+        timer: GlobalTimer,
+        id: u32,
+        destination: u8,
+        data: &Vec<u8>,
+    ) -> Result<(), &'static str> {
+        let linkno = routing_table.0[destination as usize][0] - 1;
+        partition_data(
+            linkno,
+            aux_mutex,
+            timer,
+            data,
+            |slice, status, len| Packet::SubkernelAddDataRequest {
+                id: id,
+                destination: destination,
+                status: status,
+                length: len as u16,
+                data: *slice,
+            },
+            |reply| match reply {
+                Packet::SubkernelAddDataReply { succeeded: true } => Ok(()),
+                Packet::SubkernelAddDataReply { succeeded: false } => Err("error adding subkernel on satellite"),
+                _ => Err("adding subkernel failed, unexpected aux packet"),
+            },
+        )
+        .await
+    }
+
+    pub async fn subkernel_load(
+        aux_mutex: &Rc<Mutex<bool>>,
+        routing_table: &drtio_routing::RoutingTable,
+        timer: GlobalTimer,
+        id: u32,
+        destination: u8,
+        run: bool,
+    ) -> Result<(), &'static str> {
+        let linkno = routing_table.0[destination as usize][0] - 1;
+        let reply = aux_transact(
+            aux_mutex,
+            linkno,
+            &Packet::SubkernelLoadRunRequest {
+                id: id,
+                destination: destination,
+                run: run,
+            },
+            timer,
+        )
+        .await?;
+        match reply {
+            Packet::SubkernelLoadRunReply { succeeded: true } => return Ok(()),
+            Packet::SubkernelLoadRunReply { succeeded: false } => return Err("error on subkernel run request"),
+            _ => return Err("received unexpected aux packet during subkernel run"),
+        }
+    }
+
+    pub async fn subkernel_retrieve_exception(
+        aux_mutex: &Rc<Mutex<bool>>,
+        routing_table: &drtio_routing::RoutingTable,
+        timer: GlobalTimer,
+        destination: u8,
+    ) -> Result<Vec<u8>, &'static str> {
+        let linkno = routing_table.0[destination as usize][0] - 1;
+        let mut remote_data: Vec<u8> = Vec::new();
+        loop {
+            let reply = aux_transact(
+                aux_mutex,
+                linkno,
+                &Packet::SubkernelExceptionRequest {
+                    destination: destination,
+                },
+                timer,
+            )
+            .await?;
+            match reply {
+                Packet::SubkernelException { last, length, data } => {
+                    remote_data.extend(&data[0..length as usize]);
+                    if last {
+                        return Ok(remote_data);
+                    }
+                }
+                _ => return Err("received unexpected aux packet during subkernel exception request"),
+            }
+        }
+    }
+
+    pub async fn subkernel_send_message(
+        aux_mutex: &Rc<Mutex<bool>>,
+        routing_table: &drtio_routing::RoutingTable,
+        timer: GlobalTimer,
+        id: u32,
+        destination: u8,
+        message: &[u8],
+    ) -> Result<(), &'static str> {
+        let linkno = routing_table.0[destination as usize][0] - 1;
+        partition_data(
+            linkno,
+            aux_mutex,
+            timer,
+            message,
+            |slice, status, len| Packet::SubkernelMessage {
+                destination: destination,
+                id: id,
+                status: status,
+                length: len as u16,
+                data: *slice,
+            },
+            |reply| match reply {
+                Packet::SubkernelMessageAck { .. } => Ok(()),
+                _ => Err("sending message to subkernel failed, unexpected aux packet"),
+            },
+        )
+        .await
+    }
 }
 
 #[cfg(not(has_drtio))]
 pub mod drtio {
     use super::*;
 
-    pub fn startup(_aux_mutex: &Rc<Mutex<bool>>, _routing_table: &Rc<RefCell<drtio_routing::RoutingTable>>,
-        _up_destinations: &Rc<RefCell<[bool; drtio_routing::DEST_COUNT]>>, _timer: GlobalTimer) {}
+    pub fn startup(
+        _aux_mutex: &Rc<Mutex<bool>>,
+        _routing_table: &Rc<RefCell<drtio_routing::RoutingTable>>,
+        _up_destinations: &Rc<RefCell<[bool; drtio_routing::DEST_COUNT]>>,
+        _timer: GlobalTimer,
+    ) {
+    }
 
     #[allow(dead_code)]
     pub fn reset(_aux_mutex: Rc<Mutex<bool>>, mut _timer: GlobalTimer) {}
 }
 
-pub fn startup(aux_mutex: &Rc<Mutex<bool>>,
-        routing_table: &Rc<RefCell<drtio_routing::RoutingTable>>,
-        up_destinations: &Rc<RefCell<[bool; drtio_routing::DEST_COUNT]>>, 
-        timer: GlobalTimer) {
+pub fn startup(
+    aux_mutex: &Rc<Mutex<bool>>,
+    routing_table: &Rc<RefCell<drtio_routing::RoutingTable>>,
+    up_destinations: &Rc<RefCell<[bool; drtio_routing::DEST_COUNT]>>,
+    timer: GlobalTimer,
+) {
     drtio::startup(aux_mutex, routing_table, up_destinations, timer);
     unsafe {
         csr::rtio_core::reset_phy_write(1);

src/runtime/src/subkernel.rs

@@ -0,0 +1,314 @@
+use alloc::{collections::BTreeMap, rc::Rc, vec::Vec};
+
+use libasync::task;
+use libboard_artiq::{drtio_routing::RoutingTable,
+                     drtioaux_proto::{PayloadStatus, MASTER_PAYLOAD_MAX_SIZE}};
+use libboard_zynq::{time::Milliseconds, timer::GlobalTimer};
+use libcortex_a9::mutex::Mutex;
+use log::error;
+
+use crate::rtio_mgt::drtio;
+
+#[derive(Debug, PartialEq, Clone, Copy)]
+pub enum FinishStatus {
+    Ok,
+    CommLost,
+    Exception,
+}
+
+#[derive(Debug, PartialEq, Clone, Copy)]
+pub enum SubkernelState {
+    NotLoaded,
+    Uploaded,
+    Running,
+    Finished { status: FinishStatus },
+}
+
+#[derive(Debug, PartialEq, Eq, Clone, Copy)]
+pub enum Error {
+    Timeout,
+    IncorrectState,
+    SubkernelNotFound,
+    SubkernelException,
+    CommLost,
+    DrtioError(&'static str),
+}
+
+impl From<&'static str> for Error {
+    fn from(value: &'static str) -> Error {
+        Error::DrtioError(value)
+    }
+}
+
+pub struct SubkernelFinished {
+    pub id: u32,
+    pub status: FinishStatus,
+    pub exception: Option<Vec<u8>>,
+}
+
+struct Subkernel {
+    pub destination: u8,
+    pub data: Vec<u8>,
+    pub state: SubkernelState,
+}
+
+impl Subkernel {
+    pub fn new(destination: u8, data: Vec<u8>) -> Self {
+        Subkernel {
+            destination: destination,
+            data: data,
+            state: SubkernelState::NotLoaded,
+        }
+    }
+}
+
+static SUBKERNELS: Mutex<BTreeMap<u32, Subkernel>> = Mutex::new(BTreeMap::new());
+
+pub async fn add_subkernel(id: u32, destination: u8, kernel: Vec<u8>) {
+    SUBKERNELS
+        .async_lock()
+        .await
+        .insert(id, Subkernel::new(destination, kernel));
+}
+
+pub async fn upload(
+    aux_mutex: &Rc<Mutex<bool>>,
+    routing_table: &RoutingTable,
+    timer: GlobalTimer,
+    id: u32,
+) -> Result<(), Error> {
+    if let Some(subkernel) = SUBKERNELS.async_lock().await.get_mut(&id) {
+        drtio::subkernel_upload(
+            aux_mutex,
+            routing_table,
+            timer,
+            id,
+            subkernel.destination,
+            &subkernel.data,
+        )
+        .await?;
+        subkernel.state = SubkernelState::Uploaded;
+        Ok(())
+    } else {
+        Err(Error::SubkernelNotFound)
+    }
+}
+
+pub async fn load(
+    aux_mutex: &Rc<Mutex<bool>>,
+    routing_table: &RoutingTable,
+    timer: GlobalTimer,
+    id: u32,
+    run: bool,
+) -> Result<(), Error> {
+    if let Some(subkernel) = SUBKERNELS.async_lock().await.get_mut(&id) {
+        if subkernel.state != SubkernelState::Uploaded {
+            return Err(Error::IncorrectState);
+        }
+        drtio::subkernel_load(aux_mutex, routing_table, timer, id, subkernel.destination, run).await?;
+        if run {
+            subkernel.state = SubkernelState::Running;
+        }
+        Ok(())
+    } else {
+        Err(Error::SubkernelNotFound)
+    }
+}
+
+pub async fn clear_subkernels() {
+    SUBKERNELS.async_lock().await.clear();
+    MESSAGE_QUEUE.async_lock().await.clear();
+    CURRENT_MESSAGES.async_lock().await.clear();
+}
+
+pub async fn subkernel_finished(id: u32, with_exception: bool) {
+    // called upon receiving DRTIO SubkernelRunDone
+    // may be None if session ends and is cleared
+    if let Some(subkernel) = SUBKERNELS.async_lock().await.get_mut(&id) {
+        if subkernel.state == SubkernelState::Running {
+            subkernel.state = SubkernelState::Finished {
+                status: match with_exception {
+                    true => FinishStatus::Exception,
+                    false => FinishStatus::Ok,
+                },
+            }
+        }
+    }
+}
+
+pub async fn destination_changed(
+    aux_mutex: &Rc<Mutex<bool>>,
+    routing_table: &RoutingTable,
+    timer: GlobalTimer,
+    destination: u8,
+    up: bool,
+) {
+    let mut locked_subkernels = SUBKERNELS.async_lock().await;
+    for (id, subkernel) in locked_subkernels.iter_mut() {
+        if subkernel.destination == destination {
+            if up {
+                match drtio::subkernel_upload(aux_mutex, routing_table, timer, *id, destination, &subkernel.data).await
+                {
+                    Ok(_) => subkernel.state = SubkernelState::Uploaded,
+                    Err(e) => error!("Error adding subkernel on destination {}: {}", destination, e),
+                }
+            } else {
+                subkernel.state = match subkernel.state {
+                    SubkernelState::Running => SubkernelState::Finished {
+                        status: FinishStatus::CommLost,
+                    },
+                    _ => SubkernelState::NotLoaded,
+                }
+            }
+        }
+    }
+}
+
+pub async fn await_finish(
+    aux_mutex: &Rc<Mutex<bool>>,
+    routing_table: &RoutingTable,
+    timer: GlobalTimer,
+    id: u32,
+    timeout: u64,
+) -> Result<SubkernelFinished, Error> {
+    match SUBKERNELS.async_lock().await.get(&id).unwrap().state {
+        SubkernelState::Running | SubkernelState::Finished { .. } => (),
+        _ => return Err(Error::IncorrectState),
+    }
+    let max_time = timer.get_time() + Milliseconds(timeout);
+    while timer.get_time() < max_time {
+        {
+            match SUBKERNELS.async_lock().await.get(&id).unwrap().state {
+                SubkernelState::Finished { .. } => break,
+                _ => (),
+            };
+        }
+        task::r#yield().await;
+    }
+    if timer.get_time() >= max_time {
+        error!("Remote subkernel finish await timed out");
+        return Err(Error::Timeout);
+    }
+    if let Some(subkernel) = SUBKERNELS.async_lock().await.get_mut(&id) {
+        match subkernel.state {
+            SubkernelState::Finished { status } => {
+                subkernel.state = SubkernelState::Uploaded;
+                Ok(SubkernelFinished {
+                    id: id,
+                    status: status,
+                    exception: if status == FinishStatus::Exception {
+                        Some(
+                            drtio::subkernel_retrieve_exception(aux_mutex, routing_table, timer, subkernel.destination)
+                                .await?,
+                        )
+                    } else {
+                        None
+                    },
+                })
+            }
+            _ => Err(Error::IncorrectState),
+        }
+    } else {
+        Err(Error::SubkernelNotFound)
+    }
+}
+
+pub struct Message {
+    from_id: u32,
+    pub count: u8,
+    pub data: Vec<u8>,
+}
+
+// FIFO queue of messages
+static MESSAGE_QUEUE: Mutex<Vec<Message>> = Mutex::new(Vec::new());
+// currently under construction message(s) (can be from multiple sources)
+static CURRENT_MESSAGES: Mutex<BTreeMap<u32, Message>> = Mutex::new(BTreeMap::new());
+
+pub async fn message_handle_incoming(
+    id: u32,
+    status: PayloadStatus,
+    length: usize,
+    data: &[u8; MASTER_PAYLOAD_MAX_SIZE],
+) {
+    // called when receiving a message from satellite
+    {
+        let subkernel_lock = SUBKERNELS.async_lock().await;
+        let subkernel = subkernel_lock.get(&id);
+        if subkernel.is_none() || subkernel.unwrap().state != SubkernelState::Running {
+            // do not add messages for non-existing or deleted subkernels
+            return;
+        }
+    }
+    let mut current_messages = CURRENT_MESSAGES.async_lock().await;
+
+    if status.is_first() {
+        current_messages.remove(&id);
+    }
+
+    match current_messages.get_mut(&id) {
+        Some(message) => message.data.extend(&data[..length]),
+        None => {
+            current_messages.insert(
+                id,
+                Message {
+                    from_id: id,
+                    count: data[0],
+                    data: data[1..length].to_vec(),
+                },
+            );
+        }
+    };
+    if status.is_last() {
+        // when done, remove from working queue
+        MESSAGE_QUEUE
+            .async_lock()
+            .await
+            .push(current_messages.remove(&id).unwrap());
+    }
+}
+
+pub async fn message_await(id: u32, timeout: u64, timer: GlobalTimer) -> Result<Message, Error> {
+    match SUBKERNELS.async_lock().await.get(&id).unwrap().state {
+        SubkernelState::Finished {
+            status: FinishStatus::CommLost,
+        } => return Err(Error::CommLost),
+        SubkernelState::Running | SubkernelState::Finished { .. } => (),
+        _ => return Err(Error::IncorrectState),
+    }
+    let max_time = timer.get_time() + Milliseconds(timeout);
+    while timer.get_time() < max_time {
+        {
+            let mut message_queue = MESSAGE_QUEUE.async_lock().await;
+            for i in 0..message_queue.len() {
+                let msg = &message_queue[i];
+                if msg.from_id == id {
+                    let message = message_queue.remove(i);
+                    return Ok(message);
+                }
+            }
+        }
+        match SUBKERNELS.async_lock().await.get(&id).unwrap().state {
+            SubkernelState::Finished {
+                status: FinishStatus::CommLost,
+            } => return Err(Error::CommLost),
+            SubkernelState::Finished {
+                status: FinishStatus::Exception,
+            } => return Err(Error::SubkernelException),
+            _ => (),
+        }
+        task::r#yield().await;
+    }
+    Err(Error::Timeout)
+}
+
+pub async fn message_send<'a>(
+    aux_mutex: &Rc<Mutex<bool>>,
+    routing_table: &RoutingTable,
+    timer: GlobalTimer,
+    id: u32,
+    message: Vec<u8>,
+) -> Result<(), Error> {
+    let destination = SUBKERNELS.async_lock().await.get(&id).unwrap().destination;
+    // rpc data prepared by the kernel core already
+    Ok(drtio::subkernel_send_message(aux_mutex, routing_table, timer, id, destination, &message).await?)
+}

src/rustfmt.toml

@@ -0,0 +1,68 @@
+max_width = 120
+hard_tabs = false
+tab_spaces = 4
+newline_style = "Auto"
+use_small_heuristics = "Default"
+indent_style = "Block"
+wrap_comments = false
+format_code_in_doc_comments = false
+comment_width = 100
+normalize_comments = false
+normalize_doc_attributes = false
+license_template_path = ""
+format_strings = true
+format_macro_matchers = true
+format_macro_bodies = true
+empty_item_single_line = true
+struct_lit_single_line = true
+fn_single_line = false
+where_single_line = true
+imports_indent = "Visual"
+imports_layout = "Mixed"
+merge_imports = true
+group_imports = "StdExternalCrate"
+reorder_imports = true
+reorder_modules = true
+reorder_impl_items = false
+type_punctuation_density = "Wide"
+space_before_colon = false
+space_after_colon = true
+spaces_around_ranges = false
+binop_separator = "Front"
+remove_nested_parens = true
+combine_control_expr = true
+overflow_delimited_expr = false
+struct_field_align_threshold = 0
+enum_discrim_align_threshold = 0
+match_arm_blocks = true
+match_arm_leading_pipes = "Never"
+force_multiline_blocks = false
+fn_args_layout = "Tall"
+brace_style = "SameLineWhere"
+control_brace_style = "AlwaysSameLine"
+trailing_semicolon = true
+trailing_comma = "Vertical"
+match_block_trailing_comma = false
+blank_lines_upper_bound = 1
+blank_lines_lower_bound = 0
+edition = "2018"
+version = "Two"
+inline_attribute_width = 0
+merge_derives = true
+use_try_shorthand = false
+use_field_init_shorthand = false
+force_explicit_abi = true
+condense_wildcard_suffixes = false
+color = "Auto"
+required_version = "1.4.32"
+unstable_features = false
+disable_all_formatting = false
+skip_children = false
+hide_parse_errors = false
+error_on_line_overflow = false
+error_on_unformatted = false
+report_todo = "Never"
+report_fixme = "Never"
+ignore = []
+emit_mode = "Files"
+make_backup = false

src/satman/Cargo.toml.tpl

@@ -14,15 +14,19 @@ build_zynq = { path = "../libbuild_zynq" }
 
 [dependencies]
 log = { version = "0.4", default-features = false }
+core_io = { version = "0.1", features = ["collections"] }
+cslice = "0.3"
 embedded-hal = "0.2"
 
-libboard_zynq = { git = "https://git.m-labs.hk/M-Labs/zynq-rs.git", features = ["ipv6"]}
-libsupport_zynq = { default-features = false, features = ["alloc_core"], git = "https://git.m-labs.hk/M-Labs/zynq-rs.git" }
-libcortex_a9 = { git = "https://git.m-labs.hk/M-Labs/zynq-rs.git" }
-libasync = { git = "https://git.m-labs.hk/M-Labs/zynq-rs.git" }
-libregister = { git = "https://git.m-labs.hk/M-Labs/zynq-rs.git" }
-libconfig = { git = "https://git.m-labs.hk/M-Labs/zynq-rs.git", features = ["fat_lfn", "ipv6"] }
+libboard_zynq = { path = "@@ZYNQ_RS@@/libboard_zynq", features = ["ipv6"]}
+libsupport_zynq = { path = "@@ZYNQ_RS@@/libsupport_zynq", default-features = false, features = ["alloc_core"] }
+libcortex_a9 = { path = "@@ZYNQ_RS@@/libcortex_a9" }
+libasync = { path = "@@ZYNQ_RS@@/libasync" }
+libregister = { path = "@@ZYNQ_RS@@/libregister" }
+libconfig = { path = "@@ZYNQ_RS@@/libconfig", features = ["fat_lfn", "ipv6"] }
 
 libboard_artiq = { path = "../libboard_artiq" }
 unwind = { path = "../libunwind" }
 libc = { path = "../libc" }
+io = { path = "../libio", features = ["alloc"] }
+ksupport = { path = "../libksupport" }

src/satman/src/analyzer.rs

@@ -0,0 +1,126 @@
+use core::cmp::min;
+
+use libboard_artiq::{drtioaux_proto::SAT_PAYLOAD_MAX_SIZE, pl::csr};
+use libcortex_a9::cache;
+
+const BUFFER_SIZE: usize = 512 * 1024;
+
+#[repr(align(64))]
+struct Buffer {
+    data: [u8; BUFFER_SIZE],
+}
+
+static mut BUFFER: Buffer = Buffer { data: [0; BUFFER_SIZE] };
+
+fn arm() {
+    unsafe {
+        let base_addr = &mut BUFFER.data[0] as *mut _ as usize;
+        let last_addr = &mut BUFFER.data[BUFFER_SIZE - 1] as *mut _ as usize;
+        csr::rtio_analyzer::dma_base_address_write(base_addr as u32);
+        csr::rtio_analyzer::message_encoder_overflow_reset_write(1);
+        csr::rtio_analyzer::dma_last_address_write(last_addr as u32);
+        csr::rtio_analyzer::dma_reset_write(1);
+        csr::rtio_analyzer::enable_write(1);
+    }
+}
+
+fn disarm() {
+    unsafe {
+        csr::rtio_analyzer::enable_write(0);
+        while csr::rtio_analyzer::busy_read() != 0 {}
+        cache::dcci_slice(&BUFFER.data);
+    }
+}
+
+pub struct Analyzer {
+    // necessary for keeping track of sent data
+    data_len: usize,
+    sent_bytes: usize,
+    data_pointer: usize,
+}
+
+pub struct Header {
+    pub total_byte_count: u64,
+    pub sent_bytes: u32,
+    pub error: bool,
+}
+
+pub struct AnalyzerSliceMeta {
+    pub len: u16,
+    pub last: bool,
+}
+
+impl Drop for Analyzer {
+    fn drop(&mut self) {
+        disarm();
+    }
+}
+
+impl Analyzer {
+    pub fn new() -> Analyzer {
+        // create and arm new Analyzer
+        arm();
+        Analyzer {
+            data_len: 0,
+            sent_bytes: 0,
+            data_pointer: 0,
+        }
+    }
+
+    pub fn get_header(&mut self) -> Header {
+        disarm();
+
+        let overflow = unsafe { csr::rtio_analyzer::message_encoder_overflow_read() != 0 };
+        let bus_err = unsafe { csr::rtio_analyzer::dma_bus_error_read() != 0 };
+        let total_byte_count = unsafe { csr::rtio_analyzer::dma_byte_count_read() as u64 };
+        let wraparound = total_byte_count >= BUFFER_SIZE as u64;
+        self.data_len = if wraparound {
+            BUFFER_SIZE
+        } else {
+            total_byte_count as usize
+        };
+        self.data_pointer = if wraparound {
+            (total_byte_count % BUFFER_SIZE as u64) as usize
+        } else {
+            0
+        };
+        self.sent_bytes = 0;
+
+        if overflow {
+            warn!("overflow occured");
+        }
+        if bus_err {
+            warn!("bus error occured");
+        }
+
+        Header {
+            total_byte_count: total_byte_count,
+            sent_bytes: self.data_len as u32,
+            error: overflow | bus_err,
+        }
+    }
+
+    pub fn get_data(&mut self, data_slice: &mut [u8; SAT_PAYLOAD_MAX_SIZE]) -> AnalyzerSliceMeta {
+        let data = unsafe { &BUFFER.data[..] };
+        let i = (self.data_pointer + self.sent_bytes) % BUFFER_SIZE;
+        let len = min(SAT_PAYLOAD_MAX_SIZE, self.data_len - self.sent_bytes);
+        let last = self.sent_bytes + len == self.data_len;
+
+        if i + len >= BUFFER_SIZE {
+            data_slice[..(BUFFER_SIZE - i)].clone_from_slice(&data[i..BUFFER_SIZE]);
+            data_slice[(BUFFER_SIZE - i)..len].clone_from_slice(&data[..(i + len) % BUFFER_SIZE]);
+        } else {
+            data_slice[..len].clone_from_slice(&data[i..i + len]);
+        }
+        self.sent_bytes += len;
+
+        if last {
+            arm();
+        }
+
+        AnalyzerSliceMeta {
+            len: len as u16,
+            last: last,
+        }
+    }
+}

src/satman/src/dma.rs

@@ -0,0 +1,177 @@
+use alloc::{collections::btree_map::BTreeMap, vec::Vec};
+
+use libboard_artiq::{drtioaux_proto::PayloadStatus, pl::csr};
+use libcortex_a9::cache::dcci_slice;
+
+const ALIGNMENT: usize = 64;
+
+#[derive(Debug, PartialEq)]
+enum ManagerState {
+    Idle,
+    Playback,
+}
+
+pub struct RtioStatus {
+    pub id: u32,
+    pub error: u8,
+    pub channel: u32,
+    pub timestamp: u64,
+}
+
+pub enum Error {
+    IdNotFound,
+    PlaybackInProgress,
+    EntryNotComplete,
+}
+
+#[derive(Debug)]
+struct Entry {
+    trace: Vec<u8>,
+    padding_len: usize,
+    complete: bool,
+}
+
+#[derive(Debug)]
+pub struct Manager {
+    entries: BTreeMap<u32, Entry>,
+    state: ManagerState,
+    currentid: u32,
+}
+
+impl Manager {
+    pub fn new() -> Manager {
+        // in case Manager is created during a DMA in progress
+        // wait for it to end
+        unsafe { while csr::rtio_dma::enable_read() != 0 {} }
+        Manager {
+            entries: BTreeMap::new(),
+            currentid: 0,
+            state: ManagerState::Idle,
+        }
+    }
+
+    pub fn add(&mut self, id: u32, status: PayloadStatus, trace: &[u8], trace_len: usize) -> Result<(), Error> {
+        let entry = match self.entries.get_mut(&id) {
+            Some(entry) => {
+                if entry.complete || status.is_first() {
+                    // replace entry
+                    self.entries.remove(&id);
+                    self.entries.insert(
+                        id,
+                        Entry {
+                            trace: Vec::new(),
+                            padding_len: 0,
+                            complete: false,
+                        },
+                    );
+                    self.entries.get_mut(&id).unwrap()
+                } else {
+                    entry
+                }
+            }
+            None => {
+                self.entries.insert(
+                    id,
+                    Entry {
+                        trace: Vec::new(),
+                        padding_len: 0,
+                        complete: false,
+                    },
+                );
+                self.entries.get_mut(&id).unwrap()
+            }
+        };
+        entry.trace.extend(&trace[0..trace_len]);
+
+        if status.is_last() {
+            entry.trace.push(0);
+            let data_len = entry.trace.len();
+
+            // Realign.
+            entry.trace.reserve(ALIGNMENT - 1);
+            let padding = ALIGNMENT - entry.trace.as_ptr() as usize % ALIGNMENT;
+            let padding = if padding == ALIGNMENT { 0 } else { padding };
+            for _ in 0..padding {
+                // Vec guarantees that this will not reallocate
+                entry.trace.push(0)
+            }
+            for i in 1..data_len + 1 {
+                entry.trace[data_len + padding - i] = entry.trace[data_len - i]
+            }
+            entry.complete = true;
+            entry.padding_len = padding;
+            dcci_slice(&entry.trace);
+        }
+        Ok(())
+    }
+
+    pub fn erase(&mut self, id: u32) -> Result<(), Error> {
+        match self.entries.remove(&id) {
+            Some(_) => Ok(()),
+            None => Err(Error::IdNotFound),
+        }
+    }
+
+    pub fn playback(&mut self, id: u32, timestamp: u64) -> Result<(), Error> {
+        if self.state != ManagerState::Idle {
+            return Err(Error::PlaybackInProgress);
+        }
+
+        let entry = match self.entries.get(&id) {
+            Some(entry) => entry,
+            None => {
+                return Err(Error::IdNotFound);
+            }
+        };
+        if !entry.complete {
+            return Err(Error::EntryNotComplete);
+        }
+        let ptr = entry.trace[entry.padding_len..].as_ptr();
+        assert!(ptr as u32 % 64 == 0);
+
+        self.state = ManagerState::Playback;
+        self.currentid = id;
+
+        unsafe {
+            csr::rtio_dma::base_address_write(ptr as u32);
+            csr::rtio_dma::time_offset_write(timestamp as u64);
+
+            csr::cri_con::selected_write(1);
+            csr::rtio_dma::enable_write(1);
+            // playback has begun here, for status call check_state
+        }
+        Ok(())
+    }
+
+    pub fn check_state(&mut self) -> Option<RtioStatus> {
+        if self.state != ManagerState::Playback {
+            // nothing to report
+            return None;
+        }
+        let dma_enable = unsafe { csr::rtio_dma::enable_read() };
+        if dma_enable != 0 {
+            return None;
+        } else {
+            self.state = ManagerState::Idle;
+            unsafe {
+                csr::cri_con::selected_write(0);
+                let error = csr::rtio_dma::error_read();
+                let channel = csr::rtio_dma::error_channel_read();
+                let timestamp = csr::rtio_dma::error_timestamp_read();
+                if error != 0 {
+                    csr::rtio_dma::error_write(1);
+                }
+                return Some(RtioStatus {
+                    id: self.currentid,
+                    error: error,
+                    channel: channel,
+                    timestamp: timestamp,
+                });
+            }
+        }
+    }
+
+    pub fn running(&self) -> bool {
+        self.state == ManagerState::Playback
+    }
+}

src/satman/src/repeater.rs

@@ -1,19 +1,16 @@
-use libboard_artiq::{drtioaux, drtio_routing};
-use libboard_zynq::timer::GlobalTimer;
-
-#[cfg(has_drtio_routing)]
-use libboard_artiq::{pl::csr};
-#[cfg(has_drtio_routing)]
-use libboard_zynq::time::Milliseconds;
 #[cfg(has_drtio_routing)]
 use embedded_hal::prelude::_embedded_hal_blocking_delay_DelayUs;
+#[cfg(has_drtio_routing)]
+use libboard_artiq::pl::csr;
+use libboard_artiq::{drtio_routing, drtioaux};
+#[cfg(has_drtio_routing)]
+use libboard_zynq::time::Milliseconds;
+use libboard_zynq::timer::GlobalTimer;
 
 #[cfg(has_drtio_routing)]
 fn rep_link_rx_up(repno: u8) -> bool {
     let repno = repno as usize;
-    unsafe {
-        (csr::DRTIOREP[repno].rx_up_read)() == 1
-    }
+    unsafe { (csr::DRTIOREP[repno].rx_up_read)() == 1 }
 }
 
 #[cfg(has_drtio_routing)]
@@ -23,12 +20,14 @@ enum RepeaterState {
     SendPing { ping_count: u16 },
     WaitPingReply { ping_count: u16, timeout: Milliseconds },
     Up,
-    Failed
+    Failed,
 }
 
 #[cfg(has_drtio_routing)]
 impl Default for RepeaterState {
-    fn default() -> RepeaterState { RepeaterState::Down }
+    fn default() -> RepeaterState {
+        RepeaterState::Down
+    }
 }
 
 #[cfg(has_drtio_routing)]
@@ -36,7 +35,7 @@ impl Default for RepeaterState {
 pub struct Repeater {
     repno: u8,
     auxno: u8,
-    state: RepeaterState
+    state: RepeaterState,
 }
 
 #[cfg(has_drtio_routing)]
@@ -45,7 +44,7 @@ impl Repeater {
         Repeater {
             repno: repno,
             auxno: repno + 1,
-            state: RepeaterState::Down
+            state: RepeaterState::Down,
         }
     }
 
@@ -54,8 +53,7 @@ impl Repeater {
         self.state == RepeaterState::Up
     }
 
-    pub fn service(&mut self, routing_table: &drtio_routing::RoutingTable, rank: u8,
-        timer: &mut GlobalTimer) {
+    pub fn service(&mut self, routing_table: &drtio_routing::RoutingTable, rank: u8, timer: &mut GlobalTimer) {
         self.process_local_errors();
 
         match self.state {
@@ -70,7 +68,7 @@ impl Repeater {
                     drtioaux::send(self.auxno, &drtioaux::Packet::EchoRequest).unwrap();
                     self.state = RepeaterState::WaitPingReply {
                         ping_count: ping_count + 1,
-                        timeout: timer.get_time() + Milliseconds(100)
+                        timeout: timer.get_time() + Milliseconds(100),
                     }
                 } else {
                     error!("[REP#{}] link RX went down during ping", self.repno);
@@ -132,7 +130,7 @@ impl Repeater {
         match drtioaux::recv(self.auxno) {
             Ok(Some(packet)) => warn!("[REP#{}] unsolicited aux packet: {:?}", self.repno, packet),
             Ok(None) => (),
-            Err(_) => warn!("[REP#{}] aux packet error", self.repno)
+            Err(_) => warn!("[REP#{}] aux packet error", self.repno),
         }
     }
 
@@ -155,14 +153,20 @@ impl Repeater {
                 cmd = (csr::DRTIOREP[repno].command_missed_cmd_read)();
                 chan_sel = (csr::DRTIOREP[repno].command_missed_chan_sel_read)();
             }
-            error!("[REP#{}] CRI command missed, cmd={}, chan_sel=0x{:06x}", repno, cmd, chan_sel)
+            error!(
+                "[REP#{}] CRI command missed, cmd={}, chan_sel=0x{:06x}",
+                repno, cmd, chan_sel
+            )
         }
         if errors & 8 != 0 {
             let destination;
             unsafe {
                 destination = (csr::DRTIOREP[repno].buffer_space_timeout_dest_read)();
             }
-            error!("[REP#{}] timeout attempting to get remote buffer space, destination=0x{:02x}", repno, destination);
+            error!(
+                "[REP#{}] timeout attempting to get remote buffer space, destination=0x{:02x}",
+                repno, destination
+            );
         }
         unsafe {
             (csr::DRTIOREP[repno].protocol_error_write)(errors);
@@ -181,7 +185,7 @@ impl Repeater {
             match drtioaux::recv(self.auxno) {
                 Ok(Some(packet)) => return Ok(packet),
                 Ok(None) => (),
-                Err(e) => return Err(e)
+                Err(e) => return Err(e),
             }
         }
     }
@@ -217,15 +221,24 @@ impl Repeater {
         }
     }
 
-    pub fn set_path(&self, destination: u8, hops: &[u8; drtio_routing::MAX_HOPS], timer: &mut GlobalTimer) -> Result<(), drtioaux::Error> {
+    pub fn set_path(
+        &self,
+        destination: u8,
+        hops: &[u8; drtio_routing::MAX_HOPS],
+        timer: &mut GlobalTimer,
+    ) -> Result<(), drtioaux::Error> {
         if self.state != RepeaterState::Up {
             return Ok(());
         }
 
-        drtioaux::send(self.auxno, &drtioaux::Packet::RoutingSetPath {
-            destination: destination,
-            hops: *hops
-        }).unwrap();
+        drtioaux::send(
+            self.auxno,
+            &drtioaux::Packet::RoutingSetPath {
+                destination: destination,
+                hops: *hops,
+            },
+        )
+        .unwrap();
         let reply = self.recv_aux_timeout(200, timer)?;
         if reply != drtioaux::Packet::RoutingAck {
             return Err(drtioaux::Error::UnexpectedReply);
@@ -233,7 +246,11 @@ impl Repeater {
         Ok(())
     }
 
-    pub fn load_routing_table(&self, routing_table: &drtio_routing::RoutingTable, timer: &mut GlobalTimer) -> Result<(), drtioaux::Error> {
+    pub fn load_routing_table(
+        &self,
+        routing_table: &drtio_routing::RoutingTable,
+        timer: &mut GlobalTimer,
+    ) -> Result<(), drtioaux::Error> {
         for i in 0..drtio_routing::DEST_COUNT {
             self.set_path(i as u8, &routing_table.0[i], timer)?;
         }
@@ -244,9 +261,7 @@ impl Repeater {
         if self.state != RepeaterState::Up {
             return Ok(());
         }
-        drtioaux::send(self.auxno, &drtioaux::Packet::RoutingSetRank {
-            rank: rank
-        }).unwrap();
+        drtioaux::send(self.auxno, &drtioaux::Packet::RoutingSetRank { rank: rank }).unwrap();
         let reply = self.recv_aux_timeout(200, timer)?;
         if reply != drtioaux::Packet::RoutingAck {
             return Err(drtioaux::Error::UnexpectedReply);
@@ -256,9 +271,13 @@ impl Repeater {
 
     pub fn rtio_reset(&self, timer: &mut GlobalTimer) -> Result<(), drtioaux::Error> {
         let repno = self.repno as usize;
-        unsafe { (csr::DRTIOREP[repno].reset_write)(1); }
+        unsafe {
+            (csr::DRTIOREP[repno].reset_write)(1);
+        }
         timer.delay_us(100);
-        unsafe { (csr::DRTIOREP[repno].reset_write)(0); }
+        unsafe {
+            (csr::DRTIOREP[repno].reset_write)(0);
+        }
 
         if self.state != RepeaterState::Up {
             return Ok(());
@@ -275,16 +294,21 @@ impl Repeater {
 
 #[cfg(not(has_drtio_routing))]
 #[derive(Clone, Copy, Default)]
-pub struct Repeater {
-}
+pub struct Repeater {}
 
 #[cfg(not(has_drtio_routing))]
 impl Repeater {
-    pub fn new(_repno: u8) -> Repeater { Repeater::default() }
+    pub fn new(_repno: u8) -> Repeater {
+        Repeater::default()
+    }
 
-    pub fn service(&self, _routing_table: &drtio_routing::RoutingTable, _rank: u8, _timer: &mut GlobalTimer) { }
+    pub fn service(&self, _routing_table: &drtio_routing::RoutingTable, _rank: u8, _timer: &mut GlobalTimer) {}
 
-    pub fn sync_tsc(&self, _timer: &mut GlobalTimer) -> Result<(), drtioaux::Error> { Ok(()) }
+    pub fn sync_tsc(&self, _timer: &mut GlobalTimer) -> Result<(), drtioaux::Error> {
+        Ok(())
+    }
 
-    pub fn rtio_reset(&self, _timer: &mut GlobalTimer) -> Result<(), drtioaux::Error> { Ok(()) }
+    pub fn rtio_reset(&self, _timer: &mut GlobalTimer) -> Result<(), drtioaux::Error> {
+        Ok(())
+    }
 }

src/satman/src/subkernel.rs

@@ -0,0 +1,700 @@
+use alloc::{collections::{BTreeMap, VecDeque},
+            format,
+            string::{String, ToString},
+            vec::Vec};
+use core::{cmp::min, option::NoneError, slice, str};
+
+use core_io::{Error as IoError, Write};
+use cslice::AsCSlice;
+use io::{Cursor, ProtoWrite};
+use ksupport::{eh_artiq, kernel, rpc};
+use libboard_artiq::{drtioaux_proto::{PayloadStatus, MASTER_PAYLOAD_MAX_SIZE, SAT_PAYLOAD_MAX_SIZE},
+                     pl::csr};
+use libboard_zynq::{time::Milliseconds, timer::GlobalTimer};
+use libcortex_a9::sync_channel::Receiver;
+use log::warn;
+
+#[derive(Debug, Clone, PartialEq)]
+enum KernelState {
+    Absent,
+    Loaded,
+    Running,
+    MsgAwait(Milliseconds, Vec<u8>),
+    MsgSending,
+}
+
+#[derive(Debug)]
+pub enum Error {
+    Load(String),
+    KernelNotFound,
+    Unexpected(String),
+    NoMessage,
+    AwaitingMessage,
+    SubkernelIoError,
+    KernelException(Sliceable),
+}
+
+impl From<NoneError> for Error {
+    fn from(_: NoneError) -> Error {
+        Error::KernelNotFound
+    }
+}
+
+impl From<IoError> for Error {
+    fn from(_value: IoError) -> Error {
+        Error::SubkernelIoError
+    }
+}
+
+impl From<()> for Error {
+    fn from(_: ()) -> Error {
+        Error::NoMessage
+    }
+}
+
+macro_rules! unexpected {
+    ($($arg:tt)*) => (return Err(Error::Unexpected(format!($($arg)*))));
+}
+
+/* represents data that has to be sent to Master */
+#[derive(Debug)]
+pub struct Sliceable {
+    it: usize,
+    data: Vec<u8>,
+}
+
+/* represents interkernel messages */
+struct Message {
+    count: u8,
+    data: Vec<u8>,
+}
+
+#[derive(PartialEq)]
+enum OutMessageState {
+    NoMessage,
+    MessageReady,
+    MessageBeingSent,
+    MessageSent,
+    MessageAcknowledged,
+}
+
+/* for dealing with incoming and outgoing interkernel messages */
+struct MessageManager {
+    out_message: Option<Sliceable>,
+    out_state: OutMessageState,
+    in_queue: VecDeque<Message>,
+    in_buffer: Option<Message>,
+}
+
+// Per-run state
+struct Session {
+    id: u32,
+    kernel_state: KernelState,
+    last_exception: Option<Sliceable>,
+    messages: MessageManager,
+}
+
+impl Session {
+    pub fn new(id: u32) -> Session {
+        Session {
+            id: id,
+            kernel_state: KernelState::Absent,
+            last_exception: None,
+            messages: MessageManager::new(),
+        }
+    }
+
+    fn running(&self) -> bool {
+        match self.kernel_state {
+            KernelState::Absent | KernelState::Loaded => false,
+            KernelState::Running | KernelState::MsgAwait { .. } | KernelState::MsgSending => true,
+        }
+    }
+}
+
+#[derive(Debug)]
+struct KernelLibrary {
+    library: Vec<u8>,
+    complete: bool,
+}
+
+pub struct Manager<'a> {
+    kernels: BTreeMap<u32, KernelLibrary>,
+    session: Session,
+    control: &'a mut kernel::Control,
+    cache: BTreeMap<String, Vec<i32>>,
+    last_finished: Option<SubkernelFinished>,
+}
+
+pub struct SubkernelFinished {
+    pub id: u32,
+    pub with_exception: bool,
+}
+
+pub struct SliceMeta {
+    pub len: u16,
+    pub status: PayloadStatus,
+}
+
+macro_rules! get_slice_fn {
+    ($name:tt, $size:expr) => {
+        pub fn $name(&mut self, data_slice: &mut [u8; $size]) -> SliceMeta {
+            let first = self.it == 0;
+            let len = min($size, self.data.len() - self.it);
+            let last = self.it + len == self.data.len();
+            let status = PayloadStatus::from_status(first, last);
+
+            data_slice[..len].clone_from_slice(&self.data[self.it..self.it + len]);
+            self.it += len;
+
+            SliceMeta {
+                len: len as u16,
+                status: status,
+            }
+        }
+    };
+}
+
+impl Sliceable {
+    pub fn new(data: Vec<u8>) -> Sliceable {
+        Sliceable { it: 0, data: data }
+    }
+
+    get_slice_fn!(get_slice_sat, SAT_PAYLOAD_MAX_SIZE);
+    get_slice_fn!(get_slice_master, MASTER_PAYLOAD_MAX_SIZE);
+}
+
+impl MessageManager {
+    pub fn new() -> MessageManager {
+        MessageManager {
+            out_message: None,
+            out_state: OutMessageState::NoMessage,
+            in_queue: VecDeque::new(),
+            in_buffer: None,
+        }
+    }
+
+    pub fn handle_incoming(&mut self, status: PayloadStatus, length: usize, data: &[u8; MASTER_PAYLOAD_MAX_SIZE]) {
+        // called when receiving a message from master
+        if status.is_first() {
+            self.in_buffer = None;
+        }
+        match self.in_buffer.as_mut() {
+            Some(message) => message.data.extend(&data[..length]),
+            None => {
+                self.in_buffer = Some(Message {
+                    count: data[0],
+                    data: data[1..length].to_vec(),
+                });
+            }
+        };
+        if status.is_last() {
+            // when done, remove from working queue
+            self.in_queue.push_back(self.in_buffer.take().unwrap());
+        }
+    }
+
+    pub fn is_outgoing_ready(&mut self) -> bool {
+        // called by main loop, to see if there's anything to send, will send it afterwards
+        match self.out_state {
+            OutMessageState::MessageReady => {
+                self.out_state = OutMessageState::MessageBeingSent;
+                true
+            }
+            _ => false,
+        }
+    }
+
+    pub fn was_message_acknowledged(&mut self) -> bool {
+        match self.out_state {
+            OutMessageState::MessageAcknowledged => {
+                self.out_state = OutMessageState::NoMessage;
+                true
+            }
+            _ => false,
+        }
+    }
+
+    pub fn get_outgoing_slice(&mut self, data_slice: &mut [u8; MASTER_PAYLOAD_MAX_SIZE]) -> Option<SliceMeta> {
+        if self.out_state != OutMessageState::MessageBeingSent {
+            return None;
+        }
+        let meta = self.out_message.as_mut()?.get_slice_master(data_slice);
+        if meta.status.is_last() {
+            // clear the message slot
+            self.out_message = None;
+            // notify kernel with a flag that message is sent
+            self.out_state = OutMessageState::MessageSent;
+        }
+        Some(meta)
+    }
+
+    pub fn ack_slice(&mut self) -> bool {
+        // returns whether or not there's more to be sent
+        match self.out_state {
+            OutMessageState::MessageBeingSent => true,
+            OutMessageState::MessageSent => {
+                self.out_state = OutMessageState::MessageAcknowledged;
+                false
+            }
+            _ => {
+                warn!("received unsolicited SubkernelMessageAck");
+                false
+            }
+        }
+    }
+
+    pub fn accept_outgoing(&mut self, message: Vec<u8>) -> Result<(), Error> {
+        // service tag skipped in kernel
+        self.out_message = Some(Sliceable::new(message));
+        self.out_state = OutMessageState::MessageReady;
+        Ok(())
+    }
+
+    pub fn get_incoming(&mut self) -> Option<Message> {
+        self.in_queue.pop_front()
+    }
+}
+
+impl<'a> Manager<'_> {
+    pub fn new(control: &mut kernel::Control) -> Manager {
+        Manager {
+            kernels: BTreeMap::new(),
+            session: Session::new(0),
+            control: control,
+            cache: BTreeMap::new(),
+            last_finished: None,
+        }
+    }
+
+    pub fn add(&mut self, id: u32, status: PayloadStatus, data: &[u8], data_len: usize) -> Result<(), Error> {
+        let kernel = match self.kernels.get_mut(&id) {
+            Some(kernel) => {
+                if kernel.complete || status.is_first() {
+                    // replace entry
+                    self.kernels.remove(&id);
+                    self.kernels.insert(
+                        id,
+                        KernelLibrary {
+                            library: Vec::new(),
+                            complete: false,
+                        },
+                    );
+                    self.kernels.get_mut(&id)?
+                } else {
+                    kernel
+                }
+            }
+            None => {
+                self.kernels.insert(
+                    id,
+                    KernelLibrary {
+                        library: Vec::new(),
+                        complete: false,
+                    },
+                );
+                self.kernels.get_mut(&id)?
+            }
+        };
+        kernel.library.extend(&data[0..data_len]);
+
+        kernel.complete = status.is_last();
+        Ok(())
+    }
+
+    pub fn running(&self) -> bool {
+        self.session.running()
+    }
+
+    pub fn get_current_id(&self) -> Option<u32> {
+        match self.running() {
+            true => Some(self.session.id),
+            false => None,
+        }
+    }
+
+    pub fn run(&mut self, id: u32) -> Result<(), Error> {
+        info!("starting subkernel #{}", id);
+        if self.session.kernel_state != KernelState::Loaded || self.session.id != id {
+            self.load(id)?;
+        }
+        self.session.kernel_state = KernelState::Running;
+        unsafe {
+            csr::cri_con::selected_write(2);
+        }
+
+        self.control.tx.send(kernel::Message::StartRequest);
+        Ok(())
+    }
+
+    pub fn message_handle_incoming(
+        &mut self,
+        status: PayloadStatus,
+        length: usize,
+        slice: &[u8; MASTER_PAYLOAD_MAX_SIZE],
+    ) {
+        if !self.running() {
+            return;
+        }
+        self.session.messages.handle_incoming(status, length, slice);
+    }
+
+    pub fn message_get_slice(&mut self, slice: &mut [u8; MASTER_PAYLOAD_MAX_SIZE]) -> Option<SliceMeta> {
+        if !self.running() {
+            return None;
+        }
+        self.session.messages.get_outgoing_slice(slice)
+    }
+
+    pub fn message_ack_slice(&mut self) -> bool {
+        if !self.running() {
+            warn!("received unsolicited SubkernelMessageAck");
+            return false;
+        }
+        self.session.messages.ack_slice()
+    }
+
+    pub fn message_is_ready(&mut self) -> bool {
+        self.session.messages.is_outgoing_ready()
+    }
+
+    pub fn load(&mut self, id: u32) -> Result<(), Error> {
+        if self.session.id == id && self.session.kernel_state == KernelState::Loaded {
+            return Ok(());
+        }
+        if !self.kernels.get(&id)?.complete {
+            return Err(Error::KernelNotFound);
+        }
+        self.session = Session::new(id);
+        self.control.restart();
+
+        self.control
+            .tx
+            .send(kernel::Message::LoadRequest(self.kernels.get(&id)?.library.clone()));
+        let reply = self.control.rx.recv();
+        match reply {
+            kernel::Message::LoadCompleted => Ok(()),
+            kernel::Message::LoadFailed => Err(Error::Load("kernel load failed".to_string())),
+            _ => Err(Error::Load(format!(
+                "unexpected kernel CPU reply to load request: {:?}",
+                reply
+            ))),
+        }
+    }
+
+    pub fn exception_get_slice(&mut self, data_slice: &mut [u8; SAT_PAYLOAD_MAX_SIZE]) -> SliceMeta {
+        match self.session.last_exception.as_mut() {
+            Some(exception) => exception.get_slice_sat(data_slice),
+            None => SliceMeta {
+                len: 0,
+                status: PayloadStatus::FirstAndLast,
+            },
+        }
+    }
+
+    pub fn get_last_finished(&mut self) -> Option<SubkernelFinished> {
+        self.last_finished.take()
+    }
+
+    fn kernel_stop(&mut self) {
+        self.session.kernel_state = KernelState::Absent;
+        unsafe {
+            csr::cri_con::selected_write(0);
+        }
+    }
+
+    fn runtime_exception(&mut self, cause: Error) {
+        let raw_exception: Vec<u8> = Vec::new();
+        let mut writer = Cursor::new(raw_exception);
+        match write_exception(
+            &mut writer,
+            &[Some(eh_artiq::Exception {
+                id: 11, // SubkernelError, defined in ksupport
+                message: format!("in subkernel id {}: {:?}", self.session.id, cause).as_c_slice(),
+                param: [0, 0, 0],
+                file: file!().as_c_slice(),
+                line: line!(),
+                column: column!(),
+                function: format!("subkernel id {}", self.session.id).as_c_slice(),
+            })],
+            &[eh_artiq::StackPointerBacktrace {
+                stack_pointer: 0,
+                initial_backtrace_size: 0,
+                current_backtrace_size: 0,
+            }],
+            &[],
+            0,
+        ) {
+            Ok(_) => self.session.last_exception = Some(Sliceable::new(writer.into_inner())),
+            Err(_) => error!("Error writing exception data"),
+        }
+        self.kernel_stop();
+    }
+
+    pub fn process_kern_requests(&mut self, rank: u8, timer: GlobalTimer) {
+        if !self.running() {
+            return;
+        }
+
+        match self.process_external_messages(timer) {
+            Ok(()) => (),
+            Err(Error::AwaitingMessage) => return, // kernel still waiting, do not process kernel messages
+            Err(Error::KernelException(exception)) => {
+                self.session.last_exception = Some(exception);
+                self.last_finished = Some(SubkernelFinished {
+                    id: self.session.id,
+                    with_exception: true,
+                });
+            }
+            Err(e) => {
+                error!("Error while running processing external messages: {:?}", e);
+                self.runtime_exception(e);
+                self.last_finished = Some(SubkernelFinished {
+                    id: self.session.id,
+                    with_exception: true,
+                });
+            }
+        }
+
+        match self.process_kern_message(rank, timer) {
+            Ok(true) => {
+                self.last_finished = Some(SubkernelFinished {
+                    id: self.session.id,
+                    with_exception: false,
+                });
+            }
+            Ok(false) | Err(Error::NoMessage) => (),
+            Err(Error::KernelException(exception)) => {
+                self.session.last_exception = Some(exception);
+                self.last_finished = Some(SubkernelFinished {
+                    id: self.session.id,
+                    with_exception: true,
+                });
+            }
+            Err(e) => {
+                error!("Error while running kernel: {:?}", e);
+                self.runtime_exception(e);
+                self.last_finished = Some(SubkernelFinished {
+                    id: self.session.id,
+                    with_exception: true,
+                });
+            }
+        }
+    }
+
+    fn process_kern_message(&mut self, rank: u8, timer: GlobalTimer) -> Result<bool, Error> {
+        let reply = self.control.rx.try_recv()?;
+        match reply {
+            kernel::Message::KernelFinished(_async_errors) => {
+                self.kernel_stop();
+                return Ok(true);
+            }
+            kernel::Message::KernelException(exceptions, stack_pointers, backtrace, async_errors) => {
+                error!("exception in kernel");
+                for exception in exceptions {
+                    error!("{:?}", exception.unwrap());
+                }
+                error!("stack pointers: {:?}", stack_pointers);
+                error!("backtrace: {:?}", backtrace);
+                let buf: Vec<u8> = Vec::new();
+                let mut writer = Cursor::new(buf);
+                match write_exception(&mut writer, exceptions, stack_pointers, backtrace, async_errors) {
+                    Ok(()) => (),
+                    Err(_) => error!("Error writing exception data"),
+                }
+                self.kernel_stop();
+                return Err(Error::KernelException(Sliceable::new(writer.into_inner())));
+            }
+            kernel::Message::CachePutRequest(key, value) => {
+                self.cache.insert(key, value);
+            }
+            kernel::Message::CacheGetRequest(key) => {
+                const DEFAULT: Vec<i32> = Vec::new();
+                let value = self.cache.get(&key).unwrap_or(&DEFAULT).clone();
+                self.control.tx.send(kernel::Message::CacheGetReply(value));
+            }
+            kernel::Message::SubkernelMsgSend { id: _, data } => {
+                self.session.messages.accept_outgoing(data)?;
+                self.session.kernel_state = KernelState::MsgSending;
+            }
+            kernel::Message::SubkernelMsgRecvRequest { id: _, timeout, tags } => {
+                let max_time = timer.get_time() + Milliseconds(timeout);
+                self.session.kernel_state = KernelState::MsgAwait(max_time, tags);
+            }
+            kernel::Message::UpDestinationsRequest(destination) => {
+                self.control
+                    .tx
+                    .send(kernel::Message::UpDestinationsReply(destination == (rank as i32)));
+            }
+            _ => {
+                unexpected!("unexpected message from core1 while kernel was running: {:?}", reply);
+            }
+        }
+        Ok(false)
+    }
+
+    fn process_external_messages(&mut self, timer: GlobalTimer) -> Result<(), Error> {
+        match &self.session.kernel_state {
+            KernelState::MsgAwait(timeout, tags) => {
+                if timer.get_time() > *timeout {
+                    self.control.tx.send(kernel::Message::SubkernelMsgRecvReply {
+                        status: kernel::SubkernelStatus::Timeout,
+                        count: 0,
+                    });
+                    self.session.kernel_state = KernelState::Running;
+                    return Ok(());
+                }
+                if let Some(message) = self.session.messages.get_incoming() {
+                    self.control.tx.send(kernel::Message::SubkernelMsgRecvReply {
+                        status: kernel::SubkernelStatus::NoError,
+                        count: message.count,
+                    });
+                    let tags = tags.clone();
+                    self.session.kernel_state = KernelState::Running;
+                    self.pass_message_to_kernel(&message, tags, timer)
+                } else {
+                    Err(Error::AwaitingMessage)
+                }
+            }
+            KernelState::MsgSending => {
+                if self.session.messages.was_message_acknowledged() {
+                    self.session.kernel_state = KernelState::Running;
+                    self.control.tx.send(kernel::Message::SubkernelMsgSent);
+                    Ok(())
+                } else {
+                    Err(Error::AwaitingMessage)
+                }
+            }
+            _ => Ok(()),
+        }
+    }
+
+    fn pass_message_to_kernel(&mut self, message: &Message, tags: Vec<u8>, timer: GlobalTimer) -> Result<(), Error> {
+        let mut reader = Cursor::new(&message.data);
+        let mut current_tags: &[u8] = &tags;
+        let mut i = message.count;
+        loop {
+            let slot = match recv_w_timeout(&mut self.control.rx, timer, 100)? {
+                kernel::Message::RpcRecvRequest(slot) => slot,
+                other => unexpected!("expected root value slot from core1, not {:?}", other),
+            };
+            let mut exception: Option<Sliceable> = None;
+            let mut unexpected: Option<String> = None;
+            let remaining_tags = rpc::recv_return(&mut reader, current_tags, slot, &mut |size| {
+                if size == 0 {
+                    0 as *mut ()
+                } else {
+                    self.control.tx.send(kernel::Message::RpcRecvReply(Ok(size)));
+                    match recv_w_timeout(&mut self.control.rx, timer, 100) {
+                        Ok(kernel::Message::RpcRecvRequest(slot)) => slot,
+                        Ok(kernel::Message::KernelException(exceptions, stack_pointers, backtrace, async_errors)) => {
+                            let buf: Vec<u8> = Vec::new();
+                            let mut writer = Cursor::new(buf);
+                            match write_exception(&mut writer, exceptions, stack_pointers, backtrace, async_errors) {
+                                Ok(()) => {
+                                    exception = Some(Sliceable::new(writer.into_inner()));
+                                }
+                                Err(_) => {
+                                    unexpected = Some("Error writing exception data".to_string());
+                                }
+                            };
+                            0 as *mut ()
+                        }
+                        other => {
+                            unexpected = Some(format!("expected nested value slot from kernel CPU, not {:?}", other));
+                            0 as *mut ()
+                        }
+                    }
+                }
+            })?;
+            if let Some(exception) = exception {
+                self.kernel_stop();
+                return Err(Error::KernelException(exception));
+            } else if let Some(unexpected) = unexpected {
+                self.kernel_stop();
+                unexpected!("{}", unexpected);
+            }
+            self.control.tx.send(kernel::Message::RpcRecvReply(Ok(0)));
+            i -= 1;
+            if i == 0 {
+                break;
+            } else {
+                current_tags = remaining_tags;
+            }
+        }
+        Ok(())
+    }
+}
+
+fn write_exception<W>(
+    writer: &mut W,
+    exceptions: &[Option<eh_artiq::Exception>],
+    stack_pointers: &[eh_artiq::StackPointerBacktrace],
+    backtrace: &[(usize, usize)],
+    async_errors: u8,
+) -> Result<(), Error>
+where
+    W: Write + ?Sized,
+{
+    /* header */
+    writer.write_bytes(&[0x5a, 0x5a, 0x5a, 0x5a, /*Reply::KernelException*/ 9])?;
+    writer.write_u32(exceptions.len() as u32)?;
+    for exception in exceptions.iter() {
+        let exception = exception.as_ref().unwrap();
+        writer.write_u32(exception.id)?;
+
+        if exception.message.len() == usize::MAX {
+            // exception with host string
+            writer.write_u32(u32::MAX)?;
+            writer.write_u32(exception.message.as_ptr() as u32)?;
+        } else {
+            let msg =
+                str::from_utf8(unsafe { slice::from_raw_parts(exception.message.as_ptr(), exception.message.len()) })
+                    .unwrap()
+                    .replace(
+                        "{rtio_channel_info:0}",
+                        &format!(
+                            "0x{:04x}:{}",
+                            exception.param[0],
+                            ksupport::resolve_channel_name(exception.param[0] as u32)
+                        ),
+                    );
+            writer.write_string(&msg)?;
+        }
+        writer.write_u64(exception.param[0] as u64)?;
+        writer.write_u64(exception.param[1] as u64)?;
+        writer.write_u64(exception.param[2] as u64)?;
+        writer.write_bytes(exception.file.as_ref())?;
+        writer.write_u32(exception.line)?;
+        writer.write_u32(exception.column)?;
+        writer.write_bytes(exception.function.as_ref())?;
+    }
+
+    for sp in stack_pointers.iter() {
+        writer.write_u32(sp.stack_pointer as u32)?;
+        writer.write_u32(sp.initial_backtrace_size as u32)?;
+        writer.write_u32(sp.current_backtrace_size as u32)?;
+    }
+    writer.write_u32(backtrace.len() as u32)?;
+    for &(addr, sp) in backtrace {
+        writer.write_u32(addr as u32)?;
+        writer.write_u32(sp as u32)?;
+    }
+    writer.write_u8(async_errors as u8)?;
+    Ok(())
+}
+
+fn recv_w_timeout(
+    rx: &mut Receiver<'_, kernel::Message>,
+    timer: GlobalTimer,
+    timeout: u64,
+) -> Result<kernel::Message, Error> {
+    let max_time = timer.get_time() + Milliseconds(timeout);
+    while timer.get_time() < max_time {
+        match rx.try_recv() {
+            Err(_) => (),
+            Ok(message) => return Ok(message),
+        }
+    }
+    Err(Error::NoMessage)
+}