drtio_tuple -> drtio_context

This avoids reallocation while transfering binaries.

.gitignore

@@ -3,3 +3,11 @@ examples/*.elf
 __pycache__
 
 build
+
+src/libboard_artiq/Cargo.toml
+src/libc/Cargo.toml
+src/libdyld/Cargo.toml
+src/libio/Cargo.toml
+src/libksupport/Cargo.toml
+src/runtime/Cargo.toml
+src/satman/Cargo.toml

LICENSE

@@ -0,0 +1,165 @@
+                   GNU LESSER GENERAL PUBLIC LICENSE
+                       Version 3, 29 June 2007
+
+ Copyright (C) 2007 Free Software Foundation, Inc. <http://fsf.org/>
+ Everyone is permitted to copy and distribute verbatim copies
+ of this license document, but changing it is not allowed.
+
+
+  This version of the GNU Lesser General Public License incorporates
+the terms and conditions of version 3 of the GNU General Public
+License, supplemented by the additional permissions listed below.
+
+  0. Additional Definitions.
+
+  As used herein, "this License" refers to version 3 of the GNU Lesser
+General Public License, and the "GNU GPL" refers to version 3 of the GNU
+General Public License.
+
+  "The Library" refers to a covered work governed by this License,
+other than an Application or a Combined Work as defined below.
+
+  An "Application" is any work that makes use of an interface provided
+by the Library, but which is not otherwise based on the Library.
+Defining a subclass of a class defined by the Library is deemed a mode
+of using an interface provided by the Library.
+
+  A "Combined Work" is a work produced by combining or linking an
+Application with the Library.  The particular version of the Library
+with which the Combined Work was made is also called the "Linked
+Version".
+
+  The "Minimal Corresponding Source" for a Combined Work means the
+Corresponding Source for the Combined Work, excluding any source code
+for portions of the Combined Work that, considered in isolation, are
+based on the Application, and not on the Linked Version.
+
+  The "Corresponding Application Code" for a Combined Work means the
+object code and/or source code for the Application, including any data
+and utility programs needed for reproducing the Combined Work from the
+Application, but excluding the System Libraries of the Combined Work.
+
+  1. Exception to Section 3 of the GNU GPL.
+
+  You may convey a covered work under sections 3 and 4 of this License
+without being bound by section 3 of the GNU GPL.
+
+  2. Conveying Modified Versions.
+
+  If you modify a copy of the Library, and, in your modifications, a
+facility refers to a function or data to be supplied by an Application
+that uses the facility (other than as an argument passed when the
+facility is invoked), then you may convey a copy of the modified
+version:
+
+   a) under this License, provided that you make a good faith effort to
+   ensure that, in the event an Application does not supply the
+   function or data, the facility still operates, and performs
+   whatever part of its purpose remains meaningful, or
+
+   b) under the GNU GPL, with none of the additional permissions of
+   this License applicable to that copy.
+
+  3. Object Code Incorporating Material from Library Header Files.
+
+  The object code form of an Application may incorporate material from
+a header file that is part of the Library.  You may convey such object
+code under terms of your choice, provided that, if the incorporated
+material is not limited to numerical parameters, data structure
+layouts and accessors, or small macros, inline functions and templates
+(ten or fewer lines in length), you do both of the following:
+
+   a) Give prominent notice with each copy of the object code that the
+   Library is used in it and that the Library and its use are
+   covered by this License.
+
+   b) Accompany the object code with a copy of the GNU GPL and this license
+   document.
+
+  4. Combined Works.
+
+  You may convey a Combined Work under terms of your choice that,
+taken together, effectively do not restrict modification of the
+portions of the Library contained in the Combined Work and reverse
+engineering for debugging such modifications, if you also do each of
+the following:
+
+   a) Give prominent notice with each copy of the Combined Work that
+   the Library is used in it and that the Library and its use are
+   covered by this License.
+
+   b) Accompany the Combined Work with a copy of the GNU GPL and this license
+   document.
+
+   c) For a Combined Work that displays copyright notices during
+   execution, include the copyright notice for the Library among
+   these notices, as well as a reference directing the user to the
+   copies of the GNU GPL and this license document.
+
+   d) Do one of the following:
+
+       0) Convey the Minimal Corresponding Source under the terms of this
+       License, and the Corresponding Application Code in a form
+       suitable for, and under terms that permit, the user to
+       recombine or relink the Application with a modified version of
+       the Linked Version to produce a modified Combined Work, in the
+       manner specified by section 6 of the GNU GPL for conveying
+       Corresponding Source.
+
+       1) Use a suitable shared library mechanism for linking with the
+       Library.  A suitable mechanism is one that (a) uses at run time
+       a copy of the Library already present on the user's computer
+       system, and (b) will operate properly with a modified version
+       of the Library that is interface-compatible with the Linked
+       Version.
+
+   e) Provide Installation Information, but only if you would otherwise
+   be required to provide such information under section 6 of the
+   GNU GPL, and only to the extent that such information is
+   necessary to install and execute a modified version of the
+   Combined Work produced by recombining or relinking the
+   Application with a modified version of the Linked Version. (If
+   you use option 4d0, the Installation Information must accompany
+   the Minimal Corresponding Source and Corresponding Application
+   Code. If you use option 4d1, you must provide the Installation
+   Information in the manner specified by section 6 of the GNU GPL
+   for conveying Corresponding Source.)
+
+  5. Combined Libraries.
+
+  You may place library facilities that are a work based on the
+Library side by side in a single library together with other library
+facilities that are not Applications and are not covered by this
+License, and convey such a combined library under terms of your
+choice, if you do both of the following:
+
+   a) Accompany the combined library with a copy of the same work based
+   on the Library, uncombined with any other library facilities,
+   conveyed under the terms of this License.
+
+   b) Give prominent notice with the combined library that part of it
+   is a work based on the Library, and explaining where to find the
+   accompanying uncombined form of the same work.
+
+  6. Revised Versions of the GNU Lesser General Public License.
+
+  The Free Software Foundation may publish revised and/or new versions
+of the GNU Lesser General Public License from time to time. Such new
+versions will be similar in spirit to the present version, but may
+differ in detail to address new problems or concerns.
+
+  Each version is given a distinguishing version number. If the
+Library as you received it specifies that a certain numbered version
+of the GNU Lesser General Public License "or any later version"
+applies to it, you have the option of following the terms and
+conditions either of that published version or of any later version
+published by the Free Software Foundation. If the Library as you
+received it does not specify a version number of the GNU Lesser
+General Public License, you may choose any version of the GNU Lesser
+General Public License ever published by the Free Software Foundation.
+
+  If the Library as you received it specifies that a proxy can decide
+whether future versions of the GNU Lesser General Public License shall
+apply, that proxy's public statement of acceptance of any version is
+permanent authorization for you to choose that version for the
+Library.

README.md

@@ -1,26 +1,120 @@
-Configure Nix channels:
+ARTIQ on Zynq
+=============
+
+How to use
+----------
+
+1. [Install ARTIQ](https://m-labs.hk/artiq/manual/installing.html). Get the corresponding version to the ``artiq-zynq`` version you are targeting.
+2. To obtain firmware binaries, use AFWS or build your own; see [the ARTIQ manual](https://m-labs.hk/artiq/manual/building_developing.html) for detailed instructions or skip to "Development" below. ZC706 variants only can also be downloaded from latest successful build on [Hydra](https://nixbld.m-labs.hk/).
+3. Place ``boot.bin`` file at the root ``/`` of a FAT-formatted SD card.
+4. Optionally, create a ``config.txt`` configuration file containing ``key=value`` pairs on each line and place it at the root of the SD card. See below for valid keys. The ``ip``, ``ip6`` and ``mac`` keys can be used to set networking information. If these keys are not found, the firmware will use default values which may or may not be compatible with your network.
+5. Insert the SD card into the board and set the board to boot from the SD card. For ZC706, this is achieved by placing the large DIP switch SW11 into the 00110 position. On Kasli-SoC, place the BOOT MODE switches to SD.
+6. Power up the board. After successful boot the firmware should respond to ping at its IP addresses. Boot output can be observed from UART at 115200bps 8-N-1.
+7. Create and use an ARTIQ device database as usual.
+
+Configuration
+-------------
+
+Configuring the device is done using the ``config.txt`` text file at the root of the SD card plus optionally a ``config`` folder. When searching for a configuration key, the firmware first looks for a file named ``/config/[key].bin`` and, if it exists, returns the contents of that file. If not, it looks into ``/config.txt``, which should contain a list of ``key=value`` pairs, one per line. ``config.txt`` should be used for most keys but the ``config`` folder allows for setting configuration values which consist of binary data, such as the startup kernel.
+
+The following configuration keys are available among others:
+
+- ``mac``: Ethernet MAC address.
+- ``ip``: IPv4 address.
+- ``ip6``: IPv6 address.
+- ``idle_kernel``: idle kernel in ELF format (as produced by ``artiq_compile``).
+- ``startup_kernel``: startup kernel in ELF format (as produced by ``artiq_compile``).
+- ``rtio_clock``: source of RTIO clock; valid values are ``ext0_bypass`` and ``int_125``.
+
+See [ARTIQ manual](https://m-labs.hk/artiq/manual-beta/core_device.html#configuration-storage) for full list. Configurations can be read/written/removed with ``artiq_coremgmt``. Config erase is not implemented, as it isn't particularly useful.
+
+For convenience, the ``boot`` key can be used with ``artiq_coremgmt`` and a ``boot.bin`` file to replace firmware/gateware in a running system. This key is read-only. When loading ``boot.bin`` onto the SD card directly, place it at the root and not in the ``config`` folder.
+
+Development instructions
+------------------------
+
+ARTIQ on Zynq is packaged using [Nix](https://nixos.org) Flakes. Install Nix 2.8+ and enable flakes by adding ``experimental-features = nix-command flakes`` to ``nix.conf`` (e.g. ``~/.config/nix/nix.conf``).
+
+**Pure build with Nix:**
 
 ```shell
-nix-channel --add https://nixbld.m-labs.hk/channel/custom/artiq/fast-beta/artiq-fast
-nix-channel --update
+nix build .#zc706-nist_clock-jtag  # or zc706-nist_qc2-jtag or zc706-nist_clock-sd or etc
 ```
 
-Pure build with Nix:
+Run ``nix flake show`` to see all valid build targets. Targets suffixed with ``-jtag`` produce separate firmware and gateware files, intended for use in booting via JTAG server/Ethernet, e.g. ``./remote_run.sh -i`` with a remote JTAG server. Targets suffixed with ``-sd`` will produce ``boot.bin`` file suitable for SD card boot. ``-firmware`` and ``-gateware`` respectively build firmware and gateware only.
+
+The Kasli-SoC target requires a system description file as input. See ARTIQ manual for exact instructions or use incremental build.
+
+**Impure incremental build:**
+
+For boards with fixed variants, i.e. ZC706, etc. :
 
 ```shell
-nix-build -A zc706-simple-jtag  # or zc706-nist_qc2-jtag or zc706-nist_clock-jtag
-./remote_run.sh
-```
-
-Impure incremental build:
-
-```shell
-nix-shell
+nix develop
 cd src
-./zc706.py -g ../build/gateware  # build gateware
-make                             # build firmware
-cd ..
-./remote_run.sh -i
+gateware/<board>.py -g ../build/gateware -V <variant> # gateware
+make GWARGS="-V <variant>" <runtime/satman>    # firmware
 ```
 
-The impure build process can also be used on non-Nix systems.
+For boards with system descriptions, i.e. Kasli-SoC, etc. :
+
+```shell
+nix develop
+cd src
+gateware/<board>.py -g ../build/gateware <description.json> # gateware
+make TARGET=<board> GWARGS="path/to/description.json" <runtime/satman> # firmware
+```
+
+``szl.elf`` can be obtained with:
+
+```shell
+nix build git+https://git.m-labs.hk/m-labs/zynq-rs#<board>-szl
+```
+
+To generate ``boot.bin`` use ``mkbootimage``, e.g.:
+
+```shell
+echo "the_ROM_image:
+    {
+        [bootloader]result/szl.elf
+        gateware/top.bit
+        firmware/armv7-none-eabihf/release/<runtime/satman>
+    }
+    EOF" >> boot.bif
+mkbootimage boot.bif boot.bin
+```
+
+Notes:
+
+- The impure build process is also compatible with non-Nix systems.
+- Firmware type must be either ``runtime`` for DRTIO-less or DRTIO master variants, or ``satman`` for DRTIO satellite.
+- If the board is connected to the local machine by JTAG, use the ``local_run.sh`` script.
+- A known Xilinx hardware bug prevents repeatedly loading the bootloader over JTAG without a POR reset. If booting over JTAG, install a jumper on ``PS_POR_B`` and use the POR reset script [here](https://git.m-labs.hk/M-Labs/zynq-rs/src/branch/master/kasli_soc_por.py).
+
+Pre-Commit Hooks
+----------------
+
+You are strongly recommended to use the provided pre-commit hooks to automatically reformat files and check for non-optimal Rust/C/C++ practices. Run `pre-commit install` to install the hook and `pre-commit` will automatically run `cargo fmt`, `cargo clippy`, and `clang-format` for you.
+
+Several things to note:
+
+- If `cargo fmt`, `cargo clippy`, or `clang-format` returns an error, the pre-commit hook will fail. You should fix all errors before trying to commit again.
+- If `cargo fmt` or `clang-format` reformats some files, the pre-commit hook will also fail. You should review the changes and, if satisfied, try to commit again.
+
+License
+-------
+
+Copyright (C) 2019-2024 M-Labs Limited.
+
+ARTIQ is free software: you can redistribute it and/or modify
+it under the terms of the GNU Lesser General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+ARTIQ is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU Lesser General Public License for more details.
+
+You should have received a copy of the GNU Lesser General Public License
+along with ARTIQ.  If not, see <http://www.gnu.org/licenses/>.

default.nix

@@ -1,96 +0,0 @@
-{
-  mozillaOverlay ? import (builtins.fetchTarball https://github.com/mozilla/nixpkgs-mozilla/archive/master.tar.gz),
-}:
-
-let
-  pkgs = import <nixpkgs> { overlays = [ mozillaOverlay ]; };
-  rustPlatform = (import ./rustPlatform.nix { inherit pkgs; });
-  artiqpkgs = import <artiq-fast/default.nix> { inherit pkgs; };
-  vivado = import <artiq-fast/vivado.nix> { inherit pkgs; };
-  mkbootimage = (import ./mkbootimage.nix { inherit pkgs; });
-  build-zc706 = { variant }: let
-    firmware = rustPlatform.buildRustPackage rec {
-      name = "zc706-${variant}-firmware";
-      version = "0.1.0";
-
-      src = ./src;
-      cargoSha256 = "0xminds5fyp7c9vsx651zv3yzyhxnl9a02rhjl2wfxf8m679r45l";
-
-      nativeBuildInputs = [
-        pkgs.gnumake
-        (pkgs.python3.withPackages(ps: (with artiqpkgs; [ migen migen-axi misoc artiq ])))
-        pkgs.cargo-xbuild
-        pkgs.llvmPackages_9.llvm
-        pkgs.llvmPackages_9.clang-unwrapped
-      ];
-      buildPhase = ''
-        export XARGO_RUST_SRC="${rustPlatform.rust.rustc.src}/src"
-        export CARGO_HOME=$(mktemp -d cargo-home.XXX)
-        make VARIANT=${variant}
-      '';
-
-      installPhase = ''
-        mkdir -p $out $out/nix-support
-        cp ../build/firmware/armv7-none-eabihf/release/runtime $out/runtime.elf
-        cp ../build/firmware/armv7-none-eabihf/release/szl $out/szl.elf
-        echo file binary-dist $out/runtime.elf >> $out/nix-support/hydra-build-products
-        echo file binary-dist $out/szl.elf >> $out/nix-support/hydra-build-products
-      '';
-
-      doCheck = false;
-      dontFixup = true;
-    };
-    gateware = pkgs.runCommand "zc706-${variant}-gateware"
-      {
-        nativeBuildInputs = [ 
-          (pkgs.python3.withPackages(ps: (with artiqpkgs; [ migen migen-axi misoc artiq ])))
-          vivado
-        ];
-      }
-      ''
-        python ${./src/zc706.py} -g build -V ${variant}
-        mkdir -p $out $out/nix-support
-        cp build/top.bit $out
-        echo file binary-dist $out/top.bit >> $out/nix-support/hydra-build-products
-      '';
-
-    jtag = pkgs.runCommand "zc706-${variant}-jtag" {}
-      ''
-        mkdir $out
-        ln -s ${firmware}/szl.elf $out
-        ln -s ${gateware}/top.bit $out
-      '';
-    sd = pkgs.runCommand "zc706-${variant}-sd"
-      {
-        buildInputs = [ mkbootimage ];
-      }
-      ''
-      # Do not use "long" paths in boot.bif, because embedded developers
-      # can't write software (mkbootimage will segfault).
-      bifdir=`mktemp -d`
-      cd $bifdir
-      ln -s ${firmware}/szl.elf szl.elf
-      ln -s ${gateware}/top.bit top.bit
-      cat > boot.bif << EOF
-      the_ROM_image:
-      {
-        [bootloader]szl.elf
-        top.bit
-      }
-      EOF
-      mkdir $out $out/nix-support
-      mkbootimage boot.bif $out/boot.bin
-      echo file binary-dist $out/boot.bin >> $out/nix-support/hydra-build-products
-      '';
-  in {
-    "zc706-${variant}-firmware" = firmware;
-    "zc706-${variant}-gateware" = gateware;
-    "zc706-${variant}-jtag" = jtag;
-    "zc706-${variant}-sd" = sd;
-  };
-in
-  (
-    (build-zc706 { variant = "simple"; }) //
-    (build-zc706 { variant = "nist_clock"; }) //
-    (build-zc706 { variant = "nist_qc2"; })
-  )

demo.json

@@ -0,0 +1,60 @@
+{
+    "target": "kasli_soc",
+    "variant": "demo",
+    "hw_rev": "v1.0",
+    "base": "standalone",
+    "peripherals": [
+        {
+            "type": "grabber",
+            "ports": [0]
+        },
+        {
+            "type": "dio",
+            "ports": [1],
+            "bank_direction_low": "input",
+            "bank_direction_high": "output"
+        },
+        {
+            "type": "dio",
+            "ports": [2],
+            "bank_direction_low": "output",
+            "bank_direction_high": "output"
+        },
+        {
+            "type": "urukul",
+            "dds": "ad9910",
+            "ports": [3, 4],
+            "clk_sel": 2
+        },
+        {
+            "type": "zotino",
+            "ports": [5]
+        },
+        {
+            "type": "sampler",
+            "ports": [6, 7]
+        },
+        {
+            "type": "mirny",
+            "ports": [8],
+            "clk_sel": 1,
+            "refclk": 125e6
+        },
+        {
+            "type": "fastino",
+            "ports": [9]
+        },
+        {
+            "type": "dio",
+            "ports": [10],
+            "bank_direction_low": "input",
+            "bank_direction_high": "input"
+        },
+        {
+            "type": "dio",
+            "ports": [11],
+            "bank_direction_low": "output",
+            "bank_direction_high": "input"
+        }
+    ]
+}

examples/device_db.py

@@ -1,3 +1,5 @@
+# For NIST_QC2
+
 device_db = {
     "core": {
         "type": "local",
@@ -6,33 +8,69 @@ device_db = {
         "arguments": {
             "host": "192.168.1.52",
             "ref_period": 1e-9,
-            "ref_multiplier": 1,
+            "ref_multiplier": 8,
             "target": "cortexa9"
         }
     },
+    "core_cache": {
+        "type": "local",
+        "module": "artiq.coredevice.cache",
+        "class": "CoreCache"
+    },
+    "core_dma": {
+        "type": "local",
+        "module": "artiq.coredevice.dma",
+        "class": "CoreDMA"
+    },
+
+    "i2c_switch": {
+        "type": "local",
+        "module": "artiq.coredevice.i2c",
+        "class": "PCA9548"
+    },
 
     "led0": {
         "type": "local",
         "module": "artiq.coredevice.ttl",
         "class": "TTLOut",
-        "arguments": {"channel": 0},
-    },
-    "led1": {
-        "type": "local",
-        "module": "artiq.coredevice.ttl",
-        "class": "TTLOut",
-        "arguments": {"channel": 1},
-    },
-    "led2": {
-        "type": "local",
-        "module": "artiq.coredevice.ttl",
-        "class": "TTLOut",
-        "arguments": {"channel": 2}
-    },
-    "led3": {
-        "type": "local",
-        "module": "artiq.coredevice.ttl",
-        "class": "TTLOut",
-        "arguments": {"channel": 3}
+        "arguments": {"channel": 41},
     },
 }
+
+# TTLs on QC2 backplane
+for i in range(40):
+    device_db["ttl" + str(i)] = {
+        "type": "local",
+        "module": "artiq.coredevice.ttl",
+        "class": "TTLInOut",
+        "arguments": {"channel": i}
+    }
+
+device_db["ad9914dds0"] = {
+    "type": "local",
+    "module": "artiq.coredevice.ad9914",
+    "class": "AD9914",
+    "arguments": {"sysclk": 3e9, "bus_channel": 50, "channel": 0},
+}
+device_db["ad9914dds1"] = {
+    "type": "local",
+    "module": "artiq.coredevice.ad9914",
+    "class": "AD9914",
+    "arguments": {"sysclk": 3e9, "bus_channel": 50, "channel": 1},
+}
+
+for i in range(4):
+    device_db["ttl"+str(i)+"_counter"] = {
+        "type": "local",
+        "module": "artiq.coredevice.edge_counter",
+        "class": "EdgeCounter",
+        "arguments": {"channel": 52+i}
+    }
+
+# for ARTIQ test suite
+device_db.update(
+    loop_out="ttl0",
+    loop_in="ttl1",
+    ttl_out="ttl2",
+    ttl_out_serdes="ttl2",
+)

examples/dma.py

@@ -0,0 +1,26 @@
+from artiq.experiment import *
+
+class DMAPulses(EnvExperiment):
+    def build(self):
+        self.setattr_device("core")
+        self.setattr_device("core_dma")
+        self.setattr_device("led0")
+
+    @kernel
+    def record(self):
+        with self.core_dma.record("pulse"):
+            delay(200*ms)
+            # all RTIO operations now go to the "pulse"
+            # DMA buffer, instead of being executed immediately.
+            self.led0.pulse(500*ms)
+
+
+    @kernel
+    def run(self):
+        self.core.reset()
+        self.record()
+        # prefetch the address of the DMA buffer
+        # for faster playback trigger
+        pulse_handle = self.core_dma.get_handle("pulse")
+        self.core.break_realtime()
+        self.core_dma.playback_handle(pulse_handle)

examples/ebaz4205/device_db.py

@@ -0,0 +1,78 @@
+core_addr = "192.168.1.57"
+
+device_db = {
+    "core": {
+        "type": "local",
+        "module": "artiq.coredevice.core",
+        "class": "Core",
+        "arguments": {
+            "host": core_addr,
+            "ref_period": 1e-9,
+            "target": "cortexa9",
+        },
+    },
+    "core_log": {
+        "type": "controller",
+        "host": "::1",
+        "port": 1068,
+        "command": "aqctl_corelog -p {port} --bind {bind} " + core_addr,
+    },
+    "core_moninj": {
+        "type": "controller",
+        "host": "::1",
+        "port_proxy": 1383,
+        "port": 1384,
+        "command": "aqctl_moninj_proxy --port-proxy {port_proxy} --port-control {port} --bind {bind} "
+        + core_addr,
+    },
+    "core_analyzer": {
+        "type": "controller",
+        "host": "::1",
+        "port_proxy": 1385,
+        "port": 1386,
+        "command": "aqctl_coreanalyzer_proxy --port-proxy {port_proxy} --port-control {port} --bind {bind} "
+        + core_addr,
+    },
+    "core_cache": {
+        "type": "local",
+        "module": "artiq.coredevice.cache",
+        "class": "CoreCache",
+    },
+    "core_dma": {"type": "local", "module": "artiq.coredevice.dma", "class": "CoreDMA"},
+    "led0": {
+        "type": "local",
+        "module": "artiq.coredevice.ttl",
+        "class": "TTLOut",
+        "arguments": {"channel": 0},
+    },
+    "led1": {
+        "type": "local",
+        "module": "artiq.coredevice.ttl",
+        "class": "TTLOut",
+        "arguments": {"channel": 1},
+    },
+}
+
+# TTLs starting at RTIO channel 2, ending at RTIO channel 15
+for i in range(2, 16):
+    device_db["ttl" + str(i)] = {
+        "type": "local",
+        "module": "artiq.coredevice.ttl",
+        "class": "TTLInOut",
+        "arguments": {"channel": i},
+    }
+
+device_db.update(
+    spi0={
+        "type": "local",
+        "module": "artiq.coredevice.spi2",
+        "class": "SPIMaster",
+        "arguments": {"channel": 16},
+    },
+    dds0={
+        "type": "local",
+        "module": "artiq.coredevice.ad9834",
+        "class": "AD9834",
+        "arguments": {"spi_device": "spi0"},
+    },
+)

flake.lock

@@ -0,0 +1,248 @@
+{
+  "nodes": {
+    "artiq": {
+      "inputs": {
+        "artiq-comtools": "artiq-comtools",
+        "nixpkgs": "nixpkgs",
+        "rust-overlay": "rust-overlay",
+        "sipyco": "sipyco",
+        "src-migen": "src-migen",
+        "src-misoc": "src-misoc",
+        "src-pythonparser": "src-pythonparser"
+      },
+      "locked": {
+        "lastModified": 1731734344,
+        "narHash": "sha256-2vLRo9D5wDs5FArpc2pOfuKFrhnqIKjtZos88VJahhc=",
+        "ref": "refs/heads/master",
+        "rev": "27d54cb8f3a394b4f4adcbb3c2c9160c5bf3df47",
+        "revCount": 9049,
+        "type": "git",
+        "url": "https://github.com/m-labs/artiq.git"
+      },
+      "original": {
+        "type": "git",
+        "url": "https://github.com/m-labs/artiq.git"
+      }
+    },
+    "artiq-comtools": {
+      "inputs": {
+        "flake-utils": "flake-utils",
+        "nixpkgs": [
+          "artiq",
+          "nixpkgs"
+        ],
+        "sipyco": [
+          "artiq",
+          "sipyco"
+        ]
+      },
+      "locked": {
+        "lastModified": 1720768567,
+        "narHash": "sha256-3VoK7o5MtHtbHLrc6Pv+eQWFtaz5Gd/YWyV5TD3c5Ss=",
+        "owner": "m-labs",
+        "repo": "artiq-comtools",
+        "rev": "f93570d8f2ed5a3cfb3e1c16ab00f2540551e994",
+        "type": "github"
+      },
+      "original": {
+        "owner": "m-labs",
+        "repo": "artiq-comtools",
+        "type": "github"
+      }
+    },
+    "flake-utils": {
+      "inputs": {
+        "systems": "systems"
+      },
+      "locked": {
+        "lastModified": 1710146030,
+        "narHash": "sha256-SZ5L6eA7HJ/nmkzGG7/ISclqe6oZdOZTNoesiInkXPQ=",
+        "owner": "numtide",
+        "repo": "flake-utils",
+        "rev": "b1d9ab70662946ef0850d488da1c9019f3a9752a",
+        "type": "github"
+      },
+      "original": {
+        "owner": "numtide",
+        "repo": "flake-utils",
+        "type": "github"
+      }
+    },
+    "nixpkgs": {
+      "locked": {
+        "lastModified": 1731319897,
+        "narHash": "sha256-PbABj4tnbWFMfBp6OcUK5iGy1QY+/Z96ZcLpooIbuEI=",
+        "owner": "NixOS",
+        "repo": "nixpkgs",
+        "rev": "dc460ec76cbff0e66e269457d7b728432263166c",
+        "type": "github"
+      },
+      "original": {
+        "owner": "NixOS",
+        "ref": "nixos-unstable",
+        "repo": "nixpkgs",
+        "type": "github"
+      }
+    },
+    "root": {
+      "inputs": {
+        "artiq": "artiq",
+        "zynq-rs": "zynq-rs"
+      }
+    },
+    "rust-overlay": {
+      "inputs": {
+        "nixpkgs": [
+          "artiq",
+          "nixpkgs"
+        ]
+      },
+      "locked": {
+        "lastModified": 1719454714,
+        "narHash": "sha256-MojqG0lyUINkEk0b3kM2drsU5vyaF8DFZe/FAlZVOGs=",
+        "owner": "oxalica",
+        "repo": "rust-overlay",
+        "rev": "d1c527659cf076ecc4b96a91c702d080b213801e",
+        "type": "github"
+      },
+      "original": {
+        "owner": "oxalica",
+        "ref": "snapshot/2024-08-01",
+        "repo": "rust-overlay",
+        "type": "github"
+      }
+    },
+    "rust-overlay_2": {
+      "inputs": {
+        "nixpkgs": [
+          "zynq-rs",
+          "nixpkgs"
+        ]
+      },
+      "locked": {
+        "lastModified": 1719454714,
+        "narHash": "sha256-MojqG0lyUINkEk0b3kM2drsU5vyaF8DFZe/FAlZVOGs=",
+        "owner": "oxalica",
+        "repo": "rust-overlay",
+        "rev": "d1c527659cf076ecc4b96a91c702d080b213801e",
+        "type": "github"
+      },
+      "original": {
+        "owner": "oxalica",
+        "ref": "snapshot/2024-08-01",
+        "repo": "rust-overlay",
+        "type": "github"
+      }
+    },
+    "sipyco": {
+      "inputs": {
+        "nixpkgs": [
+          "artiq",
+          "nixpkgs"
+        ]
+      },
+      "locked": {
+        "lastModified": 1728371104,
+        "narHash": "sha256-PPnAyDedUQ7Og/Cby9x5OT9wMkNGTP8GS53V6N/dk4w=",
+        "owner": "m-labs",
+        "repo": "sipyco",
+        "rev": "094a6cd63ffa980ef63698920170e50dc9ba77fd",
+        "type": "github"
+      },
+      "original": {
+        "owner": "m-labs",
+        "repo": "sipyco",
+        "type": "github"
+      }
+    },
+    "src-migen": {
+      "flake": false,
+      "locked": {
+        "lastModified": 1727677091,
+        "narHash": "sha256-Zg3SQnTwMM/VkOGKogbPyuCC2NhLy8HB2SPEUWWNgCU=",
+        "owner": "m-labs",
+        "repo": "migen",
+        "rev": "c19ae9f8ae162ffe2d310a92bfce53ac2a821bc8",
+        "type": "github"
+      },
+      "original": {
+        "owner": "m-labs",
+        "repo": "migen",
+        "type": "github"
+      }
+    },
+    "src-misoc": {
+      "flake": false,
+      "locked": {
+        "lastModified": 1729234629,
+        "narHash": "sha256-TLsTCXV5AC2xh+bS7EhBVBKqdqIU3eKrnlWcFF9LtAM=",
+        "ref": "refs/heads/master",
+        "rev": "6085a312bca26adeca6584e37d08c8ba2e1d6e38",
+        "revCount": 2460,
+        "submodules": true,
+        "type": "git",
+        "url": "https://github.com/m-labs/misoc.git"
+      },
+      "original": {
+        "submodules": true,
+        "type": "git",
+        "url": "https://github.com/m-labs/misoc.git"
+      }
+    },
+    "src-pythonparser": {
+      "flake": false,
+      "locked": {
+        "lastModified": 1628745371,
+        "narHash": "sha256-p6TgeeaK4NEmbhimEXp31W8hVRo4DgWmcCoqZ+UdN60=",
+        "owner": "m-labs",
+        "repo": "pythonparser",
+        "rev": "5413ee5c9f8760e95c6acd5d6e88dabb831ad201",
+        "type": "github"
+      },
+      "original": {
+        "owner": "m-labs",
+        "repo": "pythonparser",
+        "type": "github"
+      }
+    },
+    "systems": {
+      "locked": {
+        "lastModified": 1681028828,
+        "narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=",
+        "owner": "nix-systems",
+        "repo": "default",
+        "rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e",
+        "type": "github"
+      },
+      "original": {
+        "owner": "nix-systems",
+        "repo": "default",
+        "type": "github"
+      }
+    },
+    "zynq-rs": {
+      "inputs": {
+        "nixpkgs": [
+          "artiq",
+          "nixpkgs"
+        ],
+        "rust-overlay": "rust-overlay_2"
+      },
+      "locked": {
+        "lastModified": 1731749494,
+        "narHash": "sha256-WGigAhvVCGN5YZ1dHPyvoqAh47W1Gtph036O1aKFlLE=",
+        "ref": "refs/heads/master",
+        "rev": "12975de2e110d7948bf47b768559f727d0abc8fc",
+        "revCount": 655,
+        "type": "git",
+        "url": "https://git.m-labs.hk/m-labs/zynq-rs"
+      },
+      "original": {
+        "type": "git",
+        "url": "https://git.m-labs.hk/m-labs/zynq-rs"
+      }
+    }
+  },
+  "root": "root",
+  "version": 7
+}

flake.nix

@@ -0,0 +1,399 @@
+{
+  description = "ARTIQ port to the Zynq-7000 platform";
+
+  inputs.artiq.url = git+https://github.com/m-labs/artiq.git;
+  inputs.zynq-rs.url = git+https://git.m-labs.hk/m-labs/zynq-rs;
+  inputs.zynq-rs.inputs.nixpkgs.follows = "artiq/nixpkgs";
+
+  outputs = { self, zynq-rs, artiq }:
+  let
+    pkgs = import artiq.inputs.nixpkgs { system = "x86_64-linux"; overlays = [ (import zynq-rs.inputs.rust-overlay) ]; };
+    zynqpkgs = zynq-rs.packages.x86_64-linux;
+    artiqpkgs = artiq.packages.x86_64-linux;
+    llvmPackages_11 = zynq-rs.llvmPackages_11;
+
+    rust = zynq-rs.rust;
+    rustPlatform = zynq-rs.rustPlatform;
+
+    fastnumbers = pkgs.python3Packages.buildPythonPackage rec {
+      pname = "fastnumbers";
+      version = "5.1.0";
+
+      src = pkgs.python3Packages.fetchPypi {
+        inherit pname version;
+        sha256 = "sha256-4JLTP4uVwxcaL7NOV57+DFSwKQ3X+W/6onYkN2AdkKc=";
+      };
+    };
+
+    artiq-netboot = pkgs.python3Packages.buildPythonPackage rec {
+      pname = "artiq-netboot";
+      version = "unstable-2020-10-15";
+
+      src = pkgs.fetchgit {
+        url = "https://git.m-labs.hk/m-labs/artiq-netboot.git";
+        rev = "04f69eb07df73abe4b89fde2c24084f7664f2104";
+        sha256 = "0ql4fr8m8gpb2yql8aqsdqsssxb8zqd6l65kl1f6s9845zy7shs9";
+      };
+    };
+
+    ramda = pkgs.python3Packages.buildPythonPackage {
+      pname = "ramda";
+      version = "unstable-2020-04-11";
+
+      src = pkgs.fetchFromGitHub {
+        owner = "peteut";
+        repo = "ramda.py";
+        rev = "d315a9717ebd639366bf3fe26bad9e3d08ec3c49";
+        sha256 = "sha256-bmSt/IHDnULsZjsC6edELnNH7LoJSVF4L4XhwBAXRkY=";
+      };
+
+      nativeBuildInputs = with pkgs.python3Packages; [ pbr ];
+      propagatedBuildInputs = with pkgs.python3Packages; [ future fastnumbers ];
+
+      checkInputs = with pkgs.python3Packages; [ pytest ];
+      checkPhase = "pytest";
+      doCheck = false;
+
+      preBuild = ''
+        export PBR_VERSION=0.5.5
+      '';
+    };
+
+    migen-axi = pkgs.python3Packages.buildPythonPackage {
+      pname = "migen-axi";
+      version = "unstable-2023-01-06";
+
+      src = pkgs.fetchFromGitHub {
+        owner = "peteut";
+        repo = "migen-axi";
+        rev = "27eaa84a70a3abfe1930c86c36c4de2cd652da35";
+        sha256 = "sha256-3Y9W5ns+1wbVd14iePzgSBzE+LxnGMUDtUw3BccFt80=";
+      };
+
+      format = "pyproject";
+
+      propagatedBuildInputs = with pkgs.python3Packages; [ setuptools click numpy toolz jinja2 ramda artiqpkgs.migen artiqpkgs.misoc ];
+
+      checkInputs = with pkgs.python3Packages; [ pytestCheckHook pytest-timeout ];
+
+      # migen/misoc version checks are broken with pyproject for some reason
+      postPatch = ''
+        sed -i "1,4d" pyproject.toml
+        substituteInPlace pyproject.toml \
+          --replace '"migen@git+https://github.com/m-labs/migen",' ""
+        substituteInPlace pyproject.toml \
+          --replace '"misoc@git+https://github.com/m-labs/misoc.git",' ""
+        # pytest-flake8 is broken with recent flake8. Re-enable after fix.
+        substituteInPlace setup.cfg --replace '--flake8' ""
+      '';
+    };
+    binutils = { platform, target, zlib }: pkgs.stdenv.mkDerivation rec {
+      basename = "binutils";
+      version = "2.30";
+      name = "${basename}-${platform}-${version}";
+      src = pkgs.fetchurl {
+        url = "https://ftp.gnu.org/gnu/binutils/binutils-${version}.tar.bz2";
+        sha256 = "028cklfqaab24glva1ks2aqa1zxa6w6xmc8q34zs1sb7h22dxspg";
+      };
+      configureFlags =
+        [ "--enable-shared" "--enable-deterministic-archives" "--target=${target}"];
+      outputs = [ "out" "info" "man" ];
+      depsBuildBuild = [ pkgs.buildPackages.stdenv.cc ];
+      buildInputs = [ zlib ];
+      enableParallelBuilding = true;
+    };
+    binutils-arm = pkgs.callPackage binutils { platform = "arm"; target = "armv7-unknown-linux-gnueabihf"; };
+
+    # FSBL configuration supplied by Vivado 2020.1 for these boards:
+    fsblTargets = ["zc702" "zc706" "zed"];
+    sat_variants = [
+      # kasli-soc satellite variants
+      "satellite"
+      # zc706 satellite variants
+      "nist_clock_satellite" "nist_qc2_satellite" "acpki_nist_clock_satellite" "acpki_nist_qc2_satellite" 
+      "nist_clock_satellite_100mhz" "nist_qc2_satellite_100mhz" "acpki_nist_clock_satellite_100mhz" "acpki_nist_qc2_satellite_100mhz"
+    ];
+    board-package-set = { target, variant, json ? null }: let
+      szl = zynqpkgs."${target}-szl";
+      fsbl = zynqpkgs."${target}-fsbl";
+      fwtype = if builtins.elem variant sat_variants then "satman" else "runtime";
+
+      firmware = rustPlatform.buildRustPackage rec {
+        name = "firmware";
+        src = ./src;
+        cargoLock = { 
+          lockFile = src/Cargo.lock;
+          outputHashes = {
+              "tar-no-std-0.1.8" = "sha256-xm17108v4smXOqxdLvHl9CxTCJslmeogjm4Y87IXFuM=";
+              "nalgebra-0.32.6" = "sha256-L/YudkVOtfGYoNQKBD7LMk/sMYgRDzPDdpGL5rO7G2I=";
+            };
+        };
+
+        nativeBuildInputs = [
+          pkgs.gnumake
+          (pkgs.python3.withPackages(ps: [ ps.jsonschema artiqpkgs.migen migen-axi artiqpkgs.misoc artiqpkgs.artiq ]))
+          zynqpkgs.cargo-xbuild
+          llvmPackages_11.llvm
+          llvmPackages_11.clang-unwrapped
+        ];
+        buildPhase = ''
+          export XARGO_RUST_SRC="${rust}/lib/rustlib/src/rust/library"
+          export CLANG_EXTRA_INCLUDE_DIR="${llvmPackages_11.clang-unwrapped.lib}/lib/clang/11.1.0/include"
+          export CARGO_HOME=$(mktemp -d cargo-home.XXX)
+          export ZYNQ_RS=${zynq-rs}
+          make TARGET=${target} GWARGS="${if json == null then "-V ${variant}" else json}" ${fwtype}
+        '';
+
+        installPhase = ''
+          mkdir -p $out $out/nix-support
+          cp ../build/${fwtype}.bin $out/${fwtype}.bin
+          cp ../build/firmware/armv7-none-eabihf/release/${fwtype} $out/${fwtype}.elf
+          echo file binary-dist $out/${fwtype}.bin >> $out/nix-support/hydra-build-products
+          echo file binary-dist $out/${fwtype}.elf >> $out/nix-support/hydra-build-products
+        '';
+
+        doCheck = false;
+        dontFixup = true;
+        auditable = false;
+      };
+      gateware = pkgs.runCommand "${target}-${variant}-gateware"
+        {
+          nativeBuildInputs = [ 
+            (pkgs.python3.withPackages(ps: [ ps.jsonschema artiqpkgs.migen migen-axi artiqpkgs.misoc artiqpkgs.artiq ]))
+            artiqpkgs.vivado
+          ];
+        }
+        ''
+          python ${./src/gateware}/${target}.py -g build ${if json == null then "-V ${variant}" else json}
+          mkdir -p $out $out/nix-support
+          cp build/top.bit $out
+          echo file binary-dist $out/top.bit >> $out/nix-support/hydra-build-products
+        '';
+
+      # SZL startup
+      jtag = pkgs.runCommand "${target}-${variant}-jtag" {}
+        ''
+          mkdir $out
+          ln -s ${szl}/szl.elf $out
+          ln -s ${firmware}/${fwtype}.bin $out
+          ln -s ${gateware}/top.bit $out
+        '';
+      sd = pkgs.runCommand "${target}-${variant}-sd"
+        {
+          buildInputs = [ zynqpkgs.mkbootimage ];
+        }
+        ''
+          # Do not use "long" paths in boot.bif, because embedded developers
+          # can't write software (mkbootimage will segfault).
+          bifdir=`mktemp -d`
+          cd $bifdir
+          ln -s ${szl}/szl.elf szl.elf
+          ln -s ${firmware}/${fwtype}.elf ${fwtype}.elf
+          ln -s ${gateware}/top.bit top.bit
+          cat > boot.bif << EOF
+          the_ROM_image:
+          {
+            [bootloader]szl.elf
+            top.bit
+            ${fwtype}.elf
+          }
+          EOF
+          mkdir $out $out/nix-support
+          mkbootimage boot.bif $out/boot.bin
+          echo file binary-dist $out/boot.bin >> $out/nix-support/hydra-build-products
+        '';
+
+      # FSBL startup
+      fsbl-sd = pkgs.runCommand "${target}-${variant}-fsbl-sd"
+        {
+          buildInputs = [ zynqpkgs.mkbootimage ];
+        }
+        ''
+          bifdir=`mktemp -d`
+          cd $bifdir
+          ln -s ${fsbl}/fsbl.elf fsbl.elf
+          ln -s ${gateware}/top.bit top.bit
+          ln -s ${firmware}/${fwtype}.elf ${fwtype}.elf
+          cat > boot.bif << EOF
+          the_ROM_image:
+          {
+            [bootloader]fsbl.elf
+            top.bit
+            ${fwtype}.elf
+          }
+          EOF
+          mkdir $out $out/nix-support
+          mkbootimage boot.bif $out/boot.bin
+          echo file binary-dist $out/boot.bin >> $out/nix-support/hydra-build-products
+        '';
+    in {
+      "${target}-${variant}-firmware" = firmware;
+      "${target}-${variant}-gateware" = gateware;
+      "${target}-${variant}-jtag" = jtag;
+      "${target}-${variant}-sd" = sd;
+    } // (
+      if builtins.elem target fsblTargets
+      then {
+        "${target}-${variant}-fsbl-sd" = fsbl-sd;
+      }
+      else {}
+    );
+
+    gateware-sim = pkgs.stdenv.mkDerivation {
+      name = "gateware-sim";
+      
+      nativeBuildInputs = [ 
+        (pkgs.python3.withPackages(ps: [ artiqpkgs.migen migen-axi artiqpkgs.artiq ]))
+      ];
+
+      phases = [ "buildPhase" ];
+
+      buildPhase =
+        ''
+        python -m unittest discover ${self}/src/gateware -v
+        touch $out
+        '';
+    };
+
+    fmt-check = pkgs.stdenvNoCC.mkDerivation {
+      name = "fmt-check";
+
+      src = ./src;
+
+      nativeBuildInputs = [ rust pkgs.gnumake ];
+
+      phases = [ "unpackPhase" "buildPhase" ];
+
+      buildPhase =
+        ''
+        export ZYNQ_RS=${zynq-rs}
+        make manifests
+        cargo fmt -- --check
+        touch $out
+        '';
+    };
+
+    # for hitl-tests
+    zc706-nist_qc2 = (board-package-set { target = "zc706"; variant = "nist_qc2"; });
+    zc706-hitl-tests = pkgs.stdenv.mkDerivation {
+      name = "zc706-hitl-tests";
+
+      __networked = true;  # compatibility with old patched Nix
+      # breaks hydra, https://github.com/NixOS/hydra/issues/1216
+      #__impure = true;     # Nix 2.8+
+
+      buildInputs = [
+        pkgs.netcat pkgs.openssh pkgs.rsync artiqpkgs.artiq artiq-netboot zynqpkgs.zc706-szl
+      ];
+      phases = [ "buildPhase" ];
+
+      buildPhase =
+        ''
+        export NIX_SSHOPTS="-F /dev/null -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o LogLevel=ERROR -i /opt/hydra_id_ed25519"
+        LOCKCTL=$(mktemp -d)
+        mkfifo $LOCKCTL/lockctl
+
+        cat $LOCKCTL/lockctl | ${pkgs.openssh}/bin/ssh \
+        $NIX_SSHOPTS \
+        rpi-4 \
+        'mkdir -p /tmp/board_lock && flock /tmp/board_lock/zc706-1 -c "echo Ok; cat"' \
+        | (
+          # End remote flock via FIFO
+          atexit_unlock() {
+            echo > $LOCKCTL/lockctl
+          }
+          trap atexit_unlock EXIT
+
+          # Read "Ok" line when remote successfully locked
+          read LOCK_OK
+
+          echo Power cycling board...
+          (echo b; sleep 5; echo B; sleep 5) | nc -N -w6 192.168.1.31 3131
+          echo Power cycle done.
+
+          export USER=hydra
+          export OPENOCD_ZYNQ=${zynq-rs}/openocd
+          export SZL=${zynqpkgs.szl}
+          bash ${self}/remote_run.sh -h rpi-4 -o "$NIX_SSHOPTS" -d ${zc706-nist_qc2.zc706-nist_qc2-jtag}
+
+          echo Waiting for the firmware to boot...
+          sleep 15
+
+          echo Running test kernel...
+          artiq_run --device-db ${self}/examples/device_db.py ${self}/examples/mandelbrot.py
+
+          echo Running ARTIQ unit tests...
+          export ARTIQ_ROOT=${self}/examples
+          export ARTIQ_LOW_LATENCY=1
+          python -m unittest discover artiq.test.coredevice -v
+
+          touch $out
+
+          echo Completed
+
+          (echo b; sleep 5) | nc -N -w6 192.168.1.31 3131
+          echo Board powered off
+        )
+        '';
+    };
+  in rec {
+    packages.x86_64-linux =
+      {
+        inherit fastnumbers artiq-netboot ramda migen-axi binutils-arm;
+      } //
+      (board-package-set { target = "zc706"; variant = "nist_clock"; }) //
+      (board-package-set { target = "zc706"; variant = "nist_clock_master"; }) //
+      (board-package-set { target = "zc706"; variant = "nist_clock_master_100mhz"; }) //
+      (board-package-set { target = "zc706"; variant = "nist_clock_satellite"; }) //
+      (board-package-set { target = "zc706"; variant = "nist_clock_satellite_100mhz"; }) //
+      (board-package-set { target = "zc706"; variant = "nist_qc2"; }) //
+      (board-package-set { target = "zc706"; variant = "nist_qc2_master"; }) //
+      (board-package-set { target = "zc706"; variant = "nist_qc2_master_100mhz"; }) //
+      (board-package-set { target = "zc706"; variant = "nist_qc2_satellite"; }) //
+      (board-package-set { target = "zc706"; variant = "nist_qc2_satellite_100mhz"; }) //
+      (board-package-set { target = "zc706"; variant = "acpki_nist_clock"; }) //
+      (board-package-set { target = "zc706"; variant = "acpki_nist_clock_master"; }) //
+      (board-package-set { target = "zc706"; variant = "acpki_nist_clock_master_100mhz"; }) //
+      (board-package-set { target = "zc706"; variant = "acpki_nist_clock_satellite"; }) //
+      (board-package-set { target = "zc706"; variant = "acpki_nist_clock_satellite_100mhz"; }) //
+      (board-package-set { target = "zc706"; variant = "acpki_nist_qc2"; }) //
+      (board-package-set { target = "zc706"; variant = "acpki_nist_qc2_master"; }) //
+      (board-package-set { target = "zc706"; variant = "acpki_nist_qc2_master_100mhz"; }) //
+      (board-package-set { target = "zc706"; variant = "acpki_nist_qc2_satellite"; }) //
+      (board-package-set { target = "zc706"; variant = "acpki_nist_qc2_satellite_100mhz"; }) //
+      (board-package-set { target = "kasli_soc"; variant = "demo"; json = ./demo.json; }) //
+      (board-package-set { target = "kasli_soc"; variant = "master"; json = ./kasli-soc-master.json; }) //
+      (board-package-set { target = "kasli_soc"; variant = "satellite"; json = ./kasli-soc-satellite.json; }) //
+      (board-package-set { target = "ebaz4205"; variant = "base"; });
+
+    hydraJobs = packages.x86_64-linux // { inherit zc706-hitl-tests; inherit gateware-sim; inherit fmt-check; };
+
+    devShell.x86_64-linux = pkgs.mkShell {
+      name = "artiq-zynq-dev-shell";
+      buildInputs = with pkgs; [
+        rust
+        llvmPackages_11.llvm
+        llvmPackages_11.clang-unwrapped
+        gnumake
+        cacert
+        zynqpkgs.cargo-xbuild
+        zynqpkgs.mkbootimage
+        openocd  
+        openssh rsync
+        (python3.withPackages(ps: (with artiqpkgs; [ migen migen-axi misoc artiq artiq-netboot ps.jsonschema ps.pyftdi ])))
+        artiqpkgs.artiq
+        artiqpkgs.vivado
+        binutils-arm
+        pre-commit
+      ];
+      XARGO_RUST_SRC = "${rust}/lib/rustlib/src/rust/library";
+      CLANG_EXTRA_INCLUDE_DIR = "${llvmPackages_11.clang-unwrapped.lib}/lib/clang/11.1.0/include";
+      ZYNQ_RS = "${zynq-rs}";
+      OPENOCD_ZYNQ = "${zynq-rs}/openocd";
+      SZL = "${zynqpkgs.szl}";
+    };
+
+    makeArtiqZynqPackage = board-package-set;
+
+  };
+}

kasli-soc-master.json

@@ -0,0 +1,60 @@
+{
+    "target": "kasli_soc",
+    "variant": "master",
+    "hw_rev": "v1.0",
+    "base": "master",
+    "peripherals": [
+        {
+            "type": "grabber",
+            "ports": [0]
+        },
+        {
+            "type": "dio",
+            "ports": [1],
+            "bank_direction_low": "input",
+            "bank_direction_high": "output"
+        },
+        {
+            "type": "dio",
+            "ports": [2],
+            "bank_direction_low": "output",
+            "bank_direction_high": "output"
+        },
+        {
+            "type": "urukul",
+            "dds": "ad9910",
+            "ports": [3, 4],
+            "clk_sel": 2
+        },
+        {
+            "type": "zotino",
+            "ports": [5]
+        },
+        {
+            "type": "sampler",
+            "ports": [6, 7]
+        },
+        {
+            "type": "mirny",
+            "ports": [8],
+            "clk_sel": 1,
+            "refclk": 125e6
+        },
+        {
+            "type": "fastino",
+            "ports": [9]
+        },
+        {
+            "type": "dio",
+            "ports": [10],
+            "bank_direction_low": "input",
+            "bank_direction_high": "input"
+        },
+        {
+            "type": "dio",
+            "ports": [11],
+            "bank_direction_low": "output",
+            "bank_direction_high": "input"
+        }
+    ]
+}

kasli-soc-satellite.json

@@ -0,0 +1,60 @@
+{
+    "target": "kasli_soc",
+    "variant": "satellite",
+    "hw_rev": "v1.0",
+    "base": "satellite",
+    "peripherals": [
+        {
+            "type": "grabber",
+            "ports": [0]
+        },
+        {
+            "type": "dio",
+            "ports": [1],
+            "bank_direction_low": "input",
+            "bank_direction_high": "output"
+        },
+        {
+            "type": "dio",
+            "ports": [2],
+            "bank_direction_low": "output",
+            "bank_direction_high": "output"
+        },
+        {
+            "type": "urukul",
+            "dds": "ad9910",
+            "ports": [3, 4],
+            "clk_sel": 2
+        },
+        {
+            "type": "zotino",
+            "ports": [5]
+        },
+        {
+            "type": "sampler",
+            "ports": [6, 7]
+        },
+        {
+            "type": "mirny",
+            "ports": [8],
+            "clk_sel": 1,
+            "refclk": 125e6
+        },
+        {
+            "type": "fastino",
+            "ports": [9]
+        },
+        {
+            "type": "dio",
+            "ports": [10],
+            "bank_direction_low": "input",
+            "bank_direction_high": "input"
+        },
+        {
+            "type": "dio",
+            "ports": [11],
+            "bank_direction_low": "output",
+            "bank_direction_high": "input"
+        }
+    ]
+}

local_run.sh

@@ -2,10 +2,21 @@
 
 set -e
 
+if [ -z "$OPENOCD_ZYNQ" ]; then
+    echo "OPENOCD_ZYNQ environment variable must be set"
+    exit 1
+fi
+if [ -z "$SZL" ]; then
+    echo "SZL environment variable must be set"
+    exit 1
+fi
+
 impure=0
 load_bitstream=1
+board_type="kasli_soc"
+fw_type="runtime"
 
-while getopts "h:il" opt; do
+while getopts "ilb:t:f:" opt; do
     case "$opt" in
     \?) exit 1
         ;;
@@ -13,20 +24,38 @@ while getopts "h:il" opt; do
         ;;
     l)  load_bitstream=0
         ;;
+    b)  board_host=$OPTARG
+        ;;
+    t)  board_type=$OPTARG
+        ;;
+    f)  fw_type=$OPTARG
+        ;;
     esac
 done
 
+if [ -z "$board_host" ]; then
+    case $board_type in
+    kasli_soc) board_host="192.168.1.56";;
+    zc706) board_host="192.168.1.52";;
+    *) echo "Unknown board type"; exit 1;;
+    esac
+fi
+
 load_bitstream_cmd=""
 
-cd openocd
+build_dir=`pwd`/build
+result_dir=`pwd`/result
+cd $OPENOCD_ZYNQ
+openocd -f $board_type.cfg -c "load_image $SZL/szl-$board_type.elf; resume 0; exit"
+sleep 5
 if [ $impure -eq 1 ]; then
     if [ $load_bitstream -eq 1 ]; then
-        load_bitstream_cmd="pld load 0 ../build/gateware/top.bit;"
+        load_bitstream_cmd="-g $build_dir/gateware/top.bit"
     fi
-    openocd -f zc706.cfg -c "$load_bitstream_cmd load_image ../build/firmware/armv7-none-eabihf/release/szl; resume 0; exit"
+    artiq_netboot $load_bitstream_cmd -f $build_dir/$fw_type.bin -b $board_host
 else
     if [ $load_bitstream -eq 1 ]; then
-        load_bitstream_cmd="pld load 0 ../result/top.bit;"
+        load_bitstream_cmd="-g $result_dir/top.bit"
     fi
-    openocd -f zc706.cfg -c "$load_bitstream_cmd load_image ../result/szl.elf; resume 0; exit"
-fi
+    artiq_netboot $load_bitstream_cmd -f $result_dir/$fw_type.bin -b $board_host
+fi

mkbootimage.nix

@@ -1,24 +0,0 @@
-{ pkgs }:
-
-pkgs.stdenv.mkDerivation {
-  pname = "mkbootimage";
-  version = "2.2";
-
-  src = pkgs.fetchFromGitHub {
-    owner = "antmicro";
-    repo = "zynq-mkbootimage";
-    rev = "4ee42d782a9ba65725ed165a4916853224a8edf7";
-    sha256 = "1k1mbsngqadqihzjgvwvsrkvryxy5ladpxd9yh9iqn2s7fxqwqa9";
-  };
-
-  propagatedBuildInputs = [ pkgs.libelf pkgs.pcre ];
-  patchPhase =
-    ''
-    substituteInPlace Makefile --replace "git rev-parse --short HEAD" "echo nix"
-    '';
-  installPhase =
-    ''
-    mkdir -p $out/bin
-    cp mkbootimage $out/bin
-    '';
-}

openocd/digilent_jtag_smt2_nc.cfg

@@ -1,18 +0,0 @@
-#
-# Digilent JTAG-SMT2-NC
-#
-# http://store.digilentinc.com/jtag-smt2-nc-surface-mount-programming-module/
-# https://reference.digilentinc.com/_media/jtag_smt2nc/jtag-smt2-nc_rm.pdf
-#
-# Based on reference sheet (above) and Xilinx KCU105 schematics
-# https://www.xilinx.com/products/boards-and-kits/kcu105.html#documentation
-#
-# Note that the digilent_jtag_smt2 layout does not work and hangs while
-# the ftdi_device_desc from digilent_hs2 is wrong.
-
-interface ftdi
-ftdi_device_desc "Digilent USB Device"
-ftdi_vid_pid 0x0403 0x6014
-ftdi_channel 0
-ftdi_layout_init 0x00e8 0x60eb
-ftdi_layout_signal nSRST -data 0x2000

openocd/zc706.cfg

@@ -1,40 +0,0 @@
-source [find interface/ftdi/olimex-arm-usb-tiny-h.cfg]
-adapter_khz 1000
-
-set PL_TAPID 0x23731093
-set SMP 1
-
-source ./zynq-7000.cfg
-
-reset_config srst_only srst_open_drain
-adapter_nsrst_assert_width 250
-adapter_nsrst_delay 400
-
-set XC7_JSHUTDOWN 0x0d
-set XC7_JPROGRAM 0x0b
-set XC7_JSTART 0x0c
-set XC7_BYPASS 0x3f
-
-proc xc7_program {tap} {
-    global XC7_JSHUTDOWN XC7_JPROGRAM XC7_JSTART XC7_BYPASS
-    irscan $tap $XC7_JSHUTDOWN
-    irscan $tap $XC7_JPROGRAM
-    runtest 60000
-    #JSTART prevents this from working...
-    #irscan $tap $XC7_JSTART
-    runtest 2000
-    irscan $tap $XC7_BYPASS
-    runtest 2000
-}
-
-pld device virtex2 zynq.tap 1
-init
-xc7_program zynq.tap
-
-reset halt
-
-# Disable MMU
-targets $_TARGETNAME_1
-arm mcr 15 0 1 0 0 [expr [arm mrc 15 0 1 0 0] & ~0xd]
-targets $_TARGETNAME_0
-arm mcr 15 0 1 0 0 [expr [arm mrc 15 0 1 0 0] & ~0xd]

openocd/zynq-7000.cfg

@@ -1,95 +0,0 @@
-#
-# Xilinx Zynq 7000 SoC
-#
-#  Chris Johns <chrisj@rtems.org>
-#
-# Setup
-# -----
-#
-# Create a user configuration following the "Configuration Basics" in the user
-# documentation. In the file have:
-#
-#    source [find interface/ftdi/flyswatter2.cfg]
-#    source [find board/zynq-zc706-eval.cfg]
-#    adapter_khz 2000
-#    init
-#
-
-if { [info exists CHIPNAME] } {
-    global _CHIPNAME
-    set _CHIPNAME $CHIPNAME
-} else {
-    global _CHIPNAME
-    set _CHIPNAME zynq
-}
-
-if { [info exists ENDIAN] } {
-    set _ENDIAN $ENDIAN
-} else {
-    # this defaults to a bigendian
-    set _ENDIAN little
-}
-
-if { [info exists SMP] } {
-    global _SMP
-    set _SMP 1
-} else {
-    global _SMP
-    set _SMP 0
-}
-
-#
-# PL Tap.
-#
-# See ug585 ZYNQ-7000 TRM PSS_IDCODE for how this number is constructed.
-#   0x03731093 - ZC706 Eval board 1.1
-#   0x23731093 - ??
-#   0x23727093 - Zedboard Rev. C and D
-#
-# Set in your configuration file or board specific file.
-#
-if { [info exists PL_TAPID] } {
-    set _PL_TAPID $PL_TAPID
-} else {
-    set _PL_TAPID 0x03731093
-}
-
-jtag newtap $_CHIPNAME tap -irlen 6 -ircapture 0x001 -irmask 0x003 \
-    -expected-id $_PL_TAPID
-
-#
-# CoreSight Debug Access Port
-#
-if { [info exists DAP_TAPID] } {
-    set _DAP_TAPID $DAP_TAPID
-} else {
-    set _DAP_TAPID 0x4ba00477
-}
-
-jtag newtap $_CHIPNAME dap -irlen 4 -ircapture 0x01 -irmask 0x03 \
-    -expected-id $_DAP_TAPID
-
-#
-# GDB target: Cortex-A9, using DAP, configuring only one core
-# Base addresses of cores:
-# core 0  -  0xF8890000
-# core 1  -  0xF8892000
-#
-# Read from the ROM table with the patch to read the nested table.
-#
-
-set _TARGETNAME_0 $_CHIPNAME.cpu.0
-set _TARGETNAME_1 $_CHIPNAME.cpu.1
-
-target create $_TARGETNAME_0 cortex_a -coreid 0 \
-    -endian $_ENDIAN \
-    -chain-position $_CHIPNAME.dap \
-    -dbgbase 0x80090000
-if { $_SMP } {
-    echo "Zynq CPU1."
-    target create $_TARGETNAME_1 cortex_a -coreid 1 \
-        -endian $_ENDIAN \
-        -chain-position $_CHIPNAME.dap \
-        -dbgbase 0x80092000
-    target smp $_TARGETNAME_0 $_TARGETNAME_1
-}

remote_run.sh

@@ -1,15 +1,28 @@
 #!/usr/bin/env bash
 
+# Only ZC706 supported for now.
+
 set -e
 
+if [ -z "$OPENOCD_ZYNQ" ]; then
+    echo "OPENOCD_ZYNQ environment variable must be set"
+    exit 1
+fi
+if [ -z "$SZL" ]; then
+    echo "SZL environment variable must be set"
+    exit 1
+fi
+
 target_host="rpi-4.m-labs.hk"
 impure=0
 pure_dir="result"
 impure_dir="build"
 sshopts=""
 load_bitstream=1
+board_host="192.168.1.52"
+fw_type="runtime"
 
-while getopts "h:id:o:l" opt; do
+while getopts "h:id:o:lt:" opt; do
     case "$opt" in
     \?) exit 1
         ;;
@@ -24,29 +37,33 @@ while getopts "h:id:o:l" opt; do
         ;;
     l)  load_bitstream=0
         ;;
+    b)  board_host=$OPTARG
+        ;;
+    t)  fw_type=$OPTARG
+        ;;
     esac
 done
 
 target_folder="/tmp/zynq-$USER"
 load_bitstream_cmd=""
-if [ $load_bitstream -eq 1 ]; then
-    load_bitstream_cmd="pld load 0 top.bit;"
-fi
 
 echo "Creating $target_folder..."
 ssh $sshopts $target_host "mkdir -p $target_folder"
 echo "Copying files..."
-rsync -e "ssh $sshopts" openocd/* $target_host:$target_folder
+rsync -e "ssh $sshopts" -Lc $OPENOCD_ZYNQ/* $target_host:$target_folder
+rsync -e "ssh $sshopts" -Lc $SZL/szl-zc706.elf $target_host:$target_folder/szl.elf
 if [ $impure -eq 1 ]; then
-    rsync -e "ssh $sshopts" $impure_dir/firmware/armv7-none-eabihf/release/szl $target_host:$target_folder/szl.elf
     if [ $load_bitstream -eq 1 ]; then
-        rsync -e "ssh $sshopts" $impure_dir/gateware/top.bit $target_host:$target_folder
+        load_bitstream_cmd="-g build/gateware/top.bit"
     fi
+    firmware="build/$fw_type.bin"
 else
-    rsync -e "ssh $sshopts" -Lc $pure_dir/szl.elf $target_host:$target_folder
     if [ $load_bitstream -eq 1 ]; then
-        rsync -e "ssh $sshopts" -Lc $pure_dir/top.bit $target_host:$target_folder
+        load_bitstream_cmd="-g $pure_dir/top.bit"
     fi
+    firmware="$pure_dir/$fw_type.bin"    
 fi
 echo "Programming board..."
-ssh $sshopts $target_host "cd $target_folder; openocd -f zc706.cfg -c'$load_bitstream_cmd load_image szl.elf; resume 0; exit'"
+ssh $sshopts $target_host "cd $target_folder; openocd -f zc706.cfg -c'load_image szl.elf; resume 0; exit'"
+sleep 5
+artiq_netboot $load_bitstream_cmd -f $firmware -b $board_host

rustPlatform.nix

@@ -1,24 +0,0 @@
-{ pkgs }:
-
-let
-  rustcSrc = pkgs.fetchgit {
-    url = "https://github.com/rust-lang/rust.git";
-    # master of 2020-04-10
-    rev = "94d346360da50f159e0dc777dc9bc3c5b6b51a00";
-    sha256 = "1hcqdz4w2vqb12rrqqcjbfs5s0w4qwjn7z45d1zh0fzncdcf6f7d";
-    fetchSubmodules = true;
-  };
-  rustManifest = ./channel-rust-nightly.toml;
-
-  targets = [];
-  rustChannelOfTargets = _channel: _date: targets:
-    (pkgs.lib.rustLib.fromManifestFile rustManifest {
-      inherit (pkgs) stdenv fetchurl patchelf;
-    }).rust.override { inherit targets; };
-  rust =
-    rustChannelOfTargets "nightly" null targets;
-in
-  pkgs.recurseIntoAttrs (pkgs.makeRustPlatform {
-    rustc = rust // { src = rustcSrc; };
-    cargo = rust;
-  })

shell.nix

@@ -1,31 +0,0 @@
-let
-  mozillaOverlay = import (builtins.fetchTarball "https://github.com/mozilla/nixpkgs-mozilla/archive/master.tar.gz");
-  pkgs = import <nixpkgs> { overlays = [ mozillaOverlay ]; };
-  artiq-fast = <artiq-fast>;
-  rustPlatform = (import ./rustPlatform.nix { inherit pkgs; });
-  artiqpkgs = import "${artiq-fast}/default.nix" { inherit pkgs; };
-  vivado = import "${artiq-fast}/vivado.nix" { inherit pkgs; };
-in
-  pkgs.stdenv.mkDerivation {
-    name = "artiq-zynq-env";
-    buildInputs = [
-      pkgs.gnumake
-      rustPlatform.rust.rustc
-      rustPlatform.rust.cargo
-      pkgs.llvmPackages_9.llvm
-      pkgs.llvmPackages_9.clang-unwrapped
-      pkgs.cacert
-      pkgs.cargo-xbuild
-
-      pkgs.openocd
-      pkgs.openssh pkgs.rsync
-
-      (pkgs.python3.withPackages(ps: (with artiqpkgs; [ migen migen-axi misoc artiq ])))
-      vivado
-      artiqpkgs.binutils-arm
-
-      (import ./mkbootimage.nix { inherit pkgs; })
-    ];
-
-    XARGO_RUST_SRC = "${rustPlatform.rust.rustc.src}/src";
-  }

src/.clang-format

@@ -0,0 +1,32 @@
+BasedOnStyle: LLVM
+
+Language: Cpp
+Standard: Cpp11
+
+AccessModifierOffset: -1
+AlignEscapedNewlines: Left
+AlwaysBreakAfterReturnType: None
+AlwaysBreakTemplateDeclarations: Yes
+AllowAllParametersOfDeclarationOnNextLine: false
+AllowShortFunctionsOnASingleLine: Inline
+BinPackParameters: false
+BreakBeforeBinaryOperators: NonAssignment
+BreakBeforeTernaryOperators: true
+BreakConstructorInitializers: AfterColon
+BreakInheritanceList: AfterColon
+ColumnLimit: 120
+ConstructorInitializerAllOnOneLineOrOnePerLine: true
+ContinuationIndentWidth: 4
+DerivePointerAlignment: false
+IndentCaseLabels: true
+IndentPPDirectives: None
+IndentWidth: 4
+MaxEmptyLinesToKeep: 1
+PointerAlignment: Left
+ReflowComments: true
+SortIncludes: false
+SortUsingDeclarations: true
+SpaceAfterTemplateKeyword: false
+SpacesBeforeTrailingComments: 2
+TabWidth: 4
+UseTab: Never

src/.clippy.toml

@@ -0,0 +1 @@
+doc-valid-idents = ["CPython", "NumPy", ".."]

src/.pre-commit-config.yaml

@@ -0,0 +1,32 @@
+# See https://pre-commit.com for more information
+# See https://pre-commit.com/hooks.html for more hooks
+
+default_stages: [commit]
+
+repos:
+  - repo: local
+    hooks:
+      - id: cargo-fmt
+        name: artiq-zynq cargo format
+        entry: nix
+        language: system
+        types: [file, rust]
+        pass_filenames: false
+        description: Runs cargo fmt on the codebase.
+        args: [develop, -c, cargo, fmt, --manifest-path, src/Cargo.toml, --all]
+      - id: cargo-clippy
+        name: artiq-zynq cargo clippy
+        entry: nix
+        language: system
+        types: [file, rust]
+        pass_filenames: false
+        description: Runs cargo clippy on the codebase.
+        args: [develop, -c, cargo, clippy, --manifest-path, src/Cargo.toml, --tests]
+  - repo: https://github.com/pre-commit/mirrors-clang-format
+    rev: v19.1.0
+    hooks:
+      - id: clang-format
+        name: artiq-zynq clang-format
+        description: Runs clang-format on the codebase.
+        files: \.(cpp|h|hpp|c)$
+        args: [-style=file, -fallback-style=none, -assume-filename=src/.clang-format]

src/Cargo.lock

@@ -1,10 +1,27 @@
 # This file is automatically @generated by Cargo.
 # It is not intended for manual editing.
+version = 3
+
+[[package]]
+name = "approx"
+version = "0.5.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "cab112f0a86d568ea0e627cc1d6be74a1e9cd55214684db5561995f6dad897c6"
+dependencies = [
+ "num-traits",
+]
+
+[[package]]
+name = "arrayvec"
+version = "0.7.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "96d30a06541fbafbc7f82ed10c06164cfbd2c401138f6addd8404629c4b16711"
+
 [[package]]
 name = "async-recursion"
-version = "0.3.1"
+version = "0.3.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e5444eec77a9ec2bfe4524139e09195862e981400c4358d3b760cae634e4c4ee"
+checksum = "d7d78656ba01f1b93024b7c3a0467f1608e4be67d725749fdcd7d2c7678fd7a2"
 dependencies = [
  "proc-macro2",
  "quote",
@@ -13,33 +30,43 @@ dependencies = [
 
 [[package]]
 name = "autocfg"
-version = "1.0.0"
+version = "1.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f8aac770f1885fd7e387acedd76065302551364496e46b3dd00860b2f8359b9d"
+checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa"
 
 [[package]]
 name = "bit_field"
-version = "0.10.0"
+version = "0.10.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a165d606cf084741d4ac3a28fb6e9b1eb0bd31f6cd999098cfddb0b2ab381dc0"
+checksum = "dcb6dd1c2376d2e096796e234a70e17e94cc2d5d54ff8ce42b28cef1d0d359a4"
 
 [[package]]
 name = "bitflags"
-version = "1.2.1"
+version = "1.3.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "cf1de2fe8c75bc145a2f577add951f8134889b4795d47466a54a5c846d691693"
+checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a"
+
+[[package]]
+name = "build_const"
+version = "0.2.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b4ae4235e6dac0694637c763029ecea1a2ec9e4e06ec2729bd21ba4d9c863eb7"
+
+[[package]]
+name = "build_zynq"
+version = "0.0.0"
 
 [[package]]
 name = "byteorder"
-version = "1.3.4"
+version = "1.4.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "08c48aae112d48ed9f069b33538ea9e3e90aa263cfa3d1c24309612b1f7472de"
+checksum = "14c189c53d098945499cdfa7ecc63567cf3886b3332b312a5b4585d8d3a6a610"
 
 [[package]]
 name = "cc"
-version = "1.0.57"
+version = "1.0.77"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0fde55d2a2bfaa4c9668bbc63f531fbdeee3ffe188f4662511ce2c22b3eedebe"
+checksum = "e9f73505338f7d905b19d18738976aae232eb46b8efc15554ffc56deb5d9ebe4"
 
 [[package]]
 name = "cfg-if"
@@ -48,16 +75,33 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "4785bdd1c96b2a846b2bd7cc02e86b6b3dbf14e7e53446c4f54c92a361040822"
 
 [[package]]
-name = "compiler_builtins"
-version = "0.1.32"
+name = "cfg-if"
+version = "1.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7bc4ac2c824d2bfc612cba57708198547e9a26943af0632aff033e0693074d5c"
+checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd"
+
+[[package]]
+name = "compiler_builtins"
+version = "0.1.39"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3748f82c7d366a0b4950257d19db685d4958d2fa27c6d164a3f069fec42b748b"
 
 [[package]]
 name = "core_io"
-version = "0.1.20200410"
+version = "0.1.20210325"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "97f8932064288cc79feb4d343a399d353a6f6f001e586ece47fe518a9e8507df"
 dependencies = [
- "memchr",
+ "rustc_version",
+]
+
+[[package]]
+name = "crc"
+version = "1.8.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d663548de7f5cca343f1e0a48d14dcfb0e9eb4e079ec58883b7251539fa10aeb"
+dependencies = [
+ "build_const",
 ]
 
 [[package]]
@@ -66,28 +110,13 @@ version = "0.3.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "0f8cb7306107e4b10e64994de6d3274bd08996a7c1322a27b86482392f96be0a"
 
-[[package]]
-name = "cstr_core"
-version = "0.2.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8758514b5f03968703f1db1f1e196e031d5268f5295ff99a5bf345008790ba85"
-dependencies = [
- "cty",
- "memchr",
-]
-
-[[package]]
-name = "cty"
-version = "0.2.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7313c0d620d0cb4dbd9d019e461a4beb501071ff46ec0ab933efb4daa76d73e3"
-
 [[package]]
 name = "dwarf"
 version = "0.0.0"
 dependencies = [
- "cfg-if",
+ "cfg-if 0.1.10",
  "compiler_builtins",
+ "cslice",
  "libc",
  "unwind",
 ]
@@ -96,24 +125,25 @@ dependencies = [
 name = "dyld"
 version = "0.1.0"
 dependencies = [
+ "libcortex_a9",
  "log",
 ]
 
 [[package]]
 name = "embedded-hal"
-version = "0.2.4"
+version = "0.2.7"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "fa998ce59ec9765d15216393af37a58961ddcefb14c753b4816ba2191d865fcb"
+checksum = "35949884794ad573cf46071e41c9b60efb0cb311e3ca01f7af807af1debc66ff"
 dependencies = [
- "nb",
+ "nb 0.1.3",
  "void",
 ]
 
 [[package]]
 name = "fatfs"
-version = "0.3.3"
+version = "0.3.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f6d1df9e4503954f60504a5ee4fc435cd65cc42e98b2081f7f421be5f2e68e7d"
+checksum = "e18f80a87439240dac45d927fd8f8081b6f1e34c03e97271189fa8a8c2e96c8f"
 dependencies = [
  "bitflags",
  "byteorder",
@@ -123,9 +153,9 @@ dependencies = [
 
 [[package]]
 name = "futures"
-version = "0.3.5"
+version = "0.3.25"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1e05b85ec287aac0dc34db7d4a569323df697f9c55b99b15d6b4ef8cde49f613"
+checksum = "38390104763dc37a5145a53c29c63c1290b5d316d6086ec32c293f6736051bb0"
 dependencies = [
  "futures-channel",
  "futures-core",
@@ -137,9 +167,9 @@ dependencies = [
 
 [[package]]
 name = "futures-channel"
-version = "0.3.5"
+version = "0.3.25"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f366ad74c28cca6ba456d95e6422883cfb4b252a83bed929c83abfdbbf2967d5"
+checksum = "52ba265a92256105f45b719605a571ffe2d1f0fea3807304b522c1d778f79eed"
 dependencies = [
  "futures-core",
  "futures-sink",
@@ -147,23 +177,22 @@ dependencies = [
 
 [[package]]
 name = "futures-core"
-version = "0.3.5"
+version = "0.3.25"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "59f5fff90fd5d971f936ad674802482ba441b6f09ba5e15fd8b39145582ca399"
+checksum = "04909a7a7e4633ae6c4a9ab280aeb86da1236243a77b694a49eacd659a4bd3ac"
 
 [[package]]
 name = "futures-io"
-version = "0.3.5"
+version = "0.3.25"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "de27142b013a8e869c14957e6d2edeef89e97c289e69d042ee3a49acd8b51789"
+checksum = "00f5fb52a06bdcadeb54e8d3671f8888a39697dcb0b81b23b55174030427f4eb"
 
 [[package]]
 name = "futures-macro"
-version = "0.3.5"
+version = "0.3.25"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d0b5a30a4328ab5473878237c447333c093297bded83a4983d10f4deea240d39"
+checksum = "bdfb8ce053d86b91919aad980c220b1fb8401a9394410e1c289ed7e66b61835d"
 dependencies = [
- "proc-macro-hack",
  "proc-macro2",
  "quote",
  "syn",
@@ -171,55 +200,111 @@ dependencies = [
 
 [[package]]
 name = "futures-sink"
-version = "0.3.5"
+version = "0.3.25"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3f2032893cb734c7a05d85ce0cc8b8c4075278e93b24b66f9de99d6eb0fa8acc"
+checksum = "39c15cf1a4aa79df40f1bb462fb39676d0ad9e366c2a33b590d7c66f4f81fcf9"
 
 [[package]]
 name = "futures-task"
-version = "0.3.5"
+version = "0.3.25"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "bdb66b5f09e22019b1ab0830f7785bcea8e7a42148683f99214f73f8ec21a626"
+checksum = "2ffb393ac5d9a6eaa9d3fdf37ae2776656b706e200c8e16b1bdb227f5198e6ea"
 
 [[package]]
 name = "futures-util"
-version = "0.3.5"
+version = "0.3.25"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8764574ff08b701a084482c3c7031349104b07ac897393010494beaa18ce32c6"
+checksum = "197676987abd2f9cadff84926f410af1c183608d36641465df73ae8211dc65d6"
 dependencies = [
  "futures-core",
  "futures-macro",
  "futures-sink",
  "futures-task",
- "pin-project",
+ "pin-project-lite",
  "pin-utils",
- "proc-macro-hack",
- "proc-macro-nested",
+]
+
+[[package]]
+name = "io"
+version = "0.0.0"
+dependencies = [
+ "byteorder",
+ "core_io",
+ "libsupport_zynq",
+]
+
+[[package]]
+name = "ksupport"
+version = "0.1.0"
+dependencies = [
+ "build_zynq",
+ "byteorder",
+ "core_io",
+ "cslice",
+ "dwarf",
+ "dyld",
+ "io",
+ "libasync",
+ "libboard_artiq",
+ "libboard_zynq",
+ "libc",
+ "libconfig",
+ "libcortex_a9",
+ "libm",
+ "libregister",
+ "libsupport_zynq",
+ "log",
+ "log_buffer",
+ "nalgebra",
+ "nb 0.1.3",
+ "unwind",
+ "vcell",
+ "void",
 ]
 
 [[package]]
 name = "libasync"
 version = "0.0.0"
-source = "git+https://git.m-labs.hk/M-Labs/zc706.git#371e59cef57746e3dd4cae915be7fd3286972822"
 dependencies = [
  "embedded-hal",
  "libcortex_a9",
- "nb",
+ "nb 1.0.0",
  "pin-utils",
  "smoltcp",
 ]
 
+[[package]]
+name = "libboard_artiq"
+version = "0.0.0"
+dependencies = [
+ "build_zynq",
+ "core_io",
+ "crc",
+ "embedded-hal",
+ "io",
+ "libasync",
+ "libboard_zynq",
+ "libconfig",
+ "libcortex_a9",
+ "libregister",
+ "libsupport_zynq",
+ "log",
+ "log_buffer",
+ "nb 1.0.0",
+ "void",
+]
+
 [[package]]
 name = "libboard_zynq"
 version = "0.0.0"
-source = "git+https://git.m-labs.hk/M-Labs/zc706.git#371e59cef57746e3dd4cae915be7fd3286972822"
 dependencies = [
  "bit_field",
  "embedded-hal",
+ "libasync",
  "libcortex_a9",
  "libregister",
  "log",
- "nb",
+ "nb 0.1.3",
  "smoltcp",
  "void",
  "volatile-register",
@@ -233,19 +318,34 @@ dependencies = [
  "libboard_zynq",
 ]
 
+[[package]]
+name = "libconfig"
+version = "0.1.0"
+dependencies = [
+ "core_io",
+ "fatfs",
+ "libboard_zynq",
+ "log",
+]
+
 [[package]]
 name = "libcortex_a9"
 version = "0.0.0"
-source = "git+https://git.m-labs.hk/M-Labs/zc706.git#371e59cef57746e3dd4cae915be7fd3286972822"
 dependencies = [
  "bit_field",
  "libregister",
+ "volatile-register",
 ]
 
+[[package]]
+name = "libm"
+version = "0.2.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "348108ab3fba42ec82ff6e9564fc4ca0247bdccdc68dd8af9764bbc79c3c8ffb"
+
 [[package]]
 name = "libregister"
 version = "0.0.0"
-source = "git+https://git.m-labs.hk/M-Labs/zc706.git#371e59cef57746e3dd4cae915be7fd3286972822"
 dependencies = [
  "bit_field",
  "vcell",
@@ -255,8 +355,8 @@ dependencies = [
 [[package]]
 name = "libsupport_zynq"
 version = "0.0.0"
-source = "git+https://git.m-labs.hk/M-Labs/zc706.git#371e59cef57746e3dd4cae915be7fd3286972822"
 dependencies = [
+ "cc",
  "compiler_builtins",
  "libboard_zynq",
  "libcortex_a9",
@@ -267,17 +367,17 @@ dependencies = [
 
 [[package]]
 name = "linked_list_allocator"
-version = "0.8.4"
+version = "0.8.11"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e70e46c13c0e8374c26cec5752e3347ca1087d9711de8f45aa513a7700efd73d"
+checksum = "822add9edb1860698b79522510da17bef885171f75aa395cff099d770c609c24"
 
 [[package]]
 name = "log"
-version = "0.4.8"
+version = "0.4.17"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "14b6052be84e6b71ab17edffc2eeabf5c2c3ae1fdb464aae35ac50c67a44e1f7"
+checksum = "abb12e687cfb44aa40f41fc3978ef76448f9b6038cad6aef4259d3c095a2382e"
 dependencies = [
- "cfg-if",
+ "cfg-if 1.0.0",
 ]
 
 [[package]]
@@ -293,56 +393,94 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "c75de51135344a4f8ed3cfe2720dc27736f7711989703a0b43aadf3753c55577"
 
 [[package]]
-name = "memchr"
-version = "2.3.3"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3728d817d99e5ac407411fa471ff9800a778d88a24685968b36824eaf4bee400"
+name = "nalgebra"
+version = "0.32.6"
+source = "git+https://git.m-labs.hk/M-Labs/nalgebra.git?rev=dd00f9b#dd00f9b46046e0b931d1b470166db02fd29591be"
+dependencies = [
+ "approx",
+ "num-complex",
+ "num-rational",
+ "num-traits",
+ "simba",
+ "typenum",
+]
 
 [[package]]
 name = "nb"
-version = "0.1.2"
+version = "0.1.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b1411551beb3c11dedfb0a90a0fa256b47d28b9ec2cdff34c25a2fa59e45dbdc"
+checksum = "801d31da0513b6ec5214e9bf433a77966320625a37860f910be265be6e18d06f"
+dependencies = [
+ "nb 1.0.0",
+]
+
+[[package]]
+name = "nb"
+version = "1.0.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "546c37ac5d9e56f55e73b677106873d9d9f5190605e41a856503623648488cae"
+
+[[package]]
+name = "num-complex"
+version = "0.4.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "26873667bbbb7c5182d4a37c1add32cdf09f841af72da53318fdb81543c15085"
+dependencies = [
+ "num-traits",
+]
 
 [[package]]
 name = "num-derive"
-version = "0.3.0"
+version = "0.3.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0c8b15b261814f992e33760b1fca9fe8b693d8a65299f20c9901688636cfb746"
+checksum = "876a53fff98e03a936a674b29568b0e605f06b29372c2489ff4de23f1949743d"
 dependencies = [
  "proc-macro2",
  "quote",
  "syn",
 ]
 
+[[package]]
+name = "num-integer"
+version = "0.1.46"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7969661fd2958a5cb096e56c8e1ad0444ac2bbcd0061bd28660485a44879858f"
+dependencies = [
+ "num-traits",
+]
+
+[[package]]
+name = "num-rational"
+version = "0.4.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d41702bd167c2df5520b384281bc111a4b5efcf7fbc4c9c222c815b07e0a6a6a"
+dependencies = [
+ "autocfg",
+ "num-integer",
+ "num-traits",
+]
+
 [[package]]
 name = "num-traits"
-version = "0.2.12"
+version = "0.2.15"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ac267bcc07f48ee5f8935ab0d24f316fb722d7a1292e2913f0cc196b29ffd611"
+checksum = "578ede34cf02f8924ab9447f50c28075b4d3e5b269972345e7e0372b38c6cdcd"
 dependencies = [
  "autocfg",
+ "libm",
 ]
 
 [[package]]
-name = "pin-project"
-version = "0.4.22"
+name = "paste"
+version = "1.0.15"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "12e3a6cdbfe94a5e4572812a0201f8c0ed98c1c452c7b8563ce2276988ef9c17"
-dependencies = [
- "pin-project-internal",
-]
+checksum = "57c0d7b74b563b49d38dae00a0c37d4d6de9b432382b2892f0574ddcae73fd0a"
 
 [[package]]
-name = "pin-project-internal"
-version = "0.4.22"
+name = "pin-project-lite"
+version = "0.2.9"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6a0ffd45cf79d88737d7cc85bfd5d2894bee1139b356e616fe85dc389c61aaf7"
-dependencies = [
- "proc-macro2",
- "quote",
- "syn",
-]
+checksum = "e0a7ae3ac2f1173085d398531c705756c94a4c56843785df85a60c1a0afac116"
 
 [[package]]
 name = "pin-utils"
@@ -350,32 +488,20 @@ version = "0.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184"
 
-[[package]]
-name = "proc-macro-hack"
-version = "0.5.16"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7e0456befd48169b9f13ef0f0ad46d492cf9d2dbb918bcf38e01eed4ce3ec5e4"
-
-[[package]]
-name = "proc-macro-nested"
-version = "0.1.6"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "eba180dafb9038b050a4c280019bbedf9f2467b61e5d892dcad585bb57aadc5a"
-
 [[package]]
 name = "proc-macro2"
-version = "1.0.18"
+version = "1.0.43"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "beae6331a816b1f65d04c45b078fd8e6c93e8071771f41b8163255bbd8d7c8fa"
+checksum = "0a2ca2c61bc9f3d74d2886294ab7b9853abd9c1ad903a3ac7815c58989bb7bab"
 dependencies = [
- "unicode-xid",
+ "unicode-ident",
 ]
 
 [[package]]
 name = "quote"
-version = "1.0.7"
+version = "1.0.21"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "aa563d17ecb180e500da1cfd2b028310ac758de548efdd203e18f283af693f37"
+checksum = "bbe448f377a7d6961e30f5955f9b8d106c3f5e449d493ee1b125c1d43c2b5179"
 dependencies = [
  "proc-macro2",
 ]
@@ -391,34 +517,90 @@ name = "runtime"
 version = "0.1.0"
 dependencies = [
  "async-recursion",
+ "build_zynq",
  "byteorder",
  "core_io",
  "cslice",
  "dwarf",
  "dyld",
  "embedded-hal",
- "fatfs",
  "futures",
+ "io",
+ "ksupport",
  "libasync",
+ "libboard_artiq",
  "libboard_zynq",
  "libc",
+ "libconfig",
  "libcortex_a9",
  "libregister",
  "libsupport_zynq",
  "log",
  "log_buffer",
- "nb",
  "num-derive",
  "num-traits",
+ "tar-no-std",
  "unwind",
+ "vcell",
  "void",
 ]
 
 [[package]]
-name = "smoltcp"
-version = "0.6.0"
+name = "rustc_version"
+version = "0.1.7"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0fe46639fd2ec79eadf8fe719f237a7a0bd4dac5d957f1ca5bbdbc1c3c39e53a"
+checksum = "c5f5376ea5e30ce23c03eb77cbe4962b988deead10910c372b226388b594c084"
+dependencies = [
+ "semver",
+]
+
+[[package]]
+name = "satman"
+version = "0.0.0"
+dependencies = [
+ "build_zynq",
+ "byteorder",
+ "core_io",
+ "crc",
+ "cslice",
+ "embedded-hal",
+ "io",
+ "ksupport",
+ "libasync",
+ "libboard_artiq",
+ "libboard_zynq",
+ "libc",
+ "libconfig",
+ "libcortex_a9",
+ "libregister",
+ "libsupport_zynq",
+ "log",
+ "unwind",
+]
+
+[[package]]
+name = "semver"
+version = "0.1.20"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d4f410fedcf71af0345d7607d246e7ad15faaadd49d240ee3b24e5dc21a820ac"
+
+[[package]]
+name = "simba"
+version = "0.8.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "50582927ed6f77e4ac020c057f37a268fc6aebc29225050365aacbb9deeeddc4"
+dependencies = [
+ "approx",
+ "num-complex",
+ "num-traits",
+ "paste",
+]
+
+[[package]]
+name = "smoltcp"
+version = "0.7.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3e4a069bef843d170df47e7c0a8bf8d037f217d9f5b325865acc3e466ffe40d3"
 dependencies = [
  "bitflags",
  "byteorder",
@@ -427,48 +609,52 @@ dependencies = [
 
 [[package]]
 name = "syn"
-version = "1.0.33"
+version = "1.0.101"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e8d5d96e8cbb005d6959f119f773bfaebb5684296108fb32600c00cde305b2cd"
+checksum = "e90cde112c4b9690b8cbe810cba9ddd8bc1d7472e2cae317b69e9438c1cba7d2"
 dependencies = [
  "proc-macro2",
  "quote",
- "unicode-xid",
+ "unicode-ident",
 ]
 
 [[package]]
-name = "szl"
-version = "0.1.0"
+name = "tar-no-std"
+version = "0.1.8"
+source = "git+https://git.m-labs.hk/M-Labs/tar-no-std?rev=2ab6dc5#2ab6dc58e5249c59c4eb03eaf3a119bcdd678d32"
 dependencies = [
- "cc",
- "cstr_core",
- "libboard_zynq",
- "libcortex_a9",
- "libsupport_zynq",
+ "arrayvec",
+ "bitflags",
  "log",
 ]
 
 [[package]]
-name = "unicode-xid"
-version = "0.2.1"
+name = "typenum"
+version = "1.17.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f7fe0bb3479651439c9112f72b6c505038574c9fbb575ed1bf3b797fa39dd564"
+checksum = "42ff0bf0c66b8238c6f3b578df37d0b7848e55df8577b3f74f92a69acceeb825"
+
+[[package]]
+name = "unicode-ident"
+version = "1.0.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6ceab39d59e4c9499d4e5a8ee0e2735b891bb7308ac83dfb4e80cad195c9f6f3"
 
 [[package]]
 name = "unwind"
 version = "0.0.0"
 dependencies = [
  "cc",
- "cfg-if",
+ "cfg-if 0.1.10",
  "compiler_builtins",
  "libc",
 ]
 
 [[package]]
 name = "vcell"
-version = "0.1.2"
+version = "0.1.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "876e32dcadfe563a4289e994f7cb391197f362b6315dc45e8ba4aa6f564a4b3c"
+checksum = "77439c1b53d2303b20d9459b1ade71a83c716e3f9c34f3228c00e6f185d6c002"
 
 [[package]]
 name = "void"
@@ -478,9 +664,9 @@ checksum = "6a02e4885ed3bc0f2de90ea6dd45ebcbb66dacffe03547fadbb0eeae2770887d"
 
 [[package]]
 name = "volatile-register"
-version = "0.2.0"
+version = "0.2.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0d67cb4616d99b940db1d6bd28844ff97108b498a6ca850e5b6191a532063286"
+checksum = "9ee8f19f9d74293faf70901bc20ad067dc1ad390d2cbf1e3f75f721ffee908b6"
 dependencies = [
  "vcell",
 ]

src/Cargo.toml

@@ -2,24 +2,17 @@
 members = [
   "libc",
   "libdyld",
-  "libcoreio",
   "libdwarf",
+  "libio",
   "libunwind",
+  "libksupport",
   "runtime",
-  "szl"
+  "satman"
 ]
 
-[profile.dev]
-panic = "abort"
-lto = false
-
 [profile.release]
 panic = "abort"
 debug = true
-# Link-Time Optimization:
-# turn off if you get unusable debug symbols.
+codegen-units = 1
+opt-level = 2
 lto = true
-opt-level = 'z'  # Optimize for size.
-
-[patch.crates-io]
-core_io = { path = "./libcoreio" }

src/Makefile

@@ -1,20 +1,41 @@
-VARIANT := simple
+TARGET := zc706
+GWARGS := -V nist_clock
 
-all: ../build/firmware/armv7-none-eabihf/release/szl
+all: runtime
 
-.PHONY: all
+runtime: ../build/runtime.bin
 
+satman: ../build/satman.bin
 
-../build/pl.rs: zc706.py
+.PHONY: all manifests
+
+manifests = libboard_artiq/Cargo.toml libc/Cargo.toml libdyld/Cargo.toml libio/Cargo.toml libksupport/Cargo.toml runtime/Cargo.toml satman/Cargo.toml
+
+$(manifests): %.toml: %.toml.tpl
+	sed s+@@ZYNQ_RS@@+$(ZYNQ_RS)+g $< > $@
+
+manifests: $(manifests)
+
+../build/pl.rs ../build/rustc-cfg ../build/mem.rs: gateware/* 
 	mkdir -p ../build
-	python zc706.py -r ../build/pl.rs -V $(VARIANT)
+	python gateware/$(TARGET).py -r ../build/pl.rs -c ../build/rustc-cfg -m ../build/mem.rs $(GWARGS)
 
-../build/firmware/armv7-none-eabihf/release/runtime: ../build/pl.rs $(shell find . -path ./szl -prune -o -print)
-	XBUILD_SYSROOT_PATH=`pwd`/../build/sysroot cargo xbuild --release -p runtime --target-dir ../build/firmware
+../build/firmware/armv7-none-eabihf/release/runtime: ../build/pl.rs ../build/rustc-cfg ../build/mem.rs $(manifests) $(shell find . -type f -not -name Cargo.toml -print)
+	cd runtime && \
+	XBUILD_SYSROOT_PATH=`pwd`/../../build/sysroot \
+	cargo xbuild --release \
+		--target-dir ../../build/firmware \
+		--no-default-features --features=target_$(TARGET)
 
-../build/szl-payload.bin.lzma: ../build/firmware/armv7-none-eabihf/release/runtime
-	llvm-objcopy -O binary ../build/firmware/armv7-none-eabihf/release/runtime ../build/szl-payload.bin
-	lzma --keep -f ../build/szl-payload.bin
+../build/runtime.bin: ../build/firmware/armv7-none-eabihf/release/runtime
+	llvm-objcopy -O binary ../build/firmware/armv7-none-eabihf/release/runtime ../build/runtime.bin
 
-../build/firmware/armv7-none-eabihf/release/szl: .cargo/* armv7-none-eabihf.json Cargo.lock Cargo.toml szl/* szl/src/* ../build/szl-payload.bin.lzma
-	XBUILD_SYSROOT_PATH=`pwd`/../build/sysroot cargo xbuild --release -p szl --target-dir ../build/firmware
+../build/firmware/armv7-none-eabihf/release/satman: ../build/pl.rs ../build/rustc-cfg ../build/mem.rs $(manifests) $(shell find . -type f -not -name Cargo.toml -print)
+	cd satman && \
+	XBUILD_SYSROOT_PATH=`pwd`/../../build/sysroot \
+	cargo xbuild --release \
+		--target-dir ../../build/firmware \
+		--no-default-features --features=target_$(TARGET)
+
+../build/satman.bin: ../build/firmware/armv7-none-eabihf/release/satman
+	llvm-objcopy -O binary ../build/firmware/armv7-none-eabihf/release/satman ../build/satman.bin

src/gateware/acpki.py

@@ -0,0 +1,272 @@
+from operator import attrgetter
+
+from migen import *
+from migen.genlib.cdc import MultiReg
+from migen_axi.interconnect import axi
+from misoc.interconnect.csr import *
+
+from artiq.gateware import rtio
+
+OUT_BURST_LEN = 10
+IN_BURST_LEN = 4
+
+
+class Engine(Module, AutoCSR):
+    def __init__(self, bus, user):
+        self.addr_base = CSRStorage(32)
+        self.trig_count = CSRStatus(32)
+        self.write_count = CSRStatus(32)
+
+        self.trigger_stb = Signal()
+
+        # Dout : Data received from CPU, output by DMA module
+        # Din : Data driven into DMA module, written into CPU
+        # When stb assert, index shows word being read/written, dout/din holds
+        # data
+        #
+        # Cycle:
+        # trigger_stb pulsed at start
+        # Then out_burst_len words are strobed out of dout
+        # Then, when din_ready is high, in_burst_len words are strobed in to din
+        self.dout_stb = Signal()
+        self.din_stb = Signal()
+        self.dout_index = Signal(max=16)
+        self.din_index = Signal(max=16)
+        self.din_ready = Signal()
+        self.dout = Signal(64)
+        self.din = Signal(64)
+
+        ###
+
+        self.sync += If(self.trigger_stb, self.trig_count.status.eq(self.trig_count.status+1))
+
+        self.comb += [
+            user.aruser.eq(0x1f),
+            user.awuser.eq(0x1f)
+        ]
+
+        ar, aw, w, r, b = attrgetter("ar", "aw", "w", "r", "b")(bus)
+
+        ### Read
+        self.comb += [
+            ar.addr.eq(self.addr_base.storage),
+            self.dout.eq(r.data),
+            r.ready.eq(1),
+            ar.burst.eq(axi.Burst.incr.value),
+            ar.len.eq(OUT_BURST_LEN-1), # Number of transfers in burst (0->1 transfer, 1->2 transfers...)
+            ar.size.eq(3), # Width of burst: 3 = 8 bytes = 64 bits
+            ar.cache.eq(0xf),
+        ]
+
+        # read control
+        self.submodules.read_fsm = read_fsm = FSM(reset_state="IDLE")
+        read_fsm.act("IDLE",
+            If(self.trigger_stb,
+                ar.valid.eq(1),
+                If(ar.ready,
+                    NextState("READ")
+                ).Else(
+                    NextState("READ_START")
+                )
+            )
+        )
+        read_fsm.act("READ_START",
+            ar.valid.eq(1),
+            If(ar.ready,
+                NextState("READ"),
+            )
+        )
+        read_fsm.act("READ",
+            ar.valid.eq(0),
+            If(r.last & r.valid,
+                NextState("IDLE")
+            )
+        )
+
+        self.sync += [
+            If(read_fsm.ongoing("IDLE"),
+                self.dout_index.eq(0)
+            ).Else(If(r.valid & read_fsm.ongoing("READ"),
+                    self.dout_index.eq(self.dout_index+1)
+                )
+            )
+        ]
+
+        self.comb += self.dout_stb.eq(r.valid & r.ready)
+
+        ### Write
+        self.comb += [
+            w.data.eq(self.din),
+            aw.addr.eq(self.addr_base.storage+96),
+            w.strb.eq(0xff),
+            aw.burst.eq(axi.Burst.incr.value),
+            aw.len.eq(IN_BURST_LEN-1), # Number of transfers in burst minus 1
+            aw.size.eq(3), # Width of burst: 3 = 8 bytes = 64 bits
+            aw.cache.eq(0xf),
+            b.ready.eq(1),
+        ]
+
+        # write control
+        self.submodules.write_fsm = write_fsm = FSM(reset_state="IDLE")
+        write_fsm.act("IDLE",
+            w.valid.eq(0),
+            aw.valid.eq(0),
+            If(self.trigger_stb,
+                aw.valid.eq(1),
+                If(aw.ready, # assumes aw.ready is not randomly deasserted
+                    NextState("DATA_WAIT")
+                ).Else(
+                    NextState("AW_READY_WAIT")
+                )
+            )
+        )
+        write_fsm.act("AW_READY_WAIT",
+            aw.valid.eq(1),
+            If(aw.ready,
+                NextState("DATA_WAIT"),
+            )
+        )
+        write_fsm.act("DATA_WAIT",
+            aw.valid.eq(0),
+            If(self.din_ready,
+                w.valid.eq(1),
+                NextState("WRITE")
+            )
+        )
+        write_fsm.act("WRITE",
+            w.valid.eq(1),
+            If(w.ready & w.last,
+                NextState("IDLE")
+            )
+        )
+
+        self.sync += If(w.ready & w.valid, self.write_count.status.eq(self.write_count.status+1))
+
+        self.sync += [
+            If(write_fsm.ongoing("IDLE"),
+                self.din_index.eq(0)
+            ),
+            If(w.ready & w.valid, self.din_index.eq(self.din_index+1))
+        ]
+
+        self.comb += [
+            w.last.eq(0),
+            If(self.din_index==aw.len, w.last.eq(1))
+        ]
+
+        self.comb += self.din_stb.eq(w.valid & w.ready)
+
+
+
+class KernelInitiator(Module, AutoCSR):
+    def __init__(self, tsc, bus, user, evento):
+        # Core is disabled upon reset to avoid spurious triggering if evento toggles from e.g. boot code.
+        self.enable = CSRStorage()
+
+        self.counter = CSRStatus(64)
+        self.counter_update = CSR()
+        self.o_status = CSRStatus(3)
+        self.i_status = CSRStatus(4)
+
+        self.submodules.engine = Engine(bus, user)
+        self.cri = rtio.cri.Interface()
+
+        ###
+
+        evento_stb = Signal()
+        evento_latched = Signal()
+        evento_latched_d = Signal()
+        self.specials += MultiReg(evento, evento_latched)
+        self.sync += evento_latched_d.eq(evento_latched)
+        self.comb += self.engine.trigger_stb.eq(self.enable.storage & (evento_latched != evento_latched_d))
+
+        cri = self.cri
+
+        cmd = Signal(8)
+        cmd_write = Signal()
+        cmd_read = Signal()
+        self.comb += [
+            cmd_write.eq(cmd == 0),
+            cmd_read.eq(cmd == 1)
+        ]
+
+        out_len = Signal(8)
+        dout_cases = {}
+        dout_cases[0] = [
+            cmd.eq(self.engine.dout[:8]),
+            out_len.eq(self.engine.dout[8:16]),
+            cri.chan_sel.eq(self.engine.dout[40:]),
+            cri.o_address.eq(self.engine.dout[32:40])
+        ]
+        for i in range(8):
+            target = cri.o_data[i*64:(i+1)*64]
+            dout_cases[0] += [If(i >= self.engine.dout[8:16], target.eq(0))]
+
+        dout_cases[1] = [
+            cri.o_timestamp.eq(self.engine.dout),
+            cri.i_timeout.eq(self.engine.dout)
+        ]
+        for i in range(8):
+            target = cri.o_data[i*64:(i+1)*64]
+            dout_cases[i+2] = [target.eq(self.engine.dout)]
+
+        self.sync += [
+            cri.cmd.eq(rtio.cri.commands["nop"]),
+            If(self.engine.dout_stb,
+                Case(self.engine.dout_index, dout_cases),
+                If(self.engine.dout_index == out_len + 2,
+                    If(cmd_write, cri.cmd.eq(rtio.cri.commands["write"])),
+                    If(cmd_read, cri.cmd.eq(rtio.cri.commands["read"]))
+                )
+            )
+        ]
+
+        # If input event, wait for response before allow input data to be
+        # sampled
+        # TODO: If output, wait for wait flag clear
+        RTIO_I_STATUS_WAIT_STATUS = 4
+        RTIO_O_STATUS_WAIT = 1
+
+        self.submodules.fsm = fsm = FSM(reset_state="IDLE")
+
+        fsm.act("IDLE",
+            If(self.engine.trigger_stb, NextState("WAIT_OUT_CYCLE"))
+        )
+        fsm.act("WAIT_OUT_CYCLE",
+            self.engine.din_ready.eq(0),
+            If(self.engine.dout_stb & cmd_write & (self.engine.dout_index == out_len + 2),
+               NextState("WAIT_READY")
+            ),
+            # for some reason read requires some delay until the next state
+            If(self.engine.dout_stb & cmd_read & (self.engine.dout_index == out_len + 3),
+                NextState("WAIT_READY")
+            )
+        )
+        fsm.act("WAIT_READY",
+            If(cmd_read & (cri.i_status & RTIO_I_STATUS_WAIT_STATUS == 0) \
+                | cmd_write & ~(cri.o_status & RTIO_O_STATUS_WAIT),
+                self.engine.din_ready.eq(1),
+                NextState("IDLE")
+            )
+        )
+
+        din_cases_cmdwrite = {
+            0: [self.engine.din.eq((1<<16) | cri.o_status)],
+            1: [self.engine.din.eq(0)],
+        }
+        din_cases_cmdread = {
+            0: [self.engine.din[:32].eq((1<<16) | cri.i_status), self.engine.din[32:].eq(cri.i_data)],
+            1: [self.engine.din.eq(cri.i_timestamp)]
+        }
+
+        self.comb += [
+            If(cmd_read, Case(self.engine.din_index, din_cases_cmdread)),
+            If(cmd_write, Case(self.engine.din_index, din_cases_cmdwrite)),
+        ]
+
+        # CRI CSRs
+        self.sync += If(self.counter_update.re, self.counter.status.eq(tsc.full_ts_cri))
+        self.comb += [
+            self.o_status.status.eq(self.cri.o_status),
+            self.i_status.status.eq(self.cri.i_status),
+        ]

src/gateware/analyzer.py

@@ -0,0 +1,121 @@
+from migen import *
+
+from misoc.interconnect.csr import *
+from misoc.interconnect import stream
+from migen_axi.interconnect import axi
+
+from artiq.gateware.rtio.analyzer import message_len, MessageEncoder
+
+import endianness
+
+
+class AXIDMAWriter(Module, AutoCSR):
+    def __init__(self, membus, max_outstanding_requests):
+        aw = len(membus.aw.addr)
+        dw = len(membus.w.data)
+        assert message_len % dw == 0
+        burst_length = message_len//dw
+        alignment_bits = log2_int(message_len//8)
+
+        self.reset = CSR()  # only apply when shut down
+        # All numbers in bytes
+        self.base_address = CSRStorage(aw, alignment_bits=alignment_bits)
+        self.last_address = CSRStorage(aw, alignment_bits=alignment_bits)
+        self.byte_count = CSRStatus(32)  # only read when shut down
+        self.bus_error = CSRStatus()
+
+        self.make_request = Signal()
+        self.sink = stream.Endpoint([("data", dw)])
+
+        # # #
+
+        outstanding_requests = Signal(max=max_outstanding_requests+1)
+        current_address = Signal(aw - alignment_bits)
+        self.comb += [
+            membus.aw.addr.eq(Cat(C(0, alignment_bits), current_address)),
+            membus.aw.id.eq(0),                   # Same ID for all transactions to forbid reordering.
+            membus.aw.burst.eq(axi.Burst.incr.value),
+            membus.aw.len.eq(burst_length-1),     # Number of transfers in burst (0->1 transfer, 1->2 transfers...).
+            membus.aw.size.eq(log2_int(dw//8)),   # Width of burst: 3 = 8 bytes = 64 bits.
+            membus.aw.cache.eq(0xf),
+            membus.aw.valid.eq(outstanding_requests != 0),
+        ]
+        self.sync += [
+            outstanding_requests.eq(outstanding_requests + self.make_request - (membus.aw.valid & membus.aw.ready)),
+
+            If(self.reset.re,
+                current_address.eq(self.base_address.storage)),
+            If(membus.aw.valid & membus.aw.ready,
+                If(current_address == self.last_address.storage,
+                    current_address.eq(self.base_address.storage)
+                ).Else(
+                    current_address.eq(current_address + 1)
+                )
+            )
+        ]
+
+        self.comb += [
+            membus.w.id.eq(0),
+            membus.w.valid.eq(self.sink.stb),
+            self.sink.ack.eq(membus.w.ready),
+            membus.w.data.eq(endianness.convert_signal(self.sink.data)),
+            membus.w.strb.eq(2**(dw//8)-1),
+        ]
+        beat_count = Signal(max=burst_length)
+        self.sync += [
+            If(membus.w.valid & membus.w.ready,
+                membus.w.last.eq(0),
+                If(membus.w.last,
+                    beat_count.eq(0)
+                ).Else(
+                    If(beat_count == burst_length-2, membus.w.last.eq(1)),
+                    beat_count.eq(beat_count + 1)
+                )
+            )
+        ]
+
+        message_count = Signal(32 - log2_int(message_len//8))
+        self.comb += self.byte_count.status.eq(
+            message_count << log2_int(message_len//8))
+        self.sync += [
+            If(self.reset.re, message_count.eq(0)),
+            If(membus.w.valid & membus.w.ready & membus.w.last, message_count.eq(message_count + 1))
+        ]
+
+        self.comb += membus.b.ready.eq(1)
+        self.sync += [
+            If(self.reset.re, self.bus_error.status.eq(0)),
+            If(membus.b.valid & membus.b.ready & (membus.b.resp != axi.Response.okay),
+                self.bus_error.status.eq(1))
+        ]
+
+
+class Analyzer(Module, AutoCSR):
+    def __init__(self, tsc, cri, membus, fifo_depth=128):
+        # shutdown procedure: set enable to 0, wait until busy=0
+        self.enable = CSRStorage()
+        self.busy = CSRStatus()
+
+        self.submodules.message_encoder = MessageEncoder(
+            tsc, cri, self.enable.storage)
+        self.submodules.fifo = stream.SyncFIFO(
+            [("data", message_len)], fifo_depth, True)
+        self.submodules.converter = stream.Converter(
+            message_len, len(membus.w.data), reverse=True)
+        self.submodules.dma = AXIDMAWriter(membus, max_outstanding_requests=fifo_depth)
+
+        enable_r = Signal()
+        self.sync += [
+            enable_r.eq(self.enable.storage),
+            If(self.enable.storage & ~enable_r,
+                self.busy.status.eq(1)),
+            If(self.dma.sink.stb & self.dma.sink.ack & self.dma.sink.eop,
+                self.busy.status.eq(0))
+        ]
+
+        self.comb += [
+            self.message_encoder.source.connect(self.fifo.sink),
+            self.fifo.source.connect(self.converter.sink),
+            self.converter.source.connect(self.dma.sink),
+            self.dma.make_request.eq(self.fifo.sink.stb & self.fifo.sink.ack)
+        ]

src/gateware/config.py

@@ -0,0 +1,16 @@
+from misoc.integration import cpu_interface
+
+def write_csr_file(soc, filename):
+    with open(filename, "w") as f:
+        f.write(cpu_interface.get_csr_rust(
+            soc.get_csr_regions(), soc.get_csr_groups(), soc.get_constants()))
+
+def write_mem_file(soc, filename):
+    with open(filename, "w") as f:
+        f.write(cpu_interface.get_mem_rust(
+            soc.get_memory_regions(), soc.get_memory_groups(), None))
+
+def write_rustc_cfg_file(soc, filename):
+    with open(filename, "w") as f:
+        f.write(cpu_interface.get_rust_cfg(
+            soc.get_csr_regions(), soc.get_constants()))

src/gateware/ddmtd.py

@@ -0,0 +1,119 @@
+from migen import *
+from migen.genlib.cdc import PulseSynchronizer, MultiReg
+from misoc.interconnect.csr import *
+
+
+class DDMTDSampler(Module):
+    def __init__(self, cd_ref, main_clk_se):
+        self.ref_beating = Signal()
+        self.main_beating = Signal()
+
+        # # #
+
+        ref_clk = Signal()
+        self.specials +=[
+            # ISERDESE2 can only be driven from fabric via IDELAYE2 (see UG471)
+            Instance("IDELAYE2",
+                    p_DELAY_SRC="DATAIN",
+                    p_HIGH_PERFORMANCE_MODE="TRUE",
+                    p_REFCLK_FREQUENCY=208.3,   # REFCLK frequency from IDELAYCTRL
+                    p_IDELAY_VALUE=0,
+
+                    i_DATAIN=cd_ref.clk,
+
+                    o_DATAOUT=ref_clk
+            ),
+            Instance("ISERDESE2",
+                    p_IOBDELAY="IFD",   # use DDLY as input
+                    p_DATA_RATE="SDR",
+                    p_DATA_WIDTH=2,     # min is 2
+                    p_NUM_CE=1,
+
+                    i_DDLY=ref_clk,
+                    i_CE1=1,
+                    i_CLK=ClockSignal("helper"),
+                    i_CLKDIV=ClockSignal("helper"),
+
+                    o_Q1=self.ref_beating
+            ),
+            Instance("ISERDESE2",
+                    p_DATA_RATE="SDR",
+                    p_DATA_WIDTH=2,     # min is 2
+                    p_NUM_CE=1,
+
+                    i_D=main_clk_se,
+                    i_CE1=1,
+                    i_CLK=ClockSignal("helper"),
+                    i_CLKDIV=ClockSignal("helper"),
+
+                    o_Q1=self.main_beating,
+            ),
+        ]
+
+
+class DDMTDDeglitcherMedianEdge(Module):
+    def __init__(self, counter, input_signal, stable_0_period=100, stable_1_period=100):
+        self.tag = Signal(len(counter))
+        self.detect = Signal()
+
+        stable_0_counter = Signal(reset=stable_0_period - 1, max=stable_0_period)
+        stable_1_counter = Signal(reset=stable_1_period - 1, max=stable_1_period)
+
+        # # #
+
+        # Based on CERN's median edge deglitcher FSM
+        # https://white-rabbit.web.cern.ch/documents/Precise_time_and_frequency_transfer_in_a_White_Rabbit_network.pdf (p.72)
+        fsm = ClockDomainsRenamer("helper")(FSM(reset_state="WAIT_STABLE_0"))
+        self.submodules += fsm
+
+        fsm.act("WAIT_STABLE_0",
+            If(stable_0_counter != 0,
+                NextValue(stable_0_counter, stable_0_counter - 1)
+            ).Else(
+                NextValue(stable_0_counter, stable_0_period - 1),
+                NextState("WAIT_EDGE")
+            ),
+            If(input_signal,
+                NextValue(stable_0_counter, stable_0_period - 1)
+            ),
+        )
+        fsm.act("WAIT_EDGE",
+            If(input_signal,
+                NextValue(self.tag, counter),
+                NextState("GOT_EDGE")
+            )
+        )
+        fsm.act("GOT_EDGE",
+            If(stable_1_counter != 0,
+                NextValue(stable_1_counter, stable_1_counter - 1)
+            ).Else(
+                NextValue(stable_1_counter, stable_1_period - 1),
+                self.detect.eq(1),
+                NextState("WAIT_STABLE_0")
+            ),
+            If(~input_signal,
+                NextValue(self.tag, self.tag + 1),
+                NextValue(stable_1_counter, stable_1_period - 1)
+            ),
+        )
+        
+
+class DDMTD(Module):
+    def __init__(self, counter, input_signal):
+
+        # in helper clock domain
+        self.h_tag = Signal(len(counter))
+        self.h_tag_update = Signal()
+
+        # # #
+
+        deglitcher = DDMTDDeglitcherMedianEdge(counter, input_signal)
+        self.submodules += deglitcher
+
+        self.sync.helper += [
+            self.h_tag_update.eq(0),
+            If(deglitcher.detect,
+                self.h_tag_update.eq(1),
+                self.h_tag.eq(deglitcher.tag)
+               )
+        ]

src/gateware/dma.py

@@ -0,0 +1,157 @@
+from migen import *
+from migen.genlib.fsm import FSM
+from misoc.interconnect.csr import *
+from misoc.interconnect import stream
+from migen_axi.interconnect import axi
+
+from artiq.gateware.rtio.dma import RawSlicer, RecordConverter, RecordSlicer, TimeOffset, CRIMaster
+
+import endianness
+
+
+AXI_BURST_LEN = 16
+
+
+class AXIReader(Module, AutoCSR):
+    def __init__(self, membus):
+        aw = len(membus.ar.addr)
+        dw = len(membus.r.data)
+        alignment_bits = log2_int(AXI_BURST_LEN*dw//8)
+        self.sink = stream.Endpoint([("address", aw - alignment_bits)])
+        self.source = stream.Endpoint([("data", dw)])
+
+        self.bus_error = CSRStatus()
+
+        # # #
+
+        eop_pending = Signal()
+        self.sync += [
+            If(self.sink.stb & self.sink.ack & self.sink.eop, eop_pending.eq(1)),
+            If(self.source.stb & self.source.ack & self.source.eop, eop_pending.eq(0)),
+        ]
+
+        self.comb += [
+            membus.ar.addr.eq(Cat(C(0, alignment_bits), self.sink.address)),
+            membus.ar.id.eq(0),                   # Same ID for all transactions to forbid reordering.
+            membus.ar.burst.eq(axi.Burst.incr.value),
+            membus.ar.len.eq(AXI_BURST_LEN-1),    # Number of transfers in burst (0->1 transfer, 1->2 transfers...).
+            membus.ar.size.eq(log2_int(dw//8)),   # Width of burst: 3 = 8 bytes = 64 bits.
+            membus.ar.cache.eq(0xf),
+            membus.ar.valid.eq(self.sink.stb & ~eop_pending),
+            self.sink.ack.eq(membus.ar.ready & ~eop_pending)
+        ]
+
+        # UG585: "Large slave interface read acceptance capability in the range of 14 to 70 commands"
+        inflight_cnt = Signal(max=128)
+        request_done = Signal()
+        reply_done = Signal()
+        self.comb += [
+            request_done.eq(membus.ar.valid & membus.ar.ready),
+            reply_done.eq(membus.r.valid & membus.r.ready & membus.r.last)
+        ]
+        self.sync += inflight_cnt.eq(inflight_cnt + request_done - reply_done)
+
+        self.comb += [
+            self.source.stb.eq(membus.r.valid),
+            membus.r.ready.eq(self.source.ack),
+            self.source.data.eq(endianness.convert_signal(membus.r.data)),
+            # Note that when eop_pending=1, no new transactions are made and inflight_cnt is no longer incremented
+            self.source.eop.eq(eop_pending & membus.r.last & (inflight_cnt == 1))
+        ]
+
+        stopped = Signal(reset=1)
+        self.sync += [
+            If(self.source.stb & self.source.ack & self.source.eop, stopped.eq(1)),
+            If(self.sink.stb & self.sink.ack, stopped.eq(0)),
+            If(stopped & (self.sink.stb & self.sink.ack),
+                # reset bus error status on new run
+                self.bus_error.status.eq(0)),
+            If(membus.r.valid & membus.r.valid & (membus.r.resp != axi.Response.okay),
+                self.bus_error.status.eq(1))
+        ]
+
+
+class DMAReader(Module, AutoCSR):
+    def __init__(self, membus, enable):
+        aw = len(membus.ar.addr)
+        alignment_bits = log2_int(AXI_BURST_LEN*len(membus.r.data)//8)
+
+        self.submodules.wb_reader = AXIReader(membus)
+        self.source = self.wb_reader.source
+
+        # All numbers in bytes
+        self.base_address = CSRStorage(aw, alignment_bits=alignment_bits)
+
+        # # #
+
+        enable_r = Signal()
+        address = self.wb_reader.sink
+        assert len(address.address) == len(self.base_address.storage)
+        self.sync += [
+            enable_r.eq(enable),
+            If(enable & ~enable_r,
+                address.address.eq(self.base_address.storage),
+                address.eop.eq(0),
+                address.stb.eq(1),
+            ),
+            If(address.stb & address.ack,
+                If(address.eop,
+                    address.stb.eq(0)
+                ).Else(
+                    address.address.eq(address.address + 1),
+                    If(~enable, address.eop.eq(1))
+                )
+            )
+        ]
+
+
+class DMA(Module):
+    def __init__(self, membus):
+        self.enable = CSR()
+
+        flow_enable = Signal()
+        self.submodules.dma = DMAReader(membus, flow_enable)
+        self.submodules.slicer = RecordSlicer(len(membus.r.data))
+        self.submodules.time_offset = TimeOffset()
+        self.submodules.cri_master = CRIMaster()
+        self.cri = self.cri_master.cri
+
+        self.comb += [
+            self.dma.source.connect(self.slicer.sink),
+            self.slicer.source.connect(self.time_offset.sink),
+            self.time_offset.source.connect(self.cri_master.sink)
+        ]
+
+        fsm = FSM(reset_state="IDLE")
+        self.submodules += fsm
+
+        fsm.act("IDLE",
+            If(self.enable.re, NextState("FLOWING"))
+        )
+        fsm.act("FLOWING",
+            self.enable.w.eq(1),
+            flow_enable.eq(1),
+            If(self.slicer.end_marker_found,
+                NextState("FLUSH")
+            )
+        )
+        fsm.act("FLUSH",
+            self.enable.w.eq(1),
+            self.slicer.flush.eq(1),
+            NextState("WAIT_EOP")
+        )
+        fsm.act("WAIT_EOP",
+            self.enable.w.eq(1),
+            If(self.cri_master.sink.stb & self.cri_master.sink.ack & self.cri_master.sink.eop,
+                NextState("WAIT_CRI_MASTER")
+            )
+        )
+        fsm.act("WAIT_CRI_MASTER",
+            self.enable.w.eq(1),
+            If(~self.cri_master.busy, NextState("IDLE"))
+        )
+
+    def get_csrs(self):
+        return ([self.enable] +
+                self.dma.get_csrs() + self.time_offset.get_csrs() +
+                self.cri_master.get_csrs())

src/gateware/drtio_aux_controller.py

@@ -0,0 +1,85 @@
+"""Auxiliary controller, common to satellite and master"""
+
+from artiq.gateware.drtio.aux_controller import (max_packet, aux_buffer_count,
+    Transmitter, Receiver)
+from migen.fhdl.simplify import FullMemoryWE
+from misoc.interconnect.csr import *
+from migen_axi.interconnect.sram import SRAM
+from migen_axi.interconnect import axi
+
+
+class _DRTIOAuxControllerBase(Module):
+    def __init__(self, link_layer):
+        self.bus = axi.Interface()
+        self.submodules.transmitter = Transmitter(link_layer, len(self.bus.w.data))
+        self.submodules.receiver = Receiver(link_layer, len(self.bus.w.data))
+
+    def get_csrs(self):
+        return self.transmitter.get_csrs() + self.receiver.get_csrs()
+
+
+# TODO: FullMemoryWE should be applied by migen.build
+@FullMemoryWE()
+class DRTIOAuxControllerAxi(_DRTIOAuxControllerBase):
+    def __init__(self, link_layer):
+        _DRTIOAuxControllerBase.__init__(self, link_layer)
+
+        tx_sdram_if = SRAM(self.transmitter.mem, read_only=False)
+        rx_sdram_if = SRAM(self.receiver.mem, read_only=True)
+        aw_decoder = axi.AddressDecoder(self.bus.aw,
+            [(lambda a: a[log2_int(max_packet*aux_buffer_count)] == 0, tx_sdram_if.bus.aw),
+             (lambda a: a[log2_int(max_packet*aux_buffer_count)] == 1, rx_sdram_if.bus.aw)],
+            register=True)
+        ar_decoder = axi.AddressDecoder(self.bus.ar,
+            [(lambda a: a[log2_int(max_packet*aux_buffer_count)] == 0, tx_sdram_if.bus.ar),
+             (lambda a: a[log2_int(max_packet*aux_buffer_count)] == 1, rx_sdram_if.bus.ar)],
+            register=True)
+        # unlike wb, axi address decoder only connects ar/aw lanes,
+        # the rest must also be connected!
+        # not quite unlike an address decoder itself.
+
+        # connect bus.b with tx.b
+        self.comb += [tx_sdram_if.bus.b.ready.eq(self.bus.b.ready),
+                      self.bus.b.id.eq(tx_sdram_if.bus.b.id),
+                      self.bus.b.resp.eq(tx_sdram_if.bus.b.resp),
+                      self.bus.b.valid.eq(tx_sdram_if.bus.b.valid)]
+        # connect bus.w with tx.w
+        # no worries about w.valid and slave sel here, only tx will be written to
+        self.comb += [tx_sdram_if.bus.w.id.eq(self.bus.w.id),
+                      tx_sdram_if.bus.w.data.eq(self.bus.w.data),
+                      tx_sdram_if.bus.w.strb.eq(self.bus.w.strb),
+                      tx_sdram_if.bus.w.last.eq(self.bus.w.last),
+                      tx_sdram_if.bus.w.valid.eq(self.bus.w.valid),
+                      self.bus.w.ready.eq(tx_sdram_if.bus.w.ready)]
+        # connect bus.r with rx.r and tx.r w/o data
+        self.comb += [self.bus.r.id.eq(rx_sdram_if.bus.r.id | tx_sdram_if.bus.r.id),
+                #self.bus.r.data.eq(rx_sdram_if.bus.r.data | tx_sdram_if.bus.r.data),
+                self.bus.r.resp.eq(rx_sdram_if.bus.r.resp | tx_sdram_if.bus.r.resp),
+                self.bus.r.last.eq(rx_sdram_if.bus.r.last | tx_sdram_if.bus.r.last),
+                self.bus.r.valid.eq(rx_sdram_if.bus.r.valid | tx_sdram_if.bus.r.valid),
+                rx_sdram_if.bus.r.ready.eq(self.bus.r.ready),
+                tx_sdram_if.bus.r.ready.eq(self.bus.r.ready)]
+        # connect read data after being masked
+        masked = [Replicate(rx_sdram_if.bus.r.valid,
+                            len(self.bus.r.data)
+                            ) & rx_sdram_if.bus.r.data,
+                 Replicate(tx_sdram_if.bus.r.valid,
+                            len(self.bus.r.data)
+                            ) & tx_sdram_if.bus.r.data]
+        self.comb += self.bus.r.data.eq(reduce(or_, masked))
+
+        self.submodules += tx_sdram_if, rx_sdram_if, aw_decoder, ar_decoder
+
+
+@FullMemoryWE()
+class DRTIOAuxControllerBare(_DRTIOAuxControllerBase):
+    # Barebones version of the AuxController. No SRAM, no decoders.
+    # add memories manually from tx and rx in target code.
+    def get_tx_port(self):
+        return self.transmitter.mem.get_port(write_capable=True)
+
+    def get_rx_port(self):
+        return self.receiver.mem.get_port(write_capable=False)
+
+    def get_mem_size(self):
+        return max_packet*aux_buffer_count

src/gateware/ebaz4205.py

@@ -0,0 +1,307 @@
+#!/usr/bin/env python
+
+import argparse
+
+import analyzer
+import dma
+from artiq.gateware import rtio
+from artiq.gateware.rtio.phy import spi2, ttl_simple
+from artiq.gateware.rtio.xilinx_clocking import fix_serdes_timing_path
+from config import write_csr_file, write_mem_file, write_rustc_cfg_file
+from migen import *
+from migen.build.generic_platform import IOStandard, Misc, Pins, Subsignal
+from migen.build.platforms import ebaz4205
+from migen_axi.integration.soc_core import SoCCore
+from misoc.interconnect.csr import *
+
+_ps = [
+    (
+        "ps",
+        0,
+        Subsignal("clk", Pins("E7"), IOStandard("LVCMOS33"), Misc("SLEW=FAST")),
+        Subsignal("por_b", Pins("C7"), IOStandard("LVCMOS33"), Misc("SLEW=FAST")),
+        Subsignal("srst_b", Pins("B10"), IOStandard("LVCMOS18"), Misc("SLEW=FAST")),
+    )
+]
+
+_ddr = [
+    (
+        "ddr",
+        0,
+        Subsignal(
+            "a",
+            Pins("N2 K2 M3 K3 M4 L1 L4 K4 K1 J4 F5 G4 E4 D4 F4"),
+            IOStandard("SSTL15"),
+        ),
+        Subsignal("ba", Pins("L5 R4 J5"), IOStandard("SSTL15")),
+        Subsignal("cas_n", Pins("P5"), IOStandard("SSTL15")),
+        Subsignal("cke", Pins("N3"), IOStandard("SSTL15")),
+        Subsignal("cs_n", Pins("N1"), IOStandard("SSTL15")),
+        Subsignal("ck_n", Pins("M2"), IOStandard("DIFF_SSTL15"), Misc("SLEW=FAST")),
+        Subsignal("ck_p", Pins("L2"), IOStandard("DIFF_SSTL15"), Misc("SLEW=FAST")),
+        # Pins "T1 Y1" not connected
+        Subsignal("dm", Pins("A1 F1"), IOStandard("SSTL15_T_DCI"), Misc("SLEW=FAST")),
+        Subsignal(
+            "dq",
+            Pins("C3 B3 A2 A4 D3 D1 C1 E1 E2 E3 G3 H3 J3 H2 H1 J1"),
+            # Pins "P1 P3 R3 R1 T4 U4 U2 U3 V1 Y3 W1 Y4 Y2 W3 V2 V3" not connected
+            IOStandard("SSTL15_T_DCI"),
+            Misc("SLEW=FAST"),
+        ),
+        Subsignal(
+            "dqs_n",
+            Pins("B2 F2"),  # Pins "T2 W4" not connected
+            IOStandard("DIFF_SSTL15_T_DCI"),
+            Misc("SLEW=FAST"),
+        ),
+        Subsignal(
+            "dqs_p",
+            Pins("C2 G2"),  # Pins "R2 W5" not connected
+            IOStandard("DIFF_SSTL15_T_DCI"),
+            Misc("SLEW=FAST"),
+        ),
+        Subsignal("vrn", Pins("G5"), IOStandard("SSTL15_T_DCI"), Misc("SLEW=FAST")),
+        Subsignal("vrp", Pins("H5"), IOStandard("SSTL15_T_DCI"), Misc("SLEW=FAST")),
+        Subsignal("drst_n", Pins("B4"), IOStandard("SSTL15"), Misc("SLEW=FAST")),
+        Subsignal("odt", Pins("N5"), IOStandard("SSTL15")),
+        Subsignal("ras_n", Pins("P4"), IOStandard("SSTL15")),
+        Subsignal("we_n", Pins("M5"), IOStandard("SSTL15")),
+    )
+]
+
+# Connector J3
+_i2c = [
+    (
+        "i2c",
+        0,
+        Subsignal("scl", Pins("U12"), IOStandard("LVCMOS33")),
+        Subsignal("sda", Pins("V13"), IOStandard("LVCMOS33")),
+    )
+]
+
+_spi = [
+    (
+        "spi",
+        0,
+        Subsignal("clk", Pins("V20")),
+        Subsignal("mosi", Pins("U20")),
+        Subsignal("cs_n", Pins("P19")),
+        IOStandard("LVCMOS33"),
+    )
+]
+
+
+# Connector DATA1
+def _create_ttl():
+    _ttl = []
+
+    for idx, elem in enumerate([x for x in range(5, 21) if x not in (10, 12)]):
+        _ttl.append(
+            ("ttl", idx, Pins("DATA1:DATA1-{}".format(elem)), IOStandard("LVCMOS33")),
+        )
+    return _ttl
+
+
+class EBAZ4205(SoCCore):
+    def __init__(self, rtio_clk=125e6, acpki=False):
+        self.acpki = acpki
+
+        platform = ebaz4205.Platform()
+        platform.toolchain.bitstream_commands.extend(
+            [
+                "set_property BITSTREAM.GENERAL.COMPRESS True [current_design]",
+            ]
+        )
+        platform.add_extension(_ps)
+        platform.add_extension(_ddr)
+        platform.add_extension(_i2c)
+        platform.add_extension(_spi)
+        platform.add_extension(_create_ttl())
+
+        gmii = platform.request("gmii")
+        platform.add_period_constraint(gmii.rx_clk, 10)
+        platform.add_period_constraint(gmii.tx_clk, 10)
+        platform.add_platform_command(
+            "set_property CLOCK_DEDICATED_ROUTE FALSE [get_nets gmii_tx_clk_IBUF]"
+        )
+
+        ident = self.__class__.__name__
+        if self.acpki:
+            ident = "acpki_" + ident
+        SoCCore.__init__(self, platform=platform, csr_data_width=32, ident=ident)
+        fix_serdes_timing_path(platform)
+        self.config["RTIO_FREQUENCY"] = str(rtio_clk / 1e6)
+        platform.add_period_constraint(self.ps7.cd_sys.clk, 10)
+
+        self.comb += [
+            self.ps7.enet0.enet.gmii.tx_clk.eq(gmii.tx_clk),
+            self.ps7.enet0.enet.gmii.rx_clk.eq(gmii.rx_clk),
+        ]
+        self.clock_domains.cd_eth_rx = ClockDomain(reset_less=False)
+        self.clock_domains.cd_eth_tx = ClockDomain(reset_less=False)
+        self.comb += [
+            ClockSignal("eth_rx").eq(gmii.rx_clk),
+            ClockSignal("eth_tx").eq(gmii.tx_clk),
+        ]
+        self.sync.eth_tx += [
+            gmii.txd.eq(self.ps7.enet0.enet.gmii.txd),
+            gmii.tx_en.eq(self.ps7.enet0.enet.gmii.tx_en),
+        ]
+        self.sync.eth_rx += [
+            self.ps7.enet0.enet.gmii.rxd.eq(gmii.rxd),
+            self.ps7.enet0.enet.gmii.rx_dv.eq(gmii.rx_dv),
+        ]
+
+        # MDIO
+        mdio = platform.request("mdio")
+        self.comb += mdio.mdc.eq(self.ps7.enet0.enet.mdio.mdc)
+        self.specials += Instance(
+            "IOBUF",
+            i_I=self.ps7.enet0.enet.mdio.o,
+            io_IO=mdio.mdio,
+            o_O=self.ps7.enet0.enet.mdio.i,
+            i_T=~self.ps7.enet0.enet.mdio.t_n,
+        )
+
+        # I2C
+        i2c = self.platform.request("i2c")
+        self.specials += [
+            # SCL
+            Instance(
+                "IOBUF",
+                i_I=self.ps7.i2c0.scl.o,
+                io_IO=i2c.scl,
+                o_O=self.ps7.i2c0.scl.i,
+                i_T=~self.ps7.i2c0.scl.t_n,
+            ),
+            # SDA
+            Instance(
+                "IOBUF",
+                i_I=self.ps7.i2c0.sda.o,
+                io_IO=i2c.sda,
+                o_O=self.ps7.i2c0.sda.i,
+                i_T=~self.ps7.i2c0.sda.t_n,
+            ),
+        ]
+
+        self.rtio_channels = []
+        for i in (0, 1):
+            print("USER LED at RTIO channel 0x{:06x}".format(len(self.rtio_channels)))
+            user_led = self.platform.request("user_led", i)
+            phy = ttl_simple.Output(user_led)
+            self.submodules += phy
+            self.rtio_channels.append(rtio.Channel.from_phy(phy))
+
+        for i in range(14):
+            print("TTL at RTIO channel 0x{:06x}".format(len(self.rtio_channels)))
+            ttl = self.platform.request("ttl", i)
+            phy = ttl_simple.InOut(ttl)
+            self.submodules += phy
+            self.rtio_channels.append(rtio.Channel.from_phy(phy))
+
+        print("SPI at RTIO channel 0x{:06x}".format(len(self.rtio_channels)))
+        spi_phy = spi2.SPIMaster(platform.request("spi"))
+        self.submodules += spi_phy
+        self.rtio_channels.append(rtio.Channel.from_phy(spi_phy, ififo_depth=4))
+        self.config["RTIO_LOG_CHANNEL"] = len(self.rtio_channels)
+        self.rtio_channels.append(rtio.LogChannel())
+
+        self.submodules.rtio_tsc = rtio.TSC(glbl_fine_ts_width=3)
+        self.submodules.rtio_core = rtio.Core(self.rtio_tsc, self.rtio_channels)
+        self.csr_devices.append("rtio_core")
+        if self.acpki:
+            import acpki
+
+            self.config["KI_IMPL"] = "acp"
+            self.submodules.rtio = acpki.KernelInitiator(
+                self.rtio_tsc,
+                bus=self.ps7.s_axi_acp,
+                user=self.ps7.s_axi_acp_user,
+                evento=self.ps7.event.o,
+            )
+            self.csr_devices.append("rtio")
+        else:
+            self.config["KI_IMPL"] = "csr"
+            self.submodules.rtio = rtio.KernelInitiator(self.rtio_tsc, now64=True)
+            self.csr_devices.append("rtio")
+
+        self.submodules.rtio_dma = dma.DMA(self.ps7.s_axi_hp0)
+        self.csr_devices.append("rtio_dma")
+
+        self.submodules.cri_con = rtio.CRIInterconnectShared(
+            [self.rtio.cri, self.rtio_dma.cri],
+            [self.rtio_core.cri],
+            enable_routing=True,
+        )
+        self.csr_devices.append("cri_con")
+
+        self.submodules.rtio_moninj = rtio.MonInj(self.rtio_channels)
+        self.csr_devices.append("rtio_moninj")
+
+        self.submodules.rtio_analyzer = analyzer.Analyzer(
+            self.rtio_tsc, self.rtio_core.cri, self.ps7.s_axi_hp1
+        )
+        self.csr_devices.append("rtio_analyzer")
+
+
+class BASE(EBAZ4205):
+    def __init__(self, rtio_clk, acpki):
+        EBAZ4205.__init__(self, rtio_clk, acpki)
+
+
+VARIANTS = {cls.__name__.lower(): cls for cls in [BASE]}
+
+
+def main():
+    parser = argparse.ArgumentParser(
+        description="ARTIQ port to the EBAZ4205 control card of Ebit E9+ BTC miner"
+    )
+    parser.add_argument(
+        "-r", default=None, help="build Rust interface into the specified file"
+    )
+    parser.add_argument(
+        "-m", default=None, help="build Rust memory interface into the specified file"
+    )
+    parser.add_argument(
+        "-c",
+        default=None,
+        help="build Rust compiler configuration into the specified file",
+    )
+    parser.add_argument(
+        "-g", default=None, help="build gateware into the specified directory"
+    )
+    parser.add_argument("--rtio-clk", default=125e6, help="RTIO Clock Frequency (Hz)")
+    parser.add_argument(
+        "-V",
+        "--variant",
+        default="base",
+        help="variant: " "[acpki_]base" "(default: %(default)s)",
+    )
+    args = parser.parse_args()
+
+    rtio_clk = int(args.rtio_clk)
+    variant = args.variant.lower()
+    acpki = variant.startswith("acpki_")
+    if acpki:
+        variant = variant[6:]
+
+    try:
+        cls = VARIANTS[variant]
+    except KeyError:
+        raise SystemExit("Invalid variant (-V/--variant)")
+
+    soc = cls(rtio_clk=rtio_clk, acpki=acpki)
+    soc.finalize()
+
+    if args.r is not None:
+        write_csr_file(soc, args.r)
+    if args.m is not None:
+        write_mem_file(soc, args.m)
+    if args.c is not None:
+        write_rustc_cfg_file(soc, args.c)
+    if args.g is not None:
+        soc.build(build_dir=args.g)
+
+
+if __name__ == "__main__":
+    main()

src/gateware/endianness.py

@@ -0,0 +1,21 @@
+from migen import *
+
+
+def convert_signal(signal):
+    assert len(signal) % 8 == 0
+    nbytes = len(signal)//8
+    signal_bytes = []
+    for i in range(nbytes):
+        signal_bytes.append(signal[8*i:8*(i+1)])
+    return Cat(*reversed(signal_bytes))
+
+
+def convert_value(value, size):
+    assert size % 8 == 0
+    nbytes = size//8
+    result = 0
+    for i in range(nbytes):
+        result <<= 8
+        result |= value & 0xff
+        value >>= 8
+    return result

src/gateware/kasli_soc.py

@@ -0,0 +1,672 @@
+#!/usr/bin/env python
+
+import argparse
+from operator import itemgetter
+
+from migen import *
+from migen.build.generic_platform import *
+from migen.genlib.resetsync import AsyncResetSynchronizer
+from migen.genlib.cdc import MultiReg
+from migen_axi.integration.soc_core import SoCCore
+from migen_axi.platforms import kasli_soc
+from misoc.interconnect.csr import *
+from misoc.cores import virtual_leds
+
+from artiq.coredevice import jsondesc
+from artiq.gateware import rtio, eem_7series
+from artiq.gateware.rtio.xilinx_clocking import fix_serdes_timing_path
+from artiq.gateware.rtio.phy import ttl_simple
+from artiq.gateware.drtio.transceiver import gtx_7series, eem_serdes
+from artiq.gateware.drtio.siphaser import SiPhaser7Series
+from artiq.gateware.drtio.rx_synchronizer import XilinxRXSynchronizer
+from artiq.gateware.drtio import *
+from artiq.gateware.wrpll import wrpll
+
+import dma
+import analyzer
+import acpki
+import drtio_aux_controller
+import zynq_clocking
+from config import write_csr_file, write_mem_file, write_rustc_cfg_file
+
+eem_iostandard_dict = {
+     0: "LVDS_25",
+     1: "LVDS_25",
+     2: "LVDS",
+     3: "LVDS",
+     4: "LVDS",
+     5: "LVDS",
+     6: "LVDS",
+     7: "LVDS",
+     8: "LVDS_25",
+     9: "LVDS_25",
+    10: "LVDS",
+    11: "LVDS",
+}
+
+
+def eem_iostandard(eem):
+    return IOStandard(eem_iostandard_dict[eem])
+
+
+class SMAClkinForward(Module):
+    def __init__(self, platform):
+        sma_clkin = platform.request("sma_clkin")
+        sma_clkin_se = Signal()
+        cdr_clk_se = Signal()
+        cdr_clk = platform.request("cdr_clk")
+        self.specials += [
+            Instance("IBUFDS", i_I=sma_clkin.p, i_IB=sma_clkin.n, o_O=sma_clkin_se),
+            Instance("ODDR", i_C=sma_clkin_se, i_CE=1, i_D1=1, i_D2=0, o_Q=cdr_clk_se),
+            Instance("OBUFDS", i_I=cdr_clk_se, o_O=cdr_clk.p, o_OB=cdr_clk.n)
+        ]
+
+
+class GTPBootstrapClock(Module):
+    def __init__(self, platform, freq=125e6):
+        self.clock_domains.cd_bootstrap = ClockDomain(reset_less=True)
+        self.cd_bootstrap.clk.attr.add("keep")
+
+        bootstrap_125 = platform.request("clk125_gtp")
+        bootstrap_se = Signal()
+        clk_out = Signal()
+        platform.add_period_constraint(bootstrap_125.p, 8.0)
+        self.specials += [
+            Instance("IBUFDS_GTE2",
+                i_CEB=0,
+                i_I=bootstrap_125.p, i_IB=bootstrap_125.n, 
+                o_O=bootstrap_se,
+                p_CLKCM_CFG="TRUE",
+                p_CLKRCV_TRST="TRUE",
+                p_CLKSWING_CFG=3),
+            Instance("BUFG", i_I=bootstrap_se, o_O=clk_out)
+        ]
+        if freq == 125e6:
+            self.comb += self.cd_bootstrap.clk.eq(clk_out)
+        elif freq == 100e6:
+            pll_fb = Signal()
+            pll_out = Signal()
+            self.specials += [
+                Instance("PLLE2_BASE",
+                    p_CLKIN1_PERIOD=8.0,
+                    i_CLKIN1=clk_out,
+                    i_CLKFBIN=pll_fb,
+                    o_CLKFBOUT=pll_fb,
+
+                    # VCO @ 1GHz
+                    p_CLKFBOUT_MULT=8, p_DIVCLK_DIVIDE=1,
+
+                    # 100MHz for bootstrap
+                    p_CLKOUT1_DIVIDE=10, p_CLKOUT1_PHASE=0.0, o_CLKOUT1=pll_out,
+                ),
+                Instance("BUFG", i_I=pll_out, o_O=self.cd_bootstrap.clk)
+            ]
+        else:
+            raise ValueError("Bootstrap frequency must be 100 or 125MHz")
+
+
+class GenericStandalone(SoCCore):
+    def __init__(self, description, acpki=False):
+        self.acpki = acpki
+        clk_freq = description["rtio_frequency"]
+        with_wrpll = description["enable_wrpll"]
+
+        platform = kasli_soc.Platform()
+        platform.toolchain.bitstream_commands.extend([
+            "set_property BITSTREAM.GENERAL.COMPRESS True [current_design]",
+        ])
+        ident = description["variant"]
+        if self.acpki:
+            ident = "acpki_" + ident
+        SoCCore.__init__(self, platform=platform, csr_data_width=32, ident=ident, ps_cd_sys=False)
+
+        self.config["HW_REV"] = description["hw_rev"]
+        clk_synth = platform.request("cdr_clk_clean_fabric")
+        clk_synth_se = Signal()
+        clk_synth_se_buf = Signal()
+        platform.add_period_constraint(clk_synth.p, 8.0)
+
+        self.specials += [ 
+            Instance("IBUFGDS",
+                p_DIFF_TERM="TRUE", p_IBUF_LOW_PWR="FALSE",
+                i_I=clk_synth.p, i_IB=clk_synth.n, o_O=clk_synth_se
+            ),
+            Instance("BUFG", i_I=clk_synth_se, o_O=clk_synth_se_buf),
+        ]
+        fix_serdes_timing_path(platform)
+        self.submodules.bootstrap = GTPBootstrapClock(self.platform, clk_freq)
+        self.config["RTIO_FREQUENCY"] = str(clk_freq/1e6)
+        self.config["CLOCK_FREQUENCY"] = int(clk_freq)
+
+        self.submodules.sys_crg = zynq_clocking.SYSCRG(self.platform, self.ps7, clk_synth_se_buf)
+        platform.add_false_path_constraints(
+            self.bootstrap.cd_bootstrap.clk, self.sys_crg.cd_sys.clk)
+        self.csr_devices.append("sys_crg")
+        self.crg = self.ps7 # HACK for eem_7series to find the clock
+        self.crg.cd_sys = self.sys_crg.cd_sys
+
+        if with_wrpll:
+            self.submodules.wrpll_refclk = wrpll.FrequencyMultiplier(platform.request("sma_clkin"))
+            self.submodules.wrpll = wrpll.WRPLL(
+                platform=self.platform,
+                cd_ref=self.wrpll_refclk.cd_ref,
+                main_clk_se=clk_synth_se)
+            self.csr_devices.append("wrpll_refclk")
+            self.csr_devices.append("wrpll")
+            self.comb += self.ps7.core.core0.nfiq.eq(self.wrpll.ev.irq)
+            self.config["HAS_SI549"] = None
+            self.config["WRPLL_REF_CLK"] = "SMA_CLKIN"
+        else:
+            self.submodules += SMAClkinForward(self.platform)
+            self.config["HAS_SI5324"] = None
+            self.config["SI5324_SOFT_RESET"] = None
+
+
+        self.rtio_channels = []
+        has_grabber = any(peripheral["type"] == "grabber" for peripheral in description["peripherals"])
+        if has_grabber:
+            self.grabber_csr_group = []
+        eem_7series.add_peripherals(self, description["peripherals"], iostandard=eem_iostandard)
+        for i in (0, 1):
+            print("USER LED at RTIO channel 0x{:06x}".format(len(self.rtio_channels)))
+            user_led = self.platform.request("user_led", i)
+            phy = ttl_simple.Output(user_led)
+            self.submodules += phy
+            self.rtio_channels.append(rtio.Channel.from_phy(phy))
+        self.config["RTIO_LOG_CHANNEL"] = len(self.rtio_channels)
+        self.rtio_channels.append(rtio.LogChannel())
+
+        self.submodules.rtio_tsc = rtio.TSC(glbl_fine_ts_width=3)
+        self.submodules.rtio_core = rtio.Core(
+            self.rtio_tsc, self.rtio_channels, lane_count=description["sed_lanes"]
+        )
+        self.csr_devices.append("rtio_core")
+
+        if self.acpki:
+            self.config["KI_IMPL"] = "acp"
+            self.submodules.rtio = acpki.KernelInitiator(self.rtio_tsc,
+                                                         bus=self.ps7.s_axi_acp,
+                                                         user=self.ps7.s_axi_acp_user,
+                                                         evento=self.ps7.event.o)
+            self.csr_devices.append("rtio")
+        else:
+            self.config["KI_IMPL"] = "csr"
+            self.submodules.rtio = rtio.KernelInitiator(self.rtio_tsc, now64=True)
+            self.csr_devices.append("rtio")
+
+        self.submodules.rtio_dma = dma.DMA(self.ps7.s_axi_hp0)
+        self.csr_devices.append("rtio_dma")
+
+        self.submodules.cri_con = rtio.CRIInterconnectShared(
+            [self.rtio.cri, self.rtio_dma.cri],
+            [self.rtio_core.cri])
+        self.csr_devices.append("cri_con")
+
+        self.submodules.rtio_moninj = rtio.MonInj(self.rtio_channels)
+        self.csr_devices.append("rtio_moninj")
+
+        self.submodules.rtio_analyzer = analyzer.Analyzer(self.rtio_tsc, self.rtio_core.cri,
+                                                          self.ps7.s_axi_hp1)
+        self.csr_devices.append("rtio_analyzer")
+
+        if has_grabber:
+            self.config["HAS_GRABBER"] = None
+            self.add_csr_group("grabber", self.grabber_csr_group)
+            for grabber in self.grabber_csr_group:
+                self.platform.add_false_path_constraints(
+                    self.sys_crg.cd_sys.clk, getattr(self, grabber).deserializer.cd_cl.clk)
+
+
+class GenericMaster(SoCCore):
+    def __init__(self, description, acpki=False):
+        clk_freq = description["rtio_frequency"]
+        with_wrpll = description["enable_wrpll"]
+
+        has_drtio_over_eem = any(peripheral["type"] == "shuttler" for peripheral in description["peripherals"])
+        self.acpki = acpki
+
+        platform = kasli_soc.Platform()
+        platform.toolchain.bitstream_commands.extend([
+            "set_property BITSTREAM.GENERAL.COMPRESS True [current_design]",
+        ])
+        ident = description["variant"]
+        if self.acpki:
+            ident = "acpki_" + ident
+        SoCCore.__init__(self, platform=platform, csr_data_width=32, ident=ident, ps_cd_sys=False)
+
+        self.config["HW_REV"] = description["hw_rev"]
+
+        data_pads = [platform.request("sfp", i) for i in range(4)]
+
+        self.submodules.gt_drtio = gtx_7series.GTX(
+            clock_pads=platform.request("clk_gtp"),
+            pads=data_pads,
+            clk_freq=clk_freq)
+        self.csr_devices.append("gt_drtio")
+        self.config["RTIO_FREQUENCY"] = str(clk_freq/1e6)
+        self.config["CLOCK_FREQUENCY"] = int(clk_freq)
+
+        txout_buf = Signal()
+        gtx0 = self.gt_drtio.gtxs[0]
+        self.specials += Instance("BUFG", i_I=gtx0.txoutclk, o_O=txout_buf)
+
+        ext_async_rst = Signal()
+
+        self.submodules.bootstrap = GTPBootstrapClock(self.platform, clk_freq)
+        self.submodules.sys_crg = zynq_clocking.SYSCRG(
+            self.platform,
+            self.ps7,
+            txout_buf,
+            clk_sw=self.gt_drtio.stable_clkin.storage,
+            clk_sw_status=gtx0.tx_init.done,
+            ext_async_rst=ext_async_rst)
+        self.csr_devices.append("sys_crg")
+        self.crg = self.ps7 # HACK for eem_7series to find the clock
+        self.crg.cd_sys = self.sys_crg.cd_sys
+        platform.add_false_path_constraints(
+            self.bootstrap.cd_bootstrap.clk, self.sys_crg.cd_sys.clk)
+        fix_serdes_timing_path(platform)
+
+        self.comb += ext_async_rst.eq(self.sys_crg.clk_sw_fsm.o_clk_sw & ~gtx0.tx_init.done)
+        self.specials += MultiReg(self.sys_crg.clk_sw_fsm.o_clk_sw & self.sys_crg.mmcm_locked, self.gt_drtio.clk_path_ready, odomain="bootstrap")
+
+        if with_wrpll:
+            clk_synth = platform.request("cdr_clk_clean_fabric")
+            clk_synth_se = Signal()
+            platform.add_period_constraint(clk_synth.p, 8.0)
+            self.specials += Instance("IBUFGDS", p_DIFF_TERM="TRUE", p_IBUF_LOW_PWR="FALSE", i_I=clk_synth.p, i_IB=clk_synth.n, o_O=clk_synth_se)
+            self.submodules.wrpll_refclk = wrpll.FrequencyMultiplier(platform.request("sma_clkin"))
+            self.submodules.wrpll = wrpll.WRPLL(
+                platform=self.platform,
+                cd_ref=self.wrpll_refclk.cd_ref,
+                main_clk_se=clk_synth_se)
+            self.csr_devices.append("wrpll_refclk")
+            self.csr_devices.append("wrpll")
+            self.comb += self.ps7.core.core0.nfiq.eq(self.wrpll.ev.irq)
+            self.config["HAS_SI549"] = None
+            self.config["WRPLL_REF_CLK"] = "SMA_CLKIN"
+        else:
+            self.submodules += SMAClkinForward(self.platform)
+            self.config["HAS_SI5324"] = None
+            self.config["SI5324_SOFT_RESET"] = None
+
+        self.rtio_channels = []
+        has_grabber = any(peripheral["type"] == "grabber" for peripheral in description["peripherals"])
+        if has_drtio_over_eem:
+            self.eem_drtio_channels = []
+        if has_grabber:
+            self.grabber_csr_group = []
+        eem_7series.add_peripherals(self, description["peripherals"], iostandard=eem_iostandard)
+        for i in (0, 1):
+            print("USER LED at RTIO channel 0x{:06x}".format(len(self.rtio_channels)))
+            user_led = self.platform.request("user_led", i)
+            phy = ttl_simple.Output(user_led)
+            self.submodules += phy
+            self.rtio_channels.append(rtio.Channel.from_phy(phy))
+        self.config["RTIO_LOG_CHANNEL"] = len(self.rtio_channels)
+        self.rtio_channels.append(rtio.LogChannel())
+
+        self.submodules.rtio_tsc = rtio.TSC(glbl_fine_ts_width=3)
+
+        self.drtio_csr_group = []
+        self.drtioaux_csr_group = []
+        self.drtioaux_memory_group = []
+        self.drtio_cri = []
+        for i in range(len(self.gt_drtio.channels)):
+            core_name = "drtio" + str(i)
+            coreaux_name = "drtioaux" + str(i)
+            memory_name = "drtioaux" + str(i) + "_mem"
+            self.drtio_csr_group.append(core_name)
+            self.drtioaux_csr_group.append(coreaux_name)
+            self.drtioaux_memory_group.append(memory_name)
+
+            cdr = ClockDomainsRenamer({"rtio_rx": "rtio_rx" + str(i)})
+
+            core = cdr(DRTIOMaster(self.rtio_tsc, self.gt_drtio.channels[i]))
+            setattr(self.submodules, core_name, core)
+            self.drtio_cri.append(core.cri)
+            self.csr_devices.append(core_name)
+
+            coreaux = cdr(drtio_aux_controller.DRTIOAuxControllerBare(core.link_layer))
+            setattr(self.submodules, coreaux_name, coreaux)
+            self.csr_devices.append(coreaux_name)
+
+            size = coreaux.get_mem_size()
+            memory_address = self.axi2csr.register_port(coreaux.get_tx_port(), size)
+            self.axi2csr.register_port(coreaux.get_rx_port(), size)
+            self.add_memory_region(memory_name, self.mem_map["csr"] + memory_address, size * 2)
+        self.config["HAS_DRTIO"] = None
+        self.config["HAS_DRTIO_ROUTING"] = None
+
+        if has_drtio_over_eem:
+            self.add_eem_drtio(self.eem_drtio_channels)
+        self.add_drtio_cpuif_groups()
+
+        self.submodules.rtio_core = rtio.Core(
+            self.rtio_tsc, self.rtio_channels, lane_count=description["sed_lanes"]
+        )
+        self.csr_devices.append("rtio_core")
+
+        if self.acpki:
+            self.config["KI_IMPL"] = "acp"
+            self.submodules.rtio = acpki.KernelInitiator(self.rtio_tsc,
+                                                         bus=self.ps7.s_axi_acp,
+                                                         user=self.ps7.s_axi_acp_user,
+                                                         evento=self.ps7.event.o)
+            self.csr_devices.append("rtio")
+        else:
+            self.config["KI_IMPL"] = "csr"
+            self.submodules.rtio = rtio.KernelInitiator(self.rtio_tsc, now64=True)
+            self.csr_devices.append("rtio")
+
+        self.submodules.rtio_dma = dma.DMA(self.ps7.s_axi_hp0)
+        self.csr_devices.append("rtio_dma")
+
+        self.submodules.cri_con = rtio.CRIInterconnectShared(
+            [self.rtio.cri, self.rtio_dma.cri],
+            [self.rtio_core.cri] + self.drtio_cri,
+            enable_routing=True)
+        self.csr_devices.append("cri_con")
+
+        self.submodules.rtio_moninj = rtio.MonInj(self.rtio_channels)
+        self.csr_devices.append("rtio_moninj")
+
+        self.submodules.routing_table = rtio.RoutingTableAccess(self.cri_con)
+        self.csr_devices.append("routing_table")
+
+        self.submodules.rtio_analyzer = analyzer.Analyzer(self.rtio_tsc, self.rtio_core.cri,
+                                                          self.ps7.s_axi_hp1)
+        self.csr_devices.append("rtio_analyzer")
+
+        if has_grabber:
+            self.config["HAS_GRABBER"] = None
+            self.add_csr_group("grabber", self.grabber_csr_group)
+        
+
+        self.submodules.virtual_leds = virtual_leds.VirtualLeds()
+        self.csr_devices.append("virtual_leds")
+
+        self.comb += [self.virtual_leds.get(i).eq(channel.rx_ready)
+                for i, channel in enumerate(self.gt_drtio.channels)]
+
+    def add_eem_drtio(self, eem_drtio_channels):
+        # Must be called before invoking add_rtio() to construct the CRI
+        # interconnect properly
+        self.submodules.eem_transceiver = eem_serdes.EEMSerdes(self.platform, eem_drtio_channels)
+        self.csr_devices.append("eem_transceiver")
+        self.config["HAS_DRTIO_EEM"] = None
+        self.config["EEM_DRTIO_COUNT"] = len(eem_drtio_channels)
+
+        cdr = ClockDomainsRenamer({"rtio_rx": "sys"})
+        for i in range(len(self.eem_transceiver.channels)):
+            channel = i + len(self.gt_drtio.channels)
+            core_name = "drtio" + str(channel)
+            coreaux_name = "drtioaux" + str(channel)
+            memory_name = "drtioaux" + str(channel) + "_mem"
+            self.drtio_csr_group.append(core_name)
+            self.drtioaux_csr_group.append(coreaux_name)
+            self.drtioaux_memory_group.append(memory_name)
+
+            core = cdr(DRTIOMaster(self.rtio_tsc, self.eem_transceiver.channels[i]))
+            setattr(self.submodules, core_name, core)
+            self.drtio_cri.append(core.cri)
+            self.csr_devices.append(core_name)
+
+            coreaux = cdr(drtio_aux_controller.DRTIOAuxControllerBare(core.link_layer))
+            setattr(self.submodules, coreaux_name, coreaux)
+            self.csr_devices.append(coreaux_name)
+
+            size = coreaux.get_mem_size()
+            memory_address = self.axi2csr.register_port(coreaux.get_tx_port(), size)
+            self.axi2csr.register_port(coreaux.get_rx_port(), size)
+            self.add_memory_region(memory_name, self.mem_map["csr"] + memory_address, size * 2)
+
+    def add_drtio_cpuif_groups(self):
+        self.add_csr_group("drtio", self.drtio_csr_group)
+        self.add_csr_group("drtioaux", self.drtioaux_csr_group)
+        self.add_memory_group("drtioaux_mem", self.drtioaux_memory_group)
+
+
+class GenericSatellite(SoCCore):
+    def __init__(self, description, acpki=False):
+        clk_freq = description["rtio_frequency"]
+        with_wrpll = description["enable_wrpll"]
+
+        self.acpki = acpki
+
+        platform = kasli_soc.Platform()
+        platform.toolchain.bitstream_commands.extend([
+            "set_property BITSTREAM.GENERAL.COMPRESS True [current_design]",
+        ])
+        ident = description["variant"]
+        if self.acpki:
+            ident = "acpki_" + ident
+        SoCCore.__init__(self, platform=platform, csr_data_width=32, ident=ident, ps_cd_sys=False)
+
+        self.config["HW_REV"] = description["hw_rev"]
+
+        data_pads = [platform.request("sfp", i) for i in range(4)]
+        
+        self.submodules.gt_drtio = gtx_7series.GTX(
+            clock_pads=platform.request("clk_gtp"),  
+            pads=data_pads,
+            clk_freq=clk_freq)
+        self.csr_devices.append("gt_drtio")
+
+        txout_buf = Signal()
+        gtx0 = self.gt_drtio.gtxs[0]
+        self.specials += Instance("BUFG", i_I=gtx0.txoutclk, o_O=txout_buf)
+
+        ext_async_rst = Signal()
+
+        self.submodules.bootstrap = GTPBootstrapClock(self.platform, clk_freq)
+        self.submodules.sys_crg = zynq_clocking.SYSCRG(
+            self.platform, 
+            self.ps7,
+            txout_buf,
+            clk_sw=self.gt_drtio.stable_clkin.storage,
+            clk_sw_status=gtx0.tx_init.done,
+            ext_async_rst=ext_async_rst)
+        platform.add_false_path_constraints(
+            self.bootstrap.cd_bootstrap.clk, self.sys_crg.cd_sys.clk)
+        self.csr_devices.append("sys_crg")
+        self.crg = self.ps7 # HACK for eem_7series to find the clock
+        self.crg.cd_sys = self.sys_crg.cd_sys
+
+        fix_serdes_timing_path(platform)
+
+        self.comb += ext_async_rst.eq(self.sys_crg.clk_sw_fsm.o_clk_sw & ~gtx0.tx_init.done)
+        self.specials += MultiReg(self.sys_crg.clk_sw_fsm.o_clk_sw & self.sys_crg.mmcm_locked, self.gt_drtio.clk_path_ready, odomain="bootstrap")
+
+        self.rtio_channels = []
+        has_grabber = any(peripheral["type"] == "grabber" for peripheral in description["peripherals"])
+        if has_grabber:
+            self.grabber_csr_group = []
+        eem_7series.add_peripherals(self, description["peripherals"], iostandard=eem_iostandard)
+        for i in (0, 1):
+            print("USER LED at RTIO channel 0x{:06x}".format(len(self.rtio_channels)))
+            user_led = self.platform.request("user_led", i)
+            phy = ttl_simple.Output(user_led)
+            self.submodules += phy
+            self.rtio_channels.append(rtio.Channel.from_phy(phy))
+        self.config["RTIO_LOG_CHANNEL"] = len(self.rtio_channels)
+        self.rtio_channels.append(rtio.LogChannel())
+
+        self.submodules.rtio_tsc = rtio.TSC(glbl_fine_ts_width=3)
+
+        drtioaux_csr_group = []
+        drtioaux_memory_group = []
+        drtiorep_csr_group = []
+        self.drtio_cri = []
+        for i in range(len(self.gt_drtio.channels)):
+            coreaux_name = "drtioaux" + str(i)
+            memory_name = "drtioaux" + str(i) + "_mem"
+            drtioaux_csr_group.append(coreaux_name)
+            drtioaux_memory_group.append(memory_name)
+
+            cdr = ClockDomainsRenamer({"rtio_rx": "rtio_rx" + str(i)})
+
+            if i == 0:
+                self.submodules.rx_synchronizer = cdr(XilinxRXSynchronizer())
+                core = cdr(DRTIOSatellite(
+                    self.rtio_tsc, self.gt_drtio.channels[i],
+                    self.rx_synchronizer))
+                self.submodules.drtiosat = core
+                self.csr_devices.append("drtiosat")
+            else:
+                corerep_name = "drtiorep" + str(i-1)
+                drtiorep_csr_group.append(corerep_name)
+
+                core = cdr(DRTIORepeater(
+                    self.rtio_tsc, self.gt_drtio.channels[i]))
+                setattr(self.submodules, corerep_name, core)
+                self.drtio_cri.append(core.cri)
+                self.csr_devices.append(corerep_name)
+
+            coreaux = cdr(drtio_aux_controller.DRTIOAuxControllerBare(core.link_layer))
+            setattr(self.submodules, coreaux_name, coreaux)
+            self.csr_devices.append(coreaux_name)
+
+            mem_size = coreaux.get_mem_size()
+            tx_port = coreaux.get_tx_port()
+            rx_port = coreaux.get_rx_port()
+            memory_address = self.axi2csr.register_port(tx_port, mem_size)
+            # rcv in upper half of the memory, thus added second
+            self.axi2csr.register_port(rx_port, mem_size)
+            # and registered in PS interface
+            # manually, because software refers to rx/tx by halves of entire memory block, not names
+            self.add_memory_region(memory_name, self.mem_map["csr"] + memory_address, mem_size * 2)
+        self.config["HAS_DRTIO"] = None
+        self.config["HAS_DRTIO_ROUTING"] = None
+        self.add_csr_group("drtioaux", drtioaux_csr_group)
+        self.add_memory_group("drtioaux_mem", drtioaux_memory_group)
+        self.add_csr_group("drtiorep", drtiorep_csr_group)
+
+        if self.acpki:
+            self.config["KI_IMPL"] = "acp"
+            self.submodules.rtio = acpki.KernelInitiator(self.rtio_tsc,
+                                                         bus=self.ps7.s_axi_acp,
+                                                         user=self.ps7.s_axi_acp_user,
+                                                         evento=self.ps7.event.o)
+            self.csr_devices.append("rtio")
+        else:
+            self.config["KI_IMPL"] = "csr"
+            self.submodules.rtio = rtio.KernelInitiator(self.rtio_tsc, now64=True)
+            self.csr_devices.append("rtio")
+
+        self.submodules.rtio_dma = dma.DMA(self.ps7.s_axi_hp0)
+        self.csr_devices.append("rtio_dma")
+
+        self.submodules.local_io = SyncRTIO(
+            self.rtio_tsc, self.rtio_channels, lane_count=description["sed_lanes"]
+        )
+        self.comb += [
+            self.drtiosat.async_errors.eq(self.local_io.async_errors),
+            self.local_io.sed_spread_enable.eq(self.drtiosat.sed_spread_enable.storage)
+        ]
+
+        self.submodules.cri_con = rtio.CRIInterconnectShared(
+            [self.drtiosat.cri, self.rtio_dma.cri, self.rtio.cri],
+            [self.local_io.cri] + self.drtio_cri,
+            enable_routing=True)
+        self.csr_devices.append("cri_con")
+
+        self.submodules.routing_table = rtio.RoutingTableAccess(self.cri_con)
+        self.csr_devices.append("routing_table")     
+
+        self.submodules.rtio_moninj = rtio.MonInj(self.rtio_channels)
+        self.csr_devices.append("rtio_moninj")
+
+        self.submodules.rtio_analyzer = analyzer.Analyzer(self.rtio_tsc, self.local_io.cri,
+                                                          self.ps7.s_axi_hp1)
+        self.csr_devices.append("rtio_analyzer")
+
+        rtio_clk_period = 1e9/clk_freq
+        self.config["RTIO_FREQUENCY"] = str(clk_freq/1e6)
+        self.config["CLOCK_FREQUENCY"] = int(clk_freq)
+
+        if with_wrpll:
+            clk_synth = platform.request("cdr_clk_clean_fabric")
+            clk_synth_se = Signal()
+            platform.add_period_constraint(clk_synth.p, 8.0)
+            self.specials += Instance("IBUFGDS", p_DIFF_TERM="TRUE", p_IBUF_LOW_PWR="FALSE", i_I=clk_synth.p, i_IB=clk_synth.n, o_O=clk_synth_se)
+            self.submodules.wrpll = wrpll.WRPLL(
+                platform=self.platform,
+                cd_ref=self.gt_drtio.cd_rtio_rx0,
+                main_clk_se=clk_synth_se)
+            self.submodules.wrpll_skewtester = wrpll.SkewTester(self.rx_synchronizer)
+            self.csr_devices.append("wrpll_skewtester")
+            self.csr_devices.append("wrpll")
+            self.comb += self.ps7.core.core0.nfiq.eq(self.wrpll.ev.irq)
+            self.config["HAS_SI549"] = None
+            self.config["WRPLL_REF_CLK"] = "GT_CDR"
+        else:
+            self.submodules.siphaser = SiPhaser7Series(
+                si5324_clkin=platform.request("cdr_clk"),
+                rx_synchronizer=self.rx_synchronizer,
+                ultrascale=False,
+                rtio_clk_freq=self.gt_drtio.rtio_clk_freq)
+            self.csr_devices.append("siphaser")
+            self.config["HAS_SI5324"] = None
+            self.config["SI5324_SOFT_RESET"] = None
+
+        gtx0 = self.gt_drtio.gtxs[0]
+        platform.add_false_path_constraints(
+            gtx0.txoutclk, gtx0.rxoutclk)
+
+        if has_grabber:
+            self.config["HAS_GRABBER"] = None
+            self.add_csr_group("grabber", self.grabber_csr_group)
+            # no RTIO CRG here
+        
+        self.submodules.virtual_leds = virtual_leds.VirtualLeds()
+        self.csr_devices.append("virtual_leds")
+
+        self.comb += [self.virtual_leds.get(i).eq(channel.rx_ready)
+                for i, channel in enumerate(self.gt_drtio.channels)]
+
+def main():
+    parser = argparse.ArgumentParser(
+        description="ARTIQ device binary builder for generic Kasli-SoC systems")
+    parser.add_argument("-r", default=None,
+        help="build Rust interface into the specified file")
+    parser.add_argument("-c", default=None,
+        help="build Rust compiler configuration into the specified file")
+    parser.add_argument("-m", default=None,
+        help="build Rust memory interface into the specified file")
+    parser.add_argument("-g", default=None,
+        help="build gateware into the specified directory")
+    parser.add_argument("--acpki", default=False, action="store_true",
+        help="enable ACPKI")
+    parser.add_argument("description", metavar="DESCRIPTION",
+                        help="JSON system description file")
+    args = parser.parse_args()
+    description = jsondesc.load(args.description)
+
+    if description["target"] != "kasli_soc":
+        raise ValueError("Description is for a different target")
+
+    if description["drtio_role"] == "standalone":
+        cls = GenericStandalone
+    elif description["drtio_role"] == "master":
+        cls = GenericMaster
+    elif description["drtio_role"] == "satellite":
+        cls = GenericSatellite
+    else:
+        raise ValueError("Invalid DRTIO role")
+
+    soc = cls(description, acpki=args.acpki)
+    soc.finalize()
+
+    if args.r is not None:
+        write_csr_file(soc, args.r)
+    if args.m is not None:
+        write_mem_file(soc, args.m)
+    if args.c is not None:
+        write_rustc_cfg_file(soc, args.c)
+    if args.g is not None:
+        soc.build(build_dir=args.g)
+
+
+if __name__ == "__main__":
+    main()

src/gateware/test_dma.py

@@ -0,0 +1,236 @@
+import unittest
+import random
+import itertools
+
+from migen import *
+from migen_axi.interconnect import axi
+
+from artiq.coredevice.exceptions import RTIOUnderflow, RTIODestinationUnreachable
+from artiq.gateware import rtio
+from artiq.gateware.rtio import cri
+from artiq.gateware.rtio.phy import ttl_simple
+
+import endianness
+import dma
+
+
+class AXIMemorySim:
+    def __init__(self, bus, data, max_queue=12):
+        self.bus = bus
+        self.data = data
+        self.max_queue = max_queue
+        self.align = len(bus.r.data)//8
+        self.queue = []
+
+    @passive
+    def ar(self):
+        while True:
+            if len(self.queue) < self.max_queue:
+                request = yield from self.bus.read_ar()
+                self.queue.append(request)
+            else:
+                yield
+
+    @passive
+    def r(self):
+        while True:
+            if self.queue:
+                request = self.queue.pop()
+                if request.burst:
+                    request_len = request.len + 1
+                else:
+                    request_len = 1
+                for i in range(request_len):
+                    if request.addr % self.align:
+                        raise ValueError
+                    addr = request.addr//self.align + i
+                    if addr < len(self.data):
+                        data = self.data[addr]
+                    else:
+                        data = 0
+                    data = endianness.convert_value(data, len(self.bus.r.data))
+                    yield from self.bus.write_r(request.id, data, last=i == request_len-1)
+            else:
+                yield
+
+
+def encode_n(n, min_length, max_length):
+    r = []
+    while n:
+        r.append(n & 0xff)
+        n >>= 8
+    r += [0]*(min_length - len(r))
+    if len(r) > max_length:
+        raise ValueError
+    return r
+
+
+def encode_record(channel, timestamp, address, data):
+    r = []
+    r += encode_n(channel, 3, 3)
+    r += encode_n(timestamp, 8, 8)
+    r += encode_n(address, 1, 1)
+    r += encode_n(data, 1, 64)
+    return encode_n(len(r)+1, 1, 1) + r
+
+
+def pack(x, size):
+    r = []
+    for i in range((len(x)+size-1)//size):
+        n = 0
+        for j in range(i*size, (i+1)*size):
+            n <<= 8
+            try:
+                n |= x[j]
+            except IndexError:
+                pass
+        r.append(n)
+    return r
+
+
+def encode_sequence(writes, ws):
+    sequence = [b for write in writes for b in encode_record(*write)]
+    sequence.append(0)
+    return pack(sequence, ws)
+
+
+def do_dma(dut, address):
+    yield from dut.dma.base_address.write(address)
+    yield from dut.enable.write(1)
+    yield
+    while ((yield from dut.enable.read())):
+        yield
+    error = yield from dut.cri_master.error.read()
+    if error & 1:
+        raise RTIOUnderflow
+    if error & 2:
+        raise RTIODestinationUnreachable
+
+
+test_writes1 = [
+    (0x01, 0x23, 0x12, 0x33),
+    (0x901, 0x902, 0x11, 0xeeeeeeeeeeeeeefffffffffffffffffffffffffffffff28888177772736646717738388488),
+    (0x81, 0x288, 0x88, 0x8888)
+]
+
+
+test_writes2 = [
+    (0x10, 0x10000, 0x20, 0x77),
+    (0x11, 0x10001, 0x22, 0x7777),
+    (0x12, 0x10002, 0x30, 0x777777),
+    (0x13, 0x10003, 0x40, 0x77777788),
+    (0x14, 0x10004, 0x50, 0x7777778899),
+]
+
+
+prng = random.Random(0)
+
+
+class TB(Module):
+    def __init__(self, ws):
+        sequence1 = encode_sequence(test_writes1, ws)
+        sequence2 = encode_sequence(test_writes2, ws)
+        offset = 512//ws
+        assert len(sequence1) < offset
+        sequence = (
+            sequence1 +
+            [prng.randrange(2**(ws*8)) for _ in range(offset-len(sequence1))] +
+            sequence2)
+
+        bus = axi.Interface(ws*8)
+        self.memory = AXIMemorySim(bus, sequence)
+        self.submodules.dut = dma.DMA(bus)
+
+
+test_writes_full_stack = [
+    (0, 32, 0, 1),
+    (1, 40, 0, 1),
+    (0, 48, 0, 0),
+    (1, 50, 0, 0),
+]
+
+
+class FullStackTB(Module):
+    def __init__(self, ws):
+        self.ttl0 = Signal()
+        self.ttl1 = Signal()
+
+        self.submodules.phy0 = ttl_simple.Output(self.ttl0)
+        self.submodules.phy1 = ttl_simple.Output(self.ttl1)
+
+        rtio_channels = [
+            rtio.Channel.from_phy(self.phy0),
+            rtio.Channel.from_phy(self.phy1)
+        ]
+
+        sequence = encode_sequence(test_writes_full_stack, ws)
+
+        bus = axi.Interface(ws*8)
+        self.memory = AXIMemorySim(bus, sequence)
+        self.submodules.dut = dma.DMA(bus)
+        self.submodules.tsc = rtio.TSC()
+        self.submodules.rtio = rtio.Core(self.tsc, rtio_channels)
+        self.comb += self.dut.cri.connect(self.rtio.cri)
+
+
+class TestDMA(unittest.TestCase):
+    def test_dma_noerror(self):
+        tb = TB(8)
+
+        def do_writes():
+            yield from do_dma(tb.dut, 0)
+            yield from do_dma(tb.dut, 512)
+
+        received = []
+        @passive
+        def rtio_sim():
+            dut_cri = tb.dut.cri
+            while True:
+                cmd = yield dut_cri.cmd
+                if cmd == cri.commands["nop"]:
+                    pass
+                elif cmd == cri.commands["write"]:
+                    channel = yield dut_cri.chan_sel
+                    timestamp = yield dut_cri.o_timestamp
+                    address = yield dut_cri.o_address
+                    data = yield dut_cri.o_data
+                    received.append((channel, timestamp, address, data))
+
+                    yield dut_cri.o_status.eq(1)
+                    for i in range(prng.randrange(10)):
+                        yield
+                    yield dut_cri.o_status.eq(0)
+                else:
+                    self.fail("unexpected RTIO command")
+                yield
+
+        run_simulation(tb, [do_writes(), rtio_sim(), tb.memory.ar(), tb.memory.r()])
+        self.assertEqual(received, test_writes1 + test_writes2)
+
+    def test_full_stack(self):
+        tb = FullStackTB(8)
+
+        ttl_changes = []
+        @passive
+        def monitor():
+            old_ttl_states = [0, 0]
+            for time in itertools.count():
+                ttl_states = [
+                    (yield tb.ttl0),
+                    (yield tb.ttl1)
+                ]
+                for i, (old, new) in enumerate(zip(old_ttl_states, ttl_states)):
+                    if new != old:
+                        ttl_changes.append((time, i))
+                old_ttl_states = ttl_states
+                yield
+
+        run_simulation(tb, {"sys": [
+            do_dma(tb.dut, 0), monitor(),
+            (None for _ in range(70)),
+            tb.memory.ar(), tb.memory.r()
+        ]}, {"sys": 8, "rsys": 8, "rio": 8, "rio_phy": 8})
+
+        correct_changes = [(timestamp + 11, channel)
+                           for channel, timestamp, _, _ in test_writes_full_stack]
+        self.assertEqual(ttl_changes, correct_changes)

src/gateware/zc706.py

@@ -0,0 +1,732 @@
+#!/usr/bin/env python
+
+import argparse
+from operator import itemgetter
+
+from migen import *
+from migen.build.generic_platform import *
+from migen.genlib.resetsync import AsyncResetSynchronizer
+from migen.genlib.cdc import MultiReg
+from migen_axi.integration.soc_core import SoCCore
+from migen_axi.platforms import zc706
+from misoc.interconnect.csr import *
+from misoc.cores import gpio
+
+from artiq.gateware import rtio, nist_clock, nist_qc2
+from artiq.gateware.rtio.phy import ttl_simple, ttl_serdes_7series, dds, spi2, edge_counter
+from artiq.gateware.rtio.xilinx_clocking import fix_serdes_timing_path
+from artiq.gateware.drtio.transceiver import gtx_7series
+from artiq.gateware.drtio.siphaser import SiPhaser7Series
+from artiq.gateware.drtio.rx_synchronizer import XilinxRXSynchronizer
+from artiq.gateware.drtio import *
+
+import dma
+import analyzer
+import acpki
+import drtio_aux_controller
+import zynq_clocking
+from config import write_csr_file, write_mem_file, write_rustc_cfg_file
+
+class SMAClkinForward(Module):
+    def __init__(self, platform):
+        sma_clkin = platform.request("user_sma_clock")
+        sma_clkin_se = Signal()
+        si5324_clkin_se = Signal()
+        si5324_clkin = platform.request("si5324_clkin")
+        self.specials += [
+            Instance("IBUFDS", i_I=sma_clkin.p, i_IB=sma_clkin.n, o_O=sma_clkin_se),
+            Instance("ODDR", i_C=sma_clkin_se, i_CE=1, i_D1=1, i_D2=0, o_Q=si5324_clkin_se),
+            Instance("OBUFDS", i_I=si5324_clkin_se, o_O=si5324_clkin.p, o_OB=si5324_clkin.n)
+        ]
+
+
+class CLK200BootstrapClock(Module):
+    def __init__(self, platform, freq=125e6):
+        self.clock_domains.cd_bootstrap = ClockDomain(reset_less=True)
+        self.cd_bootstrap.clk.attr.add("keep")
+
+        clk200 = platform.request("clk200")
+        clk200_se = Signal()
+
+        pll_fb = Signal()
+        pll_clkout = Signal()
+        assert freq in [125e6, 100e6]
+        divide = int(1e9/freq)
+        self.specials += [
+            Instance("IBUFDS",
+                i_I=clk200.p, i_IB=clk200.n, o_O=clk200_se),
+            Instance("PLLE2_BASE",
+                p_CLKIN1_PERIOD=5.0,
+                i_CLKIN1=clk200_se,
+                i_CLKFBIN=pll_fb,
+                o_CLKFBOUT=pll_fb,
+
+                # VCO @ 1GHz
+                p_CLKFBOUT_MULT=5, p_DIVCLK_DIVIDE=1,
+
+                # 125MHz/100MHz for bootstrap
+                p_CLKOUT1_DIVIDE=divide, p_CLKOUT1_PHASE=0.0, o_CLKOUT1=pll_clkout,
+            ),
+            Instance("BUFG", i_I=pll_clkout, o_O=self.cd_bootstrap.clk)
+        ]
+
+
+# The NIST backplanes require setting VADJ to 3.3V by reprogramming the power supply.
+# This also changes the I/O standard for some on-board LEDs.
+leds_fmc33 = [
+    ("user_led_33", 0, Pins("Y21"), IOStandard("LVCMOS33")),
+    ("user_led_33", 1, Pins("G2"), IOStandard("LVCMOS15")),
+    ("user_led_33", 2, Pins("W21"), IOStandard("LVCMOS33")),
+    ("user_led_33", 3, Pins("A17"), IOStandard("LVCMOS15")),
+]
+
+# same deal as with LEDs - changed I/O standard.
+si5324_fmc33 = [
+    ("si5324_33", 0,
+        Subsignal("rst_n", Pins("W23"), IOStandard("LVCMOS33")),
+        Subsignal("int", Pins("AJ25"), IOStandard("LVCMOS33"))
+    ),
+]
+
+pmod1_33 = [
+    ("pmod1_33", 0, Pins("AJ21"), IOStandard("LVCMOS33")),
+    ("pmod1_33", 1, Pins("AK21"), IOStandard("LVCMOS33")),
+    ("pmod1_33", 2, Pins("AB21"), IOStandard("LVCMOS33")),
+    ("pmod1_33", 3, Pins("AB16"), IOStandard("LVCMOS33")),
+    # rest removed for use with dummy spi
+]
+
+_ams101_dac = [
+    ("ams101_dac", 0,
+        Subsignal("ldac", Pins("XADC:GPIO0")),
+        Subsignal("clk", Pins("XADC:GPIO1")),
+        Subsignal("mosi", Pins("XADC:GPIO2")),
+        Subsignal("cs_n", Pins("XADC:GPIO3")),
+        IOStandard("LVCMOS15")
+     )
+]
+
+_pmod_spi = [ 
+    ("pmod_spi", 0,
+        # PMOD_1 4-7 pins, same bank as sfp_tx_disable or user_sma_clock
+        Subsignal("miso", Pins("Y20"), IOStandard("LVCMOS25")),
+        Subsignal("clk", Pins("AA20"), IOStandard("LVCMOS25")),
+        Subsignal("mosi", Pins("AC18"), IOStandard("LVCMOS25")),
+        Subsignal("cs_n", Pins("AC19"), IOStandard("LVCMOS25")),
+        IOStandard("LVCMOS25")
+    )
+]
+
+
+def prepare_zc706_platform(platform):
+    platform.toolchain.bitstream_commands.extend([
+        "set_property BITSTREAM.GENERAL.COMPRESS True [current_design]",
+    ])
+
+class ZC706(SoCCore):
+    def __init__(self, acpki=False):
+        self.acpki = acpki
+
+        platform = zc706.Platform()
+        prepare_zc706_platform(platform)
+
+        ident = self.__class__.__name__
+        if self.acpki:
+            ident = "acpki_" + ident
+        SoCCore.__init__(self, platform=platform, csr_data_width=32, ident=ident, ps_cd_sys=False)
+
+        platform.add_extension(si5324_fmc33)
+        self.comb += platform.request("si5324_33").rst_n.eq(1)
+
+        cdr_clk = Signal()
+        cdr_clk_buf = Signal()
+        si5324_out = platform.request("si5324_clkout")
+        platform.add_period_constraint(si5324_out.p, 8.0)
+        self.specials += [
+            Instance("IBUFDS_GTE2",
+                i_CEB=0,
+                i_I=si5324_out.p, i_IB=si5324_out.n,
+                o_O=cdr_clk,
+                p_CLKCM_CFG="TRUE",
+                p_CLKRCV_TRST="TRUE",
+                p_CLKSWING_CFG=3),
+            Instance("BUFG", i_I=cdr_clk, o_O=cdr_clk_buf)
+        ]
+        self.config["HAS_SI5324"] = None
+        self.config["SI5324_AS_SYNTHESIZER"] = None
+        self.config["SI5324_SOFT_RESET"] = None
+        
+        self.submodules.bootstrap = CLK200BootstrapClock(platform)
+        self.submodules.sys_crg = zynq_clocking.SYSCRG(self.platform, self.ps7, cdr_clk_buf)
+        platform.add_false_path_constraints(
+            self.bootstrap.cd_bootstrap.clk, self.sys_crg.cd_sys.clk)
+        self.csr_devices.append("sys_crg")
+
+    def add_rtio(self, rtio_channels):
+        self.submodules.rtio_tsc = rtio.TSC(glbl_fine_ts_width=3)
+        self.submodules.rtio_core = rtio.Core(self.rtio_tsc, rtio_channels)
+        self.csr_devices.append("rtio_core")
+
+        if self.acpki:
+            self.config["KI_IMPL"] = "acp"
+            self.submodules.rtio = acpki.KernelInitiator(self.rtio_tsc,
+                                                         bus=self.ps7.s_axi_acp,
+                                                         user=self.ps7.s_axi_acp_user,
+                                                         evento=self.ps7.event.o)
+            self.csr_devices.append("rtio")
+        else:
+            self.config["KI_IMPL"] = "csr"
+            self.submodules.rtio = rtio.KernelInitiator(self.rtio_tsc, now64=True)
+            self.csr_devices.append("rtio")
+
+        self.submodules.rtio_dma = dma.DMA(self.ps7.s_axi_hp0)
+        self.csr_devices.append("rtio_dma")
+
+        self.submodules.cri_con = rtio.CRIInterconnectShared(
+            [self.rtio.cri, self.rtio_dma.cri],
+            [self.rtio_core.cri])
+        self.csr_devices.append("cri_con")
+
+        self.submodules.rtio_moninj = rtio.MonInj(rtio_channels)
+        self.csr_devices.append("rtio_moninj")
+
+        self.submodules.rtio_analyzer = analyzer.Analyzer(self.rtio_tsc, self.rtio_core.cri,
+                                                          self.ps7.s_axi_hp1)
+        self.csr_devices.append("rtio_analyzer")
+
+
+class _MasterBase(SoCCore):
+    def __init__(self, acpki=False, drtio100mhz=False):
+        self.acpki = acpki
+
+        clk_freq = 100e6 if drtio100mhz else 125e6
+
+        platform = zc706.Platform()
+        prepare_zc706_platform(platform)
+        ident = self.__class__.__name__
+        if self.acpki:
+            ident = "acpki_" + ident
+        SoCCore.__init__(self, platform=platform, csr_data_width=32, ident=ident, ps_cd_sys=False)
+
+        platform.add_extension(si5324_fmc33)
+
+        self.comb += platform.request("sfp_tx_disable_n").eq(1)
+        data_pads = [
+            platform.request("sfp"),
+            platform.request("user_sma_mgt")
+        ]
+
+        self.submodules += SMAClkinForward(self.platform)
+
+        # 1000BASE_BX10 Ethernet compatible, 125MHz RTIO clock
+        self.submodules.gt_drtio = gtx_7series.GTX(
+            clock_pads=platform.request("si5324_clkout"),
+            pads=data_pads,
+            clk_freq=clk_freq)
+        self.csr_devices.append("gt_drtio")
+
+        self.submodules.rtio_tsc = rtio.TSC(glbl_fine_ts_width=3)
+        ext_async_rst = Signal()
+        txout_buf = Signal()
+        gtx0 = self.gt_drtio.gtxs[0]
+        self.specials += Instance("BUFG", i_I=gtx0.txoutclk, o_O=txout_buf)
+        self.submodules.bootstrap = CLK200BootstrapClock(platform, clk_freq)
+        self.submodules.sys_crg = zynq_clocking.SYSCRG(
+            self.platform, 
+            self.ps7, 
+            txout_buf,
+            clk_sw=self.gt_drtio.stable_clkin.storage,
+            clk_sw_status=gtx0.tx_init.done,
+            ext_async_rst=ext_async_rst,
+            freq=clk_freq)
+        platform.add_false_path_constraints(
+            self.bootstrap.cd_bootstrap.clk, self.sys_crg.cd_sys.clk)
+        self.csr_devices.append("sys_crg")
+
+        self.comb += ext_async_rst.eq(self.sys_crg.clk_sw_fsm.o_clk_sw & ~gtx0.tx_init.done)
+        self.specials += MultiReg(self.sys_crg.clk_sw_fsm.o_clk_sw & self.sys_crg.mmcm_locked, self.gt_drtio.clk_path_ready, odomain="bootstrap")
+
+        drtio_csr_group = []
+        drtioaux_csr_group = []
+        drtioaux_memory_group = []
+        self.drtio_cri = []
+        for i in range(len(self.gt_drtio.channels)):
+            core_name = "drtio" + str(i)
+            coreaux_name = "drtioaux" + str(i)
+            memory_name = "drtioaux" + str(i) + "_mem"
+            drtio_csr_group.append(core_name)
+            drtioaux_csr_group.append(coreaux_name)
+            drtioaux_memory_group.append(memory_name)
+
+            cdr = ClockDomainsRenamer({"rtio_rx": "rtio_rx" + str(i)})
+
+            core = cdr(DRTIOMaster(
+                self.rtio_tsc, self.gt_drtio.channels[i]))
+            setattr(self.submodules, core_name, core)
+            self.drtio_cri.append(core.cri)
+            self.csr_devices.append(core_name)
+
+            coreaux = cdr(drtio_aux_controller.DRTIOAuxControllerBare(core.link_layer))
+            setattr(self.submodules, coreaux_name, coreaux)
+            self.csr_devices.append(coreaux_name)
+
+            mem_size = coreaux.get_mem_size()
+            memory_address = self.axi2csr.register_port(coreaux.get_tx_port(), mem_size)
+            self.axi2csr.register_port(coreaux.get_rx_port(), mem_size)
+            self.add_memory_region(memory_name, self.mem_map["csr"] + memory_address, mem_size * 2)
+        self.config["HAS_DRTIO"] = None
+        self.config["HAS_DRTIO_ROUTING"] = None
+        self.add_csr_group("drtio", drtio_csr_group)
+        self.add_csr_group("drtioaux", drtioaux_csr_group)
+        self.add_memory_group("drtioaux_mem", drtioaux_memory_group)
+
+        self.config["RTIO_FREQUENCY"] = str(self.gt_drtio.rtio_clk_freq/1e6)
+
+        self.submodules.si5324_rst_n = gpio.GPIOOut(platform.request("si5324_33").rst_n)
+        self.csr_devices.append("si5324_rst_n")
+        self.config["HAS_SI5324"] = None
+        self.config["SI5324_AS_SYNTHESIZER"] = None
+
+        # Constrain TX & RX timing for the first transceiver channel
+        # (First channel acts as master for phase alignment for all channels' TX)
+        platform.add_false_path_constraints(
+            gtx0.txoutclk, gtx0.rxoutclk)
+        # Constrain RX timing for the each transceiver channel
+        # (Each channel performs single-lane phase alignment for RX)
+        for gtx in self.gt_drtio.gtxs[1:]:
+            platform.add_false_path_constraints(
+                gtx0.txoutclk, gtx.rxoutclk)
+
+        fix_serdes_timing_path(platform)
+
+    def add_rtio(self, rtio_channels):
+        self.submodules.rtio_tsc = rtio.TSC(glbl_fine_ts_width=3)
+        self.submodules.rtio_core = rtio.Core(self.rtio_tsc, rtio_channels)
+        self.csr_devices.append("rtio_core")
+
+        if self.acpki:
+            self.config["KI_IMPL"] = "acp"
+            self.submodules.rtio = acpki.KernelInitiator(self.rtio_tsc,
+                                                         bus=self.ps7.s_axi_acp,
+                                                         user=self.ps7.s_axi_acp_user,
+                                                         evento=self.ps7.event.o)
+            self.csr_devices.append("rtio")
+        else:
+            self.config["KI_IMPL"] = "csr"
+            self.submodules.rtio = rtio.KernelInitiator(self.rtio_tsc, now64=True)
+            self.csr_devices.append("rtio")
+
+        self.submodules.rtio_dma = dma.DMA(self.ps7.s_axi_hp0)
+        self.csr_devices.append("rtio_dma")
+
+        self.submodules.cri_con = rtio.CRIInterconnectShared(
+            [self.rtio.cri, self.rtio_dma.cri],
+            [self.rtio_core.cri] + self.drtio_cri,
+            enable_routing=True)
+        self.csr_devices.append("cri_con")
+
+        self.submodules.rtio_moninj = rtio.MonInj(rtio_channels)
+        self.csr_devices.append("rtio_moninj")
+
+        self.submodules.rtio_analyzer = analyzer.Analyzer(self.rtio_tsc, self.rtio_core.cri,
+                                                          self.ps7.s_axi_hp1)
+        self.csr_devices.append("rtio_analyzer")
+
+        self.submodules.routing_table = rtio.RoutingTableAccess(self.cri_con)
+        self.csr_devices.append("routing_table")
+
+
+class _SatelliteBase(SoCCore):
+    def __init__(self, acpki=False, drtio100mhz=False):
+        self.acpki = acpki
+
+        clk_freq = 100e6 if drtio100mhz else 125e6
+
+        platform = zc706.Platform()
+        prepare_zc706_platform(platform)
+        ident = self.__class__.__name__
+        if self.acpki:
+            ident = "acpki_" + ident
+        SoCCore.__init__(self, platform=platform, csr_data_width=32, ident=ident, ps_cd_sys=False)
+
+        platform.add_extension(si5324_fmc33)
+
+        # SFP
+        self.comb += platform.request("sfp_tx_disable_n").eq(0)
+        data_pads = [
+            platform.request("sfp"),
+            platform.request("user_sma_mgt")
+        ]
+
+        self.submodules.rtio_tsc = rtio.TSC(glbl_fine_ts_width=3)
+
+        # 1000BASE_BX10 Ethernet compatible, 125MHz RTIO clock
+        self.submodules.gt_drtio = gtx_7series.GTX(
+            clock_pads=platform.request("si5324_clkout"),
+            pads=data_pads,
+            clk_freq=clk_freq)
+        self.csr_devices.append("gt_drtio")
+
+        ext_async_rst = Signal()
+        txout_buf = Signal()
+        txout_buf.attr.add("keep")
+        gtx0 = self.gt_drtio.gtxs[0]
+        self.specials += Instance(
+            "BUFG", 
+            i_I=gtx0.txoutclk, 
+            o_O=txout_buf)
+        self.submodules.bootstrap = CLK200BootstrapClock(platform, clk_freq)
+        self.submodules.sys_crg = zynq_clocking.SYSCRG(
+            self.platform, 
+            self.ps7, 
+            txout_buf,
+            clk_sw=self.gt_drtio.stable_clkin.storage,
+            clk_sw_status=gtx0.tx_init.done,
+            ext_async_rst=ext_async_rst,
+            freq=clk_freq)
+        platform.add_false_path_constraints(
+            self.bootstrap.cd_bootstrap.clk, self.sys_crg.cd_sys.clk)
+        self.csr_devices.append("sys_crg")
+
+        self.comb += ext_async_rst.eq(self.sys_crg.clk_sw_fsm.o_clk_sw & ~gtx0.tx_init.done)
+        self.specials += MultiReg(self.sys_crg.clk_sw_fsm.o_clk_sw & self.sys_crg.mmcm_locked, self.gt_drtio.clk_path_ready, odomain="bootstrap")
+
+        drtioaux_csr_group = []
+        drtioaux_memory_group = []
+        drtiorep_csr_group = []
+        self.drtio_cri = []
+        for i in range(len(self.gt_drtio.channels)):
+            coreaux_name = "drtioaux" + str(i)
+            memory_name = "drtioaux" + str(i) + "_mem"
+            drtioaux_csr_group.append(coreaux_name)
+            drtioaux_memory_group.append(memory_name)
+
+            cdr = ClockDomainsRenamer({"rtio_rx": "rtio_rx" + str(i)})
+
+            # Satellite
+            if i == 0:
+                self.submodules.rx_synchronizer = cdr(XilinxRXSynchronizer())
+                core = cdr(DRTIOSatellite(
+                    self.rtio_tsc, self.gt_drtio.channels[0], self.rx_synchronizer))
+                self.submodules.drtiosat = core
+                self.csr_devices.append("drtiosat")
+            # Repeaters
+            else:
+                corerep_name = "drtiorep" + str(i-1)
+                drtiorep_csr_group.append(corerep_name)
+                core = cdr(DRTIORepeater(
+                    self.rtio_tsc, self.gt_drtio.channels[i]))
+                setattr(self.submodules, corerep_name, core)
+                self.drtio_cri.append(core.cri)
+                self.csr_devices.append(corerep_name)
+
+            coreaux = cdr(drtio_aux_controller.DRTIOAuxControllerBare(core.link_layer))
+            setattr(self.submodules, coreaux_name, coreaux)
+            self.csr_devices.append(coreaux_name)
+
+            mem_size = coreaux.get_mem_size()
+            tx_port = coreaux.get_tx_port()
+            rx_port = coreaux.get_rx_port()
+            memory_address = self.axi2csr.register_port(tx_port, mem_size)
+            # rcv in upper half of the memory, thus added second
+            self.axi2csr.register_port(rx_port, mem_size)
+            # and registered in PS interface
+            # manually, because software refers to rx/tx by halves of entire memory block, not names
+            self.add_memory_region(memory_name, self.mem_map["csr"] + memory_address, mem_size * 2)
+        self.config["HAS_DRTIO"] = None
+        self.config["HAS_DRTIO_ROUTING"] = None
+        self.add_csr_group("drtioaux", drtioaux_csr_group)
+        self.add_csr_group("drtiorep", drtiorep_csr_group)
+        self.add_memory_group("drtioaux_mem", drtioaux_memory_group)
+
+        self.config["RTIO_FREQUENCY"] = str(self.gt_drtio.rtio_clk_freq/1e6)
+
+        # Si5324 Phaser
+        self.submodules.siphaser = SiPhaser7Series(
+            si5324_clkin=platform.request("si5324_clkin"),
+            rx_synchronizer=self.rx_synchronizer,
+            ultrascale=False,
+            rtio_clk_freq=self.gt_drtio.rtio_clk_freq)
+        platform.add_false_path_constraints(
+            self.sys_crg.cd_sys.clk, self.siphaser.mmcm_freerun_output)
+        self.csr_devices.append("siphaser")
+        self.submodules.si5324_rst_n = gpio.GPIOOut(platform.request("si5324_33").rst_n)
+        self.csr_devices.append("si5324_rst_n")
+        self.config["HAS_SI5324"] = None
+
+        rtio_clk_period = 1e9/self.gt_drtio.rtio_clk_freq
+        # Constrain TX & RX timing for the first transceiver channel
+        # (First channel acts as master for phase alignment for all channels' TX)
+        platform.add_false_path_constraints(
+            gtx0.txoutclk, gtx0.rxoutclk)
+        # Constrain RX timing for the each transceiver channel
+        # (Each channel performs single-lane phase alignment for RX)
+        for gtx in self.gt_drtio.gtxs[1:]:
+            platform.add_false_path_constraints(
+                self.sys_crg.cd_sys.clk, gtx.rxoutclk)
+
+        fix_serdes_timing_path(platform)
+
+    def add_rtio(self, rtio_channels):
+        self.submodules.rtio_moninj = rtio.MonInj(rtio_channels)
+        self.csr_devices.append("rtio_moninj")
+
+        if self.acpki:
+            self.config["KI_IMPL"] = "acp"
+            self.submodules.rtio = acpki.KernelInitiator(self.rtio_tsc,
+                                                         bus=self.ps7.s_axi_acp,
+                                                         user=self.ps7.s_axi_acp_user,
+                                                         evento=self.ps7.event.o)
+            self.csr_devices.append("rtio")
+        else:
+            self.config["KI_IMPL"] = "csr"
+            self.submodules.rtio = rtio.KernelInitiator(self.rtio_tsc, now64=True)
+            self.csr_devices.append("rtio")
+
+        self.submodules.rtio_dma = dma.DMA(self.ps7.s_axi_hp0)
+        self.csr_devices.append("rtio_dma")
+
+        self.submodules.local_io = SyncRTIO(self.rtio_tsc, rtio_channels)
+        self.comb += [
+            self.drtiosat.async_errors.eq(self.local_io.async_errors),
+            self.local_io.sed_spread_enable.eq(self.drtiosat.sed_spread_enable.storage)
+        ]
+        self.submodules.cri_con = rtio.CRIInterconnectShared(
+            [self.drtiosat.cri, self.rtio_dma.cri, self.rtio.cri],
+            [self.local_io.cri] + self.drtio_cri,
+            enable_routing=True)
+        self.csr_devices.append("cri_con")
+
+        self.submodules.rtio_analyzer = analyzer.Analyzer(self.rtio_tsc, self.local_io.cri,
+                                                          self.ps7.s_axi_hp1)
+        self.csr_devices.append("rtio_analyzer")
+
+        self.submodules.routing_table = rtio.RoutingTableAccess(self.cri_con)
+        self.csr_devices.append("routing_table")
+
+
+
+class _NIST_CLOCK_RTIO:
+    """
+    NIST clock hardware, with old backplane and 11 DDS channels
+    """
+    def __init__(self):
+        platform = self.platform
+        platform.add_extension(nist_clock.fmc_adapter_io)
+        platform.add_extension(leds_fmc33)
+        platform.add_extension(pmod1_33)
+        platform.add_extension(_ams101_dac)
+        platform.add_extension(_pmod_spi)
+
+        rtio_channels = []
+
+        for i in range(16):
+            if i % 4 == 3:
+                phy = ttl_serdes_7series.InOut_8X(platform.request("ttl", i))
+                self.submodules += phy
+                rtio_channels.append(rtio.Channel.from_phy(phy, ififo_depth=512))
+            else:
+                phy = ttl_serdes_7series.Output_8X(platform.request("ttl", i))
+                self.submodules += phy
+                rtio_channels.append(rtio.Channel.from_phy(phy))
+
+        for i in range(2):
+            phy = ttl_serdes_7series.InOut_8X(platform.request("pmt", i))
+            self.submodules += phy
+            rtio_channels.append(rtio.Channel.from_phy(phy, ififo_depth=512))
+
+        # no SMA GPIO, replaced with PMOD1_0
+        phy = ttl_serdes_7series.InOut_8X(platform.request("pmod1_33", 0))
+        self.submodules += phy
+        rtio_channels.append(rtio.Channel.from_phy(phy, ififo_depth=512))
+
+        phy = ttl_simple.Output(platform.request("user_led_33", 0))
+        self.submodules += phy
+        rtio_channels.append(rtio.Channel.from_phy(phy))
+
+        ams101_dac = self.platform.request("ams101_dac", 0)
+        phy = ttl_simple.Output(ams101_dac.ldac)
+        self.submodules += phy
+        rtio_channels.append(rtio.Channel.from_phy(phy))
+
+        phy = ttl_simple.ClockGen(platform.request("la32_p"))
+        self.submodules += phy
+        rtio_channels.append(rtio.Channel.from_phy(phy))
+
+        phy = spi2.SPIMaster(ams101_dac)
+        self.submodules += phy
+        rtio_channels.append(rtio.Channel.from_phy(
+            phy, ififo_depth=4))
+
+        for i in range(3):
+            phy = spi2.SPIMaster(self.platform.request("spi", i))
+            self.submodules += phy
+            rtio_channels.append(rtio.Channel.from_phy(
+                phy, ififo_depth=128))
+
+        # no SDIO on PL side, dummy SPI placeholder instead
+        phy = spi2.SPIMaster(platform.request("pmod_spi"))
+        self.submodules += phy
+        rtio_channels.append(rtio.Channel.from_phy(phy, ififo_depth=4))
+
+        phy = dds.AD9914(platform.request("dds"), 11, onehot=True)
+        self.submodules += phy
+        rtio_channels.append(rtio.Channel.from_phy(phy, ififo_depth=4))
+
+        self.config["RTIO_LOG_CHANNEL"] = len(rtio_channels)
+        rtio_channels.append(rtio.LogChannel())
+
+        self.add_rtio(rtio_channels)
+
+
+class _NIST_QC2_RTIO:
+    """
+    NIST QC2 hardware, as used in Quantum I and Quantum II, with new backplane
+    and 24 DDS channels.  Two backplanes are used.
+    """
+    def __init__(self):
+        platform = self.platform
+        platform.add_extension(nist_qc2.fmc_adapter_io)
+        platform.add_extension(leds_fmc33)
+        platform.add_extension(_ams101_dac)
+        platform.add_extension(pmod1_33)
+
+        rtio_channels = []
+        edge_counter_phy = []
+
+        # All TTL channels are In+Out capable
+        for i in range(40):
+            phy = ttl_serdes_7series.InOut_8X(platform.request("ttl", i))
+            self.submodules += phy
+            rtio_channels.append(rtio.Channel.from_phy(phy, ififo_depth=512))
+            # first four TTLs will also have edge counters
+            if i < 4:
+                edge_counter_phy.append(phy)
+
+        # no SMA GPIO, replaced with PMOD1_0
+        phy = ttl_serdes_7series.InOut_8X(platform.request("pmod1_33", 0))
+        self.submodules += phy
+        rtio_channels.append(rtio.Channel.from_phy(phy, ififo_depth=512))
+
+        phy = ttl_simple.Output(platform.request("user_led_33", 0))
+        self.submodules += phy
+        rtio_channels.append(rtio.Channel.from_phy(phy))
+
+        ams101_dac = self.platform.request("ams101_dac", 0)
+        phy = ttl_simple.Output(ams101_dac.ldac)
+        self.submodules += phy
+        rtio_channels.append(rtio.Channel.from_phy(phy))
+
+        # CLK0, CLK1 are for clock generators, on backplane SMP connectors
+        for i in range(2):
+            phy = ttl_simple.ClockGen(
+                platform.request("clkout", i))
+            self.submodules += phy
+            rtio_channels.append(rtio.Channel.from_phy(phy))
+        
+        phy = spi2.SPIMaster(ams101_dac)
+        self.submodules += phy
+        rtio_channels.append(rtio.Channel.from_phy(
+            phy, ififo_depth=4))
+
+        for i in range(4):
+            phy = spi2.SPIMaster(self.platform.request("spi", i))
+            self.submodules += phy
+            rtio_channels.append(rtio.Channel.from_phy(
+                phy, ififo_depth=128))
+
+        for backplane_offset in range(2):
+            phy = dds.AD9914(
+                platform.request("dds", backplane_offset), 12, onehot=True)
+            self.submodules += phy
+            rtio_channels.append(rtio.Channel.from_phy(phy, ififo_depth=4))
+        
+        for phy in edge_counter_phy:
+            counter = edge_counter.SimpleEdgeCounter(phy.input_state)
+            self.submodules += counter
+            rtio_channels.append(rtio.Channel.from_phy(counter))
+
+        self.config["RTIO_LOG_CHANNEL"] = len(rtio_channels)
+        rtio_channels.append(rtio.LogChannel())
+
+        self.add_rtio(rtio_channels)
+
+
+class NIST_CLOCK(ZC706, _NIST_CLOCK_RTIO):
+    def __init__(self, acpki, drtio100mhz):
+        ZC706.__init__(self, acpki)
+        self.submodules += SMAClkinForward(self.platform)
+        _NIST_CLOCK_RTIO.__init__(self)
+
+class NIST_CLOCK_Master(_MasterBase, _NIST_CLOCK_RTIO):
+    def __init__(self, acpki, drtio100mhz):
+        _MasterBase.__init__(self, acpki, drtio100mhz)
+        _NIST_CLOCK_RTIO.__init__(self)
+
+class NIST_CLOCK_Satellite(_SatelliteBase, _NIST_CLOCK_RTIO):
+    def __init__(self, acpki, drtio100mhz):
+        _SatelliteBase.__init__(self, acpki, drtio100mhz)
+        _NIST_CLOCK_RTIO.__init__(self)
+
+class NIST_QC2(ZC706, _NIST_QC2_RTIO):
+    def __init__(self, acpki, drtio100mhz):
+        ZC706.__init__(self, acpki)
+        self.submodules += SMAClkinForward(self.platform)
+        _NIST_QC2_RTIO.__init__(self)
+
+class NIST_QC2_Master(_MasterBase, _NIST_QC2_RTIO):
+    def __init__(self, acpki, drtio100mhz):
+        _MasterBase.__init__(self, acpki, drtio100mhz)
+        _NIST_QC2_RTIO.__init__(self)
+
+class NIST_QC2_Satellite(_SatelliteBase, _NIST_QC2_RTIO):
+    def __init__(self, acpki, drtio100mhz):
+        _SatelliteBase.__init__(self, acpki, drtio100mhz)
+        _NIST_QC2_RTIO.__init__(self)
+
+VARIANTS = {cls.__name__.lower(): cls for cls in [NIST_CLOCK, NIST_CLOCK_Master, NIST_CLOCK_Satellite,
+                                                  NIST_QC2, NIST_QC2_Master, NIST_QC2_Satellite]}
+
+def main():
+    parser = argparse.ArgumentParser(
+        description="ARTIQ port to the ZC706 Zynq development kit")
+    parser.add_argument("-r", default=None,
+        help="build Rust interface into the specified file")
+    parser.add_argument("-m", default=None,
+        help="build Rust memory interface into the specified file")
+    parser.add_argument("-c", default=None,
+        help="build Rust compiler configuration into the specified file")
+    parser.add_argument("-g", default=None,
+        help="build gateware into the specified directory")
+    parser.add_argument("-V", "--variant", default="nist_clock",
+                        help="variant: "
+                             "[acpki_]nist_clock/nist_qc2[_master/_satellite][_100mhz]"
+                             "(default: %(default)s)")
+    args = parser.parse_args()
+
+    variant = args.variant.lower()
+    acpki = variant.startswith("acpki_")
+    if acpki:
+        variant = variant[6:]
+    drtio100mhz = variant.endswith("_100mhz")
+    if drtio100mhz:
+        variant = variant[:-7]
+    try:
+        cls = VARIANTS[variant]
+    except KeyError:
+        raise SystemExit("Invalid variant (-V/--variant)")
+
+    soc = cls(acpki=acpki, drtio100mhz=drtio100mhz)
+    soc.finalize()
+
+    if args.r is not None:
+        write_csr_file(soc, args.r)
+    if args.m is not None:
+        write_mem_file(soc, args.m)
+    if args.c is not None:
+        write_rustc_cfg_file(soc, args.c)
+    if args.g is not None:
+        soc.build(build_dir=args.g)
+
+if __name__ == "__main__":
+    main()

src/gateware/zynq_clocking.py

@@ -0,0 +1,154 @@
+from migen import *
+from migen.genlib.cdc import MultiReg
+from migen.genlib.resetsync import AsyncResetSynchronizer
+from misoc.interconnect.csr import *
+
+
+class ClockSwitchFSM(Module):
+    def __init__(self):
+        self.i_clk_sw = Signal()
+
+        self.o_clk_sw = Signal()
+        self.o_reset = Signal()
+
+        ###
+
+        i_switch = Signal()
+        o_switch = Signal()
+        reset = Signal()
+
+        # at 125MHz bootstrap cd, will get around 0.5ms
+        delay_counter = Signal(16, reset=0xFFFF)
+
+        # register to prevent glitches
+        self.sync.bootstrap += [
+            self.o_clk_sw.eq(o_switch),
+            self.o_reset.eq(reset),
+        ]
+
+        self.o_clk_sw.attr.add("no_retiming")
+        self.o_reset.attr.add("no_retiming")
+        self.i_clk_sw.attr.add("no_retiming")
+        i_switch.attr.add("no_retiming")
+
+        self.specials += MultiReg(self.i_clk_sw, i_switch, "bootstrap")
+
+        fsm = ClockDomainsRenamer("bootstrap")(FSM(reset_state="START"))
+
+        self.submodules += fsm
+
+        fsm.act("START",
+            If(i_switch & ~o_switch,
+                NextState("RESET_START"))
+        )
+        
+        fsm.act("RESET_START",
+            reset.eq(1),
+            If(delay_counter == 0,
+                NextValue(delay_counter, 0xFFFF),
+                NextState("CLOCK_SWITCH")
+            ).Else(
+                NextValue(delay_counter, delay_counter-1),
+            )
+        )
+
+        fsm.act("CLOCK_SWITCH",
+            reset.eq(1),
+            NextValue(o_switch, 1),
+            NextValue(delay_counter, delay_counter-1),
+            If(delay_counter == 0,
+                NextState("END"))
+        )
+        fsm.act("END",
+            NextValue(o_switch, 1),
+            reset.eq(0))
+
+
+class SYSCRG(Module, AutoCSR):
+    def __init__(self, platform, ps7, main_clk, clk_sw=None, clk_sw_status=None, freq=125e6, ext_async_rst=None, ):
+        # assumes bootstrap clock is same freq as main and sys output
+        self.clock_domains.cd_sys = ClockDomain()
+        self.clock_domains.cd_sys4x = ClockDomain(reset_less=True)
+        self.clock_domains.cd_sys5x = ClockDomain(reset_less=True)
+        self.clock_domains.cd_clk200 = ClockDomain()
+
+        self.current_clock = CSRStatus()
+
+        self.cd_sys.clk.attr.add("keep")
+
+        bootstrap_clk = ClockSignal("bootstrap")
+
+        period = 1e9/freq
+
+        self.submodules.clk_sw_fsm = ClockSwitchFSM()
+
+        if clk_sw is None:
+            self.clock_switch = CSRStorage()
+            self.comb += self.clk_sw_fsm.i_clk_sw.eq(self.clock_switch.storage)
+        else:
+            self.comb += self.clk_sw_fsm.i_clk_sw.eq(clk_sw)
+
+        self.mmcm_locked = Signal()
+        mmcm_sys = Signal()
+        mmcm_sys4x = Signal()
+        mmcm_sys5x = Signal()
+        mmcm_clk208 = Signal()
+        mmcm_fb_clk = Signal()
+        self.specials += [
+            Instance("MMCME2_ADV",
+                p_STARTUP_WAIT="FALSE", o_LOCKED=self.mmcm_locked,
+                p_BANDWIDTH="HIGH",
+                p_REF_JITTER1=0.001,
+                p_CLKIN1_PERIOD=period, i_CLKIN1=main_clk,
+                p_CLKIN2_PERIOD=period, i_CLKIN2=bootstrap_clk,
+                i_CLKINSEL=self.clk_sw_fsm.o_clk_sw,
+
+                # VCO @ 1.25GHz
+                p_CLKFBOUT_MULT_F=10, p_DIVCLK_DIVIDE=1,
+                i_CLKFBIN=mmcm_fb_clk,
+                i_RST=self.clk_sw_fsm.o_reset,
+
+                o_CLKFBOUT=mmcm_fb_clk,
+
+                p_CLKOUT0_DIVIDE_F=2.5, p_CLKOUT0_PHASE=0.0, o_CLKOUT0=mmcm_sys4x,
+
+                # 125MHz
+                p_CLKOUT1_DIVIDE=10, p_CLKOUT1_PHASE=0.0, o_CLKOUT1=mmcm_sys,
+
+                # 625MHz
+                p_CLKOUT2_DIVIDE=2, p_CLKOUT2_PHASE=0.0, o_CLKOUT2=mmcm_sys5x,
+
+                # 208MHz
+                p_CLKOUT3_DIVIDE=6, p_CLKOUT3_PHASE=0.0, o_CLKOUT3=mmcm_clk208,
+            ),
+            Instance("BUFG", i_I=mmcm_sys5x, o_O=self.cd_sys5x.clk),
+            Instance("BUFG", i_I=mmcm_sys, o_O=self.cd_sys.clk),
+            Instance("BUFG", i_I=mmcm_sys4x, o_O=self.cd_sys4x.clk),
+            Instance("BUFG", i_I=mmcm_clk208, o_O=self.cd_clk200.clk),
+        ]
+
+        if ext_async_rst is not None:
+            self.specials += [
+                AsyncResetSynchronizer(self.cd_sys, ~self.mmcm_locked | ext_async_rst),
+                AsyncResetSynchronizer(self.cd_clk200, ~self.mmcm_locked | ext_async_rst),
+            ]
+        else:
+            self.specials += [
+                AsyncResetSynchronizer(self.cd_sys, ~self.mmcm_locked),
+                AsyncResetSynchronizer(self.cd_clk200, ~self.mmcm_locked),
+            ]
+
+        reset_counter = Signal(4, reset=15)
+        ic_reset = Signal(reset=1)
+        self.sync.clk200 += \
+            If(reset_counter != 0,
+                reset_counter.eq(reset_counter - 1)
+            ).Else(
+                ic_reset.eq(0)
+            )
+        self.specials += Instance("IDELAYCTRL", i_REFCLK=ClockSignal("clk200"), i_RST=ic_reset)
+
+        if clk_sw_status is None:
+            self.comb += self.current_clock.status.eq(self.clk_sw_fsm.o_clk_sw)
+        else:
+            self.comb += self.current_clock.status.eq(clk_sw_status)

src/libboard_artiq/Cargo.toml.tpl

@@ -0,0 +1,34 @@
+[package]
+name = "libboard_artiq"
+version = "0.0.0"
+authors = ["M-Labs"]
+edition = "2018"
+
+[lib]
+name = "libboard_artiq"
+
+[features]
+target_zc706 = ["libboard_zynq/target_zc706", "libconfig/target_zc706"]
+target_kasli_soc = ["libboard_zynq/target_kasli_soc", "libconfig/target_kasli_soc"]
+target_ebaz4205 = ["libboard_zynq/target_ebaz4205", "libconfig/target_ebaz4205"]
+calibrate_wrpll_skew = []
+
+[build-dependencies]
+build_zynq = { path = "../libbuild_zynq" }
+
+[dependencies]
+log = "0.4"
+log_buffer = { version = "1.2" }
+crc = { version = "1.7", default-features = false }
+core_io = { version = "0.1", features = ["collections"] }
+embedded-hal = "0.2"
+nb = "1.0"
+void = { version = "1", default-features = false }
+
+io = { path = "../libio", features = ["byteorder"] }
+libboard_zynq = { path = "@@ZYNQ_RS@@/libboard_zynq" }
+libsupport_zynq = { path = "@@ZYNQ_RS@@/libsupport_zynq", default-features = false, features = ["alloc_core"] }
+libregister = { path = "@@ZYNQ_RS@@/libregister" }
+libconfig = { path = "@@ZYNQ_RS@@/libconfig", features = ["fat_lfn"] }
+libcortex_a9 = { path = "@@ZYNQ_RS@@/libcortex_a9" }
+libasync = { path = "@@ZYNQ_RS@@/libasync" }

src/libboard_artiq/build.rs

@@ -0,0 +1,5 @@
+extern crate build_zynq;
+
+fn main() {
+    build_zynq::cfg();
+}

src/libboard_artiq/src/drtio_eem.rs

@@ -0,0 +1,245 @@
+use embedded_hal::prelude::_embedded_hal_blocking_delay_DelayUs;
+use libboard_zynq::timer::GlobalTimer;
+use libconfig::Config;
+use libsupport_zynq::alloc::format;
+use log::{debug, error, info};
+
+use crate::pl;
+
+struct SerdesConfig {
+    pub delay: [u8; 4],
+}
+
+impl SerdesConfig {
+    pub fn as_bytes(&self) -> &[u8] {
+        unsafe {
+            core::slice::from_raw_parts(
+                (self as *const SerdesConfig) as *const u8,
+                core::mem::size_of::<SerdesConfig>(),
+            )
+        }
+    }
+}
+
+fn select_lane(lane_no: u8) {
+    unsafe {
+        pl::csr::eem_transceiver::lane_sel_write(lane_no);
+    }
+}
+
+fn apply_delay(tap: u8, timer: &mut GlobalTimer) {
+    unsafe {
+        pl::csr::eem_transceiver::dly_cnt_in_write(tap);
+        pl::csr::eem_transceiver::dly_ld_write(1);
+        timer.delay_us(1);
+        assert!(tap as u8 == pl::csr::eem_transceiver::dly_cnt_out_read());
+    }
+}
+
+fn apply_config(config: &SerdesConfig, timer: &mut GlobalTimer) {
+    for lane_no in 0..4 {
+        select_lane(lane_no as u8);
+        apply_delay(config.delay[lane_no], timer);
+    }
+}
+
+unsafe fn assign_delay(timer: &mut GlobalTimer) -> SerdesConfig {
+    // Select an appropriate delay for lane 0
+    select_lane(0);
+
+    //
+
+    let mut best_dly = None;
+
+    loop {
+        let mut prev = None;
+        for curr_dly in 0..32 {
+            //let read_align = read_align_fn(curr_dly, timer);
+            let curr_low_rate = read_align(curr_dly, timer);
+
+            if let Some(prev_low_rate) = prev {
+                // This is potentially a crossover position
+                if prev_low_rate <= curr_low_rate && curr_low_rate >= 0.5 {
+                    let prev_dev = 0.5 - prev_low_rate;
+                    let curr_dev = curr_low_rate - 0.5;
+                    let selected_idx = if prev_dev < curr_dev { curr_dly - 1 } else { curr_dly };
+
+                    // The setup setup/hold calibration timing (even with
+                    // tolerance) might be invalid in other lanes due to skew.
+                    // 5 taps is very conservative, generally it is 1 or 2
+                    if selected_idx < 5 {
+                        prev = None;
+                        continue;
+                    } else {
+                        best_dly = Some(selected_idx);
+                        break;
+                    }
+                }
+            }
+
+            // Only rising slope from <= 0.5 can result in a rising low rate
+            // crossover at 50%.
+            if curr_low_rate <= 0.5 {
+                prev = Some(curr_low_rate);
+            }
+        }
+
+        if best_dly.is_none() {
+            error!("setup/hold timing calibration failed, retry in 1s...");
+            timer.delay_us(1_000_000);
+        } else {
+            break;
+        }
+    }
+
+    let best_dly = best_dly.unwrap();
+
+    apply_delay(best_dly, timer);
+    let mut delay_list = [best_dly; 4];
+
+    // Assign delay for other lanes
+    for lane_no in 1..=3 {
+        select_lane(lane_no as u8);
+
+        let mut min_deviation = 0.5;
+        let mut min_idx = 0;
+        for dly_delta in -3..=3 {
+            let index = (best_dly as isize + dly_delta) as u8;
+            let low_rate = read_align(index, timer);
+            // abs() from f32 is not available in core library
+            let deviation = if low_rate < 0.5 { 0.5 - low_rate } else { low_rate - 0.5 };
+
+            if deviation < min_deviation {
+                min_deviation = deviation;
+                min_idx = index;
+            }
+        }
+
+        apply_delay(min_idx, timer);
+        delay_list[lane_no] = min_idx;
+    }
+
+    debug!("setup/hold timing calibration: {:?}", delay_list);
+
+    SerdesConfig { delay: delay_list }
+}
+
+fn read_align(dly: u8, timer: &mut GlobalTimer) -> f32 {
+    unsafe {
+        apply_delay(dly, timer);
+        pl::csr::eem_transceiver::counter_reset_write(1);
+
+        pl::csr::eem_transceiver::counter_enable_write(1);
+        timer.delay_us(2000);
+        pl::csr::eem_transceiver::counter_enable_write(0);
+
+        let (high, low) = (
+            pl::csr::eem_transceiver::counter_high_count_read(),
+            pl::csr::eem_transceiver::counter_low_count_read(),
+        );
+        if pl::csr::eem_transceiver::counter_overflow_read() == 1 {
+            panic!("Unexpected phase detector counter overflow");
+        }
+
+        low as f32 / (low + high) as f32
+    }
+}
+
+unsafe fn align_comma(timer: &mut GlobalTimer) {
+    loop {
+        for slip in 1..=10 {
+            // The soft transceiver has 2 8b10b decoders, which receives lane
+            // 0/1 and lane 2/3 respectively. The decoder are time-multiplexed
+            // to decode exactly 1 lane each sysclk cycle.
+            //
+            // The decoder decodes lane 0/2 data on odd sysclk cycles, buffer
+            // on even cycles, and vice versa for lane 1/3. Data/Clock latency
+            // could change timing. The extend bit flips the decoding timing,
+            // so lane 0/2 data are decoded on even cycles, and lane 1/3 data
+            // are decoded on odd cycles.
+            //
+            // This is needed because transmitting/receiving a 8b10b character
+            // takes 2 sysclk cycles. Adjusting bitslip only via ISERDES
+            // limits the range to 1 cycle. The wordslip bit extends the range
+            // to 2 sysclk cycles.
+            pl::csr::eem_transceiver::wordslip_write((slip > 5) as u8);
+
+            // Apply a double bitslip since the ISERDES is 2x oversampled.
+            // Bitslip is used for comma alignment purposes once setup/hold
+            // timing is met.
+            pl::csr::eem_transceiver::bitslip_write(1);
+            pl::csr::eem_transceiver::bitslip_write(1);
+            timer.delay_us(1);
+
+            pl::csr::eem_transceiver::comma_align_reset_write(1);
+            timer.delay_us(100);
+
+            if pl::csr::eem_transceiver::comma_read() == 1 {
+                debug!("comma alignment completed after {} bitslips", slip);
+                return;
+            }
+        }
+
+        error!("comma alignment failed, retrying in 1s...");
+        timer.delay_us(1_000_000);
+    }
+}
+
+pub unsafe fn align_wordslip(timer: &mut GlobalTimer, trx_no: u8) -> bool {
+    pl::csr::eem_transceiver::transceiver_sel_write(trx_no);
+
+    for slip in 0..=1 {
+        pl::csr::eem_transceiver::wordslip_write(slip as u8);
+        timer.delay_us(1);
+        pl::csr::eem_transceiver::comma_align_reset_write(1);
+        timer.delay_us(100);
+
+        if pl::csr::eem_transceiver::comma_read() == 1 {
+            debug!("comma alignment completed with {} wordslip", slip);
+            return true;
+        }
+    }
+
+    false
+}
+
+pub fn init(timer: &mut GlobalTimer, cfg: &Config) {
+    for trx_no in 0..pl::csr::CONFIG_EEM_DRTIO_COUNT {
+        unsafe {
+            pl::csr::eem_transceiver::transceiver_sel_write(trx_no as u8);
+        }
+
+        let key = format!("eem_drtio_delay{}", trx_no);
+
+        let cfg_read = cfg.read(&key);
+        match cfg_read {
+            Ok(record) => {
+                info!("loading calibrated timing values from sd card");
+                unsafe {
+                    apply_config(&*(record.as_ptr() as *const SerdesConfig), timer);
+                }
+            }
+            Err(_) => {
+                info!("calibrating...");
+                let config = unsafe { assign_delay(timer) };
+
+                match cfg.write(&key, config.as_bytes().to_vec()) {
+                    Ok(()) => {
+                        info!("storing calibration timing values into sd card");
+                    }
+                    Err(e) => {
+                        error!(
+                            "calibration successful but calibration timing values cannot be stored into sd card. \
+                             Error:{}",
+                            e
+                        );
+                    }
+                };
+            }
+        }
+
+        unsafe {
+            align_comma(timer);
+        }
+    }
+}

src/libboard_artiq/src/drtio_routing.rs

@@ -0,0 +1,107 @@
+use core::fmt;
+
+use libconfig::Config;
+use log::{info, warn};
+
+#[cfg(has_drtio_routing)]
+use crate::pl::csr;
+
+#[cfg(has_drtio_routing)]
+pub const DEST_COUNT: usize = 256;
+#[cfg(not(has_drtio_routing))]
+pub const DEST_COUNT: usize = 0;
+pub const MAX_HOPS: usize = 32;
+pub const INVALID_HOP: u8 = 0xff;
+
+pub struct RoutingTable(pub [[u8; MAX_HOPS]; DEST_COUNT]);
+
+impl RoutingTable {
+    // default routing table is for star topology with no repeaters
+    pub fn default_master(default_n_links: usize) -> RoutingTable {
+        let mut ret = RoutingTable([[INVALID_HOP; MAX_HOPS]; DEST_COUNT]);
+        let n_entries = default_n_links + 1; // include local RTIO
+        for i in 0..n_entries {
+            ret.0[i][0] = i as u8;
+        }
+        for i in 1..n_entries {
+            ret.0[i][1] = 0x00;
+        }
+        ret
+    }
+
+    // use this by default on satellite, as they receive
+    // the routing table from the master
+    pub fn default_empty() -> RoutingTable {
+        RoutingTable([[INVALID_HOP; MAX_HOPS]; DEST_COUNT])
+    }
+}
+
+impl fmt::Display for RoutingTable {
+    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+        write!(f, "RoutingTable {{")?;
+        for i in 0..DEST_COUNT {
+            if self.0[i][0] != INVALID_HOP {
+                write!(f, " {}:", i)?;
+                for j in 0..MAX_HOPS {
+                    if self.0[i][j] == INVALID_HOP {
+                        break;
+                    }
+                    write!(f, " {}", self.0[i][j])?;
+                }
+                write!(f, ";")?;
+            }
+        }
+        write!(f, " }}")?;
+        Ok(())
+    }
+}
+
+pub fn config_routing_table(default_n_links: usize, cfg: &Config) -> RoutingTable {
+    let mut ret = RoutingTable::default_master(default_n_links);
+    if let Ok(data) = cfg.read("routing_table") {
+        if data.len() == DEST_COUNT * MAX_HOPS {
+            for i in 0..DEST_COUNT {
+                for j in 0..MAX_HOPS {
+                    ret.0[i][j] = data[i * MAX_HOPS + j];
+                }
+            }
+        } else {
+            warn!("length of the configured routing table is incorrect, using default");
+        }
+    } else {
+        info!("could not read routing table from configuration, using default");
+    }
+    info!("routing table: {}", ret);
+    ret
+}
+
+#[cfg(has_drtio_routing)]
+pub fn interconnect_enable(routing_table: &RoutingTable, rank: u8, destination: u8) {
+    let hop = routing_table.0[destination as usize][rank as usize];
+    unsafe {
+        csr::routing_table::destination_write(destination);
+        csr::routing_table::hop_write(hop);
+    }
+}
+
+#[cfg(has_drtio_routing)]
+pub fn interconnect_disable(destination: u8) {
+    unsafe {
+        csr::routing_table::destination_write(destination);
+        csr::routing_table::hop_write(INVALID_HOP);
+    }
+}
+
+#[cfg(has_drtio_routing)]
+pub fn interconnect_enable_all(routing_table: &RoutingTable, rank: u8) {
+    for i in 0..DEST_COUNT {
+        interconnect_enable(routing_table, rank, i as u8);
+    }
+}
+
+#[cfg(has_drtio_routing)]
+pub fn interconnect_disable_all() {
+    for i in 0..DEST_COUNT {
+        interconnect_disable(i as u8);
+    }
+}

src/libboard_artiq/src/drtioaux.rs

@@ -0,0 +1,142 @@
+use core::slice;
+
+use core_io::{Error as IoError, ErrorKind as IoErrorKind};
+use crc;
+use io::{proto::{ProtoRead, ProtoWrite},
+         Cursor};
+use libboard_zynq::{time::Milliseconds, timer::GlobalTimer};
+
+pub use crate::drtioaux_proto::Packet;
+use crate::{drtioaux_proto::Error as ProtocolError, mem::mem::DRTIOAUX_MEM, pl::csr::DRTIOAUX};
+
+#[derive(Debug)]
+pub enum Error {
+    GatewareError,
+    CorruptedPacket,
+
+    LinkDown,
+    TimedOut,
+    UnexpectedReply,
+
+    RoutingError,
+
+    Protocol(ProtocolError),
+}
+
+impl From<ProtocolError> for Error {
+    fn from(value: ProtocolError) -> Error {
+        Error::Protocol(value)
+    }
+}
+
+impl From<IoError> for Error {
+    fn from(value: IoError) -> Error {
+        Error::Protocol(ProtocolError::Io(value))
+    }
+}
+
+pub fn reset(linkno: u8) {
+    let linkno = linkno as usize;
+    unsafe {
+        // clear buffer first to limit race window with buffer overflow
+        // error. We assume the CPU is fast enough so that no two packets
+        // will be received between the buffer and the error flag are cleared.
+        (DRTIOAUX[linkno].aux_rx_present_write)(1);
+        (DRTIOAUX[linkno].aux_rx_error_write)(1);
+    }
+}
+
+pub fn has_rx_error(linkno: u8) -> bool {
+    let linkno = linkno as usize;
+    unsafe {
+        let error = (DRTIOAUX[linkno].aux_rx_error_read)() != 0;
+        if error {
+            (DRTIOAUX[linkno].aux_rx_error_write)(1)
+        }
+        error
+    }
+}
+
+fn receive<F, T>(linkno: u8, f: F) -> Result<Option<T>, Error>
+where F: FnOnce(&[u8]) -> Result<T, Error> {
+    let linkidx = linkno as usize;
+    unsafe {
+        if (DRTIOAUX[linkidx].aux_rx_present_read)() == 1 {
+            let read_ptr = (DRTIOAUX[linkidx].aux_read_pointer_read)() as usize;
+            let ptr = (DRTIOAUX_MEM[linkidx].base + DRTIOAUX_MEM[linkidx].size / 2 + read_ptr * 0x400) as *mut u32;
+            let result = f(slice::from_raw_parts(ptr as *mut u8, 0x400 as usize));
+            (DRTIOAUX[linkidx].aux_rx_present_write)(1);
+            Ok(Some(result?))
+        } else {
+            Ok(None)
+        }
+    }
+}
+
+pub fn recv(linkno: u8) -> Result<Option<Packet>, Error> {
+    if has_rx_error(linkno) {
+        return Err(Error::GatewareError);
+    }
+
+    receive(linkno, |buffer| {
+        if buffer.len() < 8 {
+            return Err(IoError::new(IoErrorKind::UnexpectedEof, "Unexpected end").into());
+        }
+
+        let mut reader = Cursor::new(buffer);
+
+        let packet = Packet::read_from(&mut reader)?;
+        let padding = (12 - (reader.position() % 8)) % 8;
+        let checksum_at = reader.position() + padding;
+        let checksum = crc::crc32::checksum_ieee(&reader.get_ref()[0..checksum_at]);
+        reader.set_position(checksum_at);
+        if reader.read_u32()? != checksum {
+            return Err(Error::CorruptedPacket);
+        }
+        Ok(packet)
+    })
+}
+
+pub fn recv_timeout(linkno: u8, timeout_ms: Option<u64>, timer: GlobalTimer) -> Result<Packet, Error> {
+    let timeout_ms = Milliseconds(timeout_ms.unwrap_or(10));
+    let limit = timer.get_time() + timeout_ms;
+    while timer.get_time() < limit {
+        match recv(linkno)? {
+            None => (),
+            Some(packet) => return Ok(packet),
+        }
+    }
+    Err(Error::TimedOut)
+}
+
+fn transmit<F>(linkno: u8, f: F) -> Result<(), Error>
+where F: FnOnce(&mut [u8]) -> Result<usize, Error> {
+    let linkno = linkno as usize;
+    unsafe {
+        while (DRTIOAUX[linkno].aux_tx_read)() != 0 {}
+        let ptr = DRTIOAUX_MEM[linkno].base as *mut u32;
+        let len = f(slice::from_raw_parts_mut(ptr as *mut u8, 0x400 as usize))?;
+        (DRTIOAUX[linkno].aux_tx_length_write)(len as u16);
+        (DRTIOAUX[linkno].aux_tx_write)(1);
+        Ok(())
+    }
+}
+
+pub fn send(linkno: u8, packet: &Packet) -> Result<(), Error> {
+    transmit(linkno, |buffer| {
+        let mut writer = Cursor::new(buffer);
+
+        packet.write_to(&mut writer)?;
+
+        // Pad till offset 4, insert checksum there
+        let padding = (12 - (writer.position() % 8)) % 8;
+        for _ in 0..padding {
+            writer.write_u8(0)?;
+        }
+
+        let checksum = crc::crc32::checksum_ieee(&writer.get_ref()[0..writer.position()]);
+        writer.write_u32(checksum)?;
+
+        Ok(writer.position())
+    })
+}

src/libboard_artiq/src/drtioaux_async.rs

@@ -0,0 +1,130 @@
+use core::slice;
+
+use core_io::{Error as IoError, ErrorKind as IoErrorKind};
+use crc;
+use io::{proto::{ProtoRead, ProtoWrite},
+         Cursor};
+use libasync::{block_async, task};
+use libboard_zynq::{time::Milliseconds, timer::GlobalTimer};
+use nb;
+use void::Void;
+
+pub use crate::drtioaux_proto::Packet;
+use crate::{drtioaux::{has_rx_error, Error},
+            mem::mem::DRTIOAUX_MEM,
+            pl::csr::DRTIOAUX};
+
+pub async fn reset(linkno: u8) {
+    let linkno = linkno as usize;
+    unsafe {
+        // clear buffer first to limit race window with buffer overflow
+        // error. We assume the CPU is fast enough so that no two packets
+        // will be received between the buffer and the error flag are cleared.
+        (DRTIOAUX[linkno].aux_rx_present_write)(1);
+        (DRTIOAUX[linkno].aux_rx_error_write)(1);
+    }
+}
+
+fn tx_ready(linkno: usize) -> nb::Result<(), Void> {
+    unsafe {
+        if (DRTIOAUX[linkno].aux_tx_read)() != 0 {
+            Err(nb::Error::WouldBlock)
+        } else {
+            Ok(())
+        }
+    }
+}
+
+async fn receive<F, T>(linkno: u8, f: F) -> Result<Option<T>, Error>
+where F: FnOnce(&[u8]) -> Result<T, Error> {
+    let linkidx = linkno as usize;
+    unsafe {
+        if (DRTIOAUX[linkidx].aux_rx_present_read)() == 1 {
+            let read_ptr = (DRTIOAUX[linkidx].aux_read_pointer_read)() as usize;
+            let ptr = (DRTIOAUX_MEM[linkidx].base + DRTIOAUX_MEM[linkidx].size / 2 + read_ptr * 0x400) as *mut u32;
+            let result = f(slice::from_raw_parts(ptr as *mut u8, 0x400 as usize));
+            (DRTIOAUX[linkidx].aux_rx_present_write)(1);
+            Ok(Some(result?))
+        } else {
+            Ok(None)
+        }
+    }
+}
+
+pub async fn recv(linkno: u8) -> Result<Option<Packet>, Error> {
+    if has_rx_error(linkno) {
+        return Err(Error::GatewareError);
+    }
+
+    receive(linkno, |buffer| {
+        if buffer.len() < 8 {
+            return Err(IoError::new(IoErrorKind::UnexpectedEof, "Unexpected end").into());
+        }
+
+        let mut reader = Cursor::new(buffer);
+
+        let packet = Packet::read_from(&mut reader)?;
+        let padding = (12 - (reader.position() % 8)) % 8;
+        let checksum_at = reader.position() + padding;
+        let checksum = crc::crc32::checksum_ieee(&reader.get_ref()[0..checksum_at]);
+        reader.set_position(checksum_at);
+        if reader.read_u32()? != checksum {
+            return Err(Error::CorruptedPacket);
+        }
+        Ok(packet)
+    })
+    .await
+}
+
+pub async fn recv_timeout(linkno: u8, timeout_ms: Option<u64>, timer: GlobalTimer) -> Result<Packet, Error> {
+    let timeout_ms = Milliseconds(timeout_ms.unwrap_or(10));
+    let limit = timer.get_time() + timeout_ms;
+    let mut would_block = false;
+    while timer.get_time() < limit {
+        // to ensure one last time recv would run one last time
+        // in case async would return after timeout
+        if would_block {
+            task::r#yield().await;
+        }
+        match recv(linkno).await? {
+            None => {
+                would_block = true;
+            }
+            Some(packet) => return Ok(packet),
+        }
+    }
+    Err(Error::TimedOut)
+}
+
+async fn transmit<F>(linkno: u8, f: F) -> Result<(), Error>
+where F: FnOnce(&mut [u8]) -> Result<usize, Error> {
+    let linkno = linkno as usize;
+    unsafe {
+        let _ = block_async!(tx_ready(linkno)).await;
+        let ptr = DRTIOAUX_MEM[linkno].base as *mut u32;
+        let len = f(slice::from_raw_parts_mut(ptr as *mut u8, 0x400 as usize))?;
+        (DRTIOAUX[linkno].aux_tx_length_write)(len as u16);
+        (DRTIOAUX[linkno].aux_tx_write)(1);
+        Ok(())
+    }
+}
+
+pub async fn send(linkno: u8, packet: &Packet) -> Result<(), Error> {
+    transmit(linkno, |buffer| {
+        let mut writer = Cursor::new(buffer);
+
+        packet.write_to(&mut writer)?;
+
+        // Pad till offset 4, insert checksum there
+        let padding = (12 - (writer.position() % 8)) % 8;
+        for _ in 0..padding {
+            writer.write_u8(0)?;
+        }
+
+        let checksum = crc::crc32::checksum_ieee(&writer.get_ref()[0..writer.position()]);
+        writer.write_u32(checksum)?;
+
+        Ok(writer.position())
+    })
+    .await
+}

src/libboard_artiq/src/fiq.rs

@@ -0,0 +1,22 @@
+use libboard_zynq::{println, stdio};
+use libcortex_a9::{interrupt_handler, regs::MPIDR};
+use libregister::RegisterR;
+
+#[cfg(has_si549)]
+use crate::si549;
+
+interrupt_handler!(FIQ, fiq, __irq_stack0_start, __irq_stack1_start, {
+    match MPIDR.read().cpu_id() {
+        0 => {
+            // nFIQ is driven directly and bypass GIC
+            #[cfg(has_si549)]
+            si549::wrpll::interrupt_handler();
+            return;
+        }
+        _ => {}
+    };
+
+    stdio::drop_uart();
+    println!("FIQ");
+    loop {}
+});

src/libboard_artiq/src/grabber.rs

@@ -0,0 +1,163 @@
+use log::info;
+
+use crate::pl::csr;
+
+#[derive(PartialEq, Clone, Copy)]
+enum State {
+    Reset,
+    ExitReset,
+    Lock,
+    Align,
+    Watch,
+}
+
+#[derive(Clone, Copy)]
+struct Info {
+    state: State,
+    frame_size: (u16, u16),
+}
+
+static mut INFO: [Info; csr::GRABBER_LEN] = [Info {
+    state: State::Reset,
+    frame_size: (0, 0),
+}; csr::GRABBER_LEN];
+
+fn get_pll_reset(g: usize) -> bool {
+    unsafe { (csr::GRABBER[g].pll_reset_read)() != 0 }
+}
+
+fn set_pll_reset(g: usize, reset: bool) {
+    let val = if reset { 1 } else { 0 };
+    unsafe { (csr::GRABBER[g].pll_reset_write)(val) }
+}
+
+fn pll_locked(g: usize) -> bool {
+    unsafe { (csr::GRABBER[g].pll_locked_read)() != 0 }
+}
+
+fn clock_pattern_ok(g: usize) -> bool {
+    unsafe { (csr::GRABBER[g].clk_sampled_read)() == 0b1100011 }
+}
+
+fn clock_pattern_ok_filter(g: usize) -> bool {
+    for _ in 0..128 {
+        if !clock_pattern_ok(g) {
+            return false;
+        }
+    }
+    true
+}
+
+fn phase_shift(g: usize, direction: u8) {
+    unsafe {
+        (csr::GRABBER[g].phase_shift_write)(direction);
+        while (csr::GRABBER[g].phase_shift_done_read)() == 0 {}
+    }
+}
+
+fn clock_align(g: usize) -> bool {
+    while clock_pattern_ok_filter(g) {
+        phase_shift(g, 1);
+    }
+    phase_shift(g, 1);
+
+    let mut count = 0;
+    while !clock_pattern_ok_filter(g) {
+        phase_shift(g, 1);
+        count += 1;
+        if count > 1024 {
+            return false;
+        }
+    }
+
+    let mut window = 1;
+    phase_shift(g, 1);
+    while clock_pattern_ok_filter(g) {
+        phase_shift(g, 1);
+        window += 1;
+    }
+
+    for _ in 0..window / 2 {
+        phase_shift(g, 0);
+    }
+
+    true
+}
+
+fn get_last_pixels(g: usize) -> (u16, u16) {
+    unsafe { ((csr::GRABBER[g].last_x_read)(), (csr::GRABBER[g].last_y_read)()) }
+}
+
+fn get_video_clock(g: usize) -> u32 {
+    let freq_count = unsafe { (csr::GRABBER[g].freq_count_read)() } as u32;
+    2 * freq_count * (csr::CONFIG_CLOCK_FREQUENCY / 1000) / (511 * 1000)
+}
+
+pub fn tick() {
+    for g in 0..csr::GRABBER.len() {
+        let next = match unsafe { INFO[g].state } {
+            State::Reset => {
+                set_pll_reset(g, true);
+                unsafe {
+                    INFO[g].frame_size = (0, 0);
+                }
+                State::ExitReset
+            }
+            State::ExitReset => {
+                if get_pll_reset(g) {
+                    set_pll_reset(g, false);
+                    State::Lock
+                } else {
+                    State::ExitReset
+                }
+            }
+            State::Lock => {
+                if pll_locked(g) {
+                    info!("grabber{} locked: {}MHz", g, get_video_clock(g));
+                    State::Align
+                } else {
+                    State::Lock
+                }
+            }
+            State::Align => {
+                if pll_locked(g) {
+                    if clock_align(g) {
+                        info!("grabber{} alignment success", g);
+                        State::Watch
+                    } else {
+                        info!("grabber{} alignment failure", g);
+                        State::Reset
+                    }
+                } else {
+                    info!("grabber{} lock lost", g);
+                    State::Reset
+                }
+            }
+            State::Watch => {
+                if pll_locked(g) {
+                    if clock_pattern_ok(g) {
+                        let last_xy = get_last_pixels(g);
+                        if last_xy != unsafe { INFO[g].frame_size } {
+                            // x capture is on ~LVAL which is after
+                            // the last increment on DVAL
+                            // y capture is on ~FVAL which coincides with the
+                            // last increment on ~LVAL
+                            info!("grabber{} frame size: {}x{}", g, last_xy.0, last_xy.1 + 1);
+                            unsafe { INFO[g].frame_size = last_xy }
+                        }
+                        State::Watch
+                    } else {
+                        info!("grabber{} alignment lost", g);
+                        State::Reset
+                    }
+                } else {
+                    info!("grabber{} lock lost", g);
+                    State::Reset
+                }
+            }
+        };
+        unsafe {
+            INFO[g].state = next;
+        }
+    }
+}

src/libboard_artiq/src/io_expander.rs

@@ -0,0 +1,183 @@
+use libboard_zynq::i2c;
+use log::info;
+
+#[cfg(has_virtual_leds)]
+use crate::pl::csr;
+
+// Only the bare minimum registers. Bits/IO connections equivalent between IC types.
+struct Registers {
+    // PCA9539 equivalent register names in comments
+    iodira: u8, // Configuration Port 0
+    iodirb: u8, // Configuration Port 1
+    gpioa: u8,  // Output Port 0
+    gpiob: u8,  // Output Port 1
+}
+
+//IO expanders pins
+const IODIR_OUT_SFP_TX_DISABLE: u8 = 0x02;
+const IODIR_OUT_SFP_LED: u8 = 0x40;
+#[cfg(hw_rev = "v1.0")]
+const IODIR_OUT_SFP0_LED: u8 = 0x40;
+#[cfg(hw_rev = "v1.1")]
+const IODIR_OUT_SFP0_LED: u8 = 0x80;
+#[cfg(has_si549)]
+const IODIR_CLK_SEL: u8 = 0x80; // out
+#[cfg(has_si5324)]
+const IODIR_CLK_SEL: u8 = 0x00; // in
+
+//IO expander port direction
+const IODIR0: [u8; 2] = [
+    0xFF & !IODIR_OUT_SFP_TX_DISABLE & !IODIR_OUT_SFP0_LED,
+    0xFF & !IODIR_OUT_SFP_TX_DISABLE & !IODIR_OUT_SFP_LED & !IODIR_CLK_SEL,
+];
+
+const IODIR1: [u8; 2] = [
+    0xFF & !IODIR_OUT_SFP_TX_DISABLE & !IODIR_OUT_SFP_LED,
+    0xFF & !IODIR_OUT_SFP_TX_DISABLE & !IODIR_OUT_SFP_LED,
+];
+
+pub struct IoExpander {
+    address: u8,
+    #[cfg(has_virtual_leds)]
+    virtual_led_mapping: &'static [(u8, u8, u8)],
+    iodir: [u8; 2],
+    out_current: [u8; 2],
+    out_target: [u8; 2],
+    registers: Registers,
+}
+
+impl IoExpander {
+    pub fn new(i2c: &mut i2c::I2c, index: u8) -> Result<Self, &'static str> {
+        #[cfg(all(hw_rev = "v1.0", has_virtual_leds))]
+        const VIRTUAL_LED_MAPPING0: [(u8, u8, u8); 2] = [(0, 0, 6), (1, 1, 6)];
+        #[cfg(all(hw_rev = "v1.1", has_virtual_leds))]
+        const VIRTUAL_LED_MAPPING0: [(u8, u8, u8); 2] = [(0, 0, 7), (1, 1, 6)];
+        #[cfg(has_virtual_leds)]
+        const VIRTUAL_LED_MAPPING1: [(u8, u8, u8); 2] = [(2, 0, 6), (3, 1, 6)];
+
+        // Both expanders on SHARED I2C bus
+        let mut io_expander = match index {
+            0 => IoExpander {
+                address: 0x40,
+                #[cfg(has_virtual_leds)]
+                virtual_led_mapping: &VIRTUAL_LED_MAPPING0,
+                iodir: IODIR0,
+                out_current: [0; 2],
+                out_target: [0; 2],
+                registers: Registers {
+                    iodira: 0x00,
+                    iodirb: 0x01,
+                    gpioa: 0x12,
+                    gpiob: 0x13,
+                },
+            },
+            1 => IoExpander {
+                address: 0x42,
+                #[cfg(has_virtual_leds)]
+                virtual_led_mapping: &VIRTUAL_LED_MAPPING1,
+                iodir: IODIR1,
+                out_current: [0; 2],
+                out_target: [0; 2],
+                registers: Registers {
+                    iodira: 0x00,
+                    iodirb: 0x01,
+                    gpioa: 0x12,
+                    gpiob: 0x13,
+                },
+            },
+            _ => return Err("incorrect I/O expander index"),
+        };
+        if !io_expander.check_ack(i2c)? {
+            info!("MCP23017 io expander {} not found. Checking for PCA9539.", index);
+            io_expander.address += 0xa8; // translate to PCA9539 addresses (see schematic)
+            io_expander.registers = Registers {
+                iodira: 0x06,
+                iodirb: 0x07,
+                gpioa: 0x02,
+                gpiob: 0x03,
+            };
+            if !io_expander.check_ack(i2c)? {
+                return Err("Neither MCP23017 nor PCA9539 io expander found.");
+            };
+        }
+        Ok(io_expander)
+    }
+
+    fn select(&self, i2c: &mut i2c::I2c) -> Result<(), &'static str> {
+        i2c.pca954x_select(0x70, None)?;
+        i2c.pca954x_select(0x71, Some(3))?;
+        Ok(())
+    }
+
+    fn write(&self, i2c: &mut i2c::I2c, addr: u8, value: u8) -> Result<(), &'static str> {
+        i2c.start()?;
+        i2c.write(self.address)?;
+        i2c.write(addr)?;
+        i2c.write(value)?;
+        i2c.stop()?;
+        Ok(())
+    }
+
+    fn check_ack(&self, i2c: &mut i2c::I2c) -> Result<bool, &'static str> {
+        // Check for ack from io expander
+        self.select(i2c)?;
+        i2c.start()?;
+        let ack = i2c.write(self.address)?;
+        i2c.stop()?;
+        Ok(ack)
+    }
+
+    fn update_iodir(&self, i2c: &mut i2c::I2c) -> Result<(), &'static str> {
+        self.write(i2c, self.registers.iodira, self.iodir[0])?;
+        self.write(i2c, self.registers.iodirb, self.iodir[1])?;
+        Ok(())
+    }
+
+    pub fn init(&mut self, i2c: &mut i2c::I2c) -> Result<(), &'static str> {
+        self.select(i2c)?;
+
+        self.update_iodir(i2c)?;
+
+        self.out_current[0] = 0x00;
+        self.write(i2c, self.registers.gpioa, 0x00)?;
+        self.out_current[1] = 0x00;
+        self.write(i2c, self.registers.gpiob, 0x00)?;
+        Ok(())
+    }
+
+    pub fn set_oe(&mut self, i2c: &mut i2c::I2c, port: u8, outputs: u8) -> Result<(), &'static str> {
+        self.iodir[port as usize] &= !outputs;
+        self.update_iodir(i2c)?;
+        Ok(())
+    }
+
+    pub fn set(&mut self, port: u8, bit: u8, high: bool) {
+        if high {
+            self.out_target[port as usize] |= 1 << bit;
+        } else {
+            self.out_target[port as usize] &= !(1 << bit);
+        }
+    }
+
+    pub fn service(&mut self, i2c: &mut i2c::I2c) -> Result<(), &'static str> {
+        #[cfg(has_virtual_leds)]
+        for (led, port, bit) in self.virtual_led_mapping.iter() {
+            let level = unsafe { csr::virtual_leds::status_read() >> led & 1 };
+            self.set(*port, *bit, level != 0);
+        }
+
+        if self.out_target != self.out_current {
+            self.select(i2c)?;
+            if self.out_target[0] != self.out_current[0] {
+                self.write(i2c, self.registers.gpioa, self.out_target[0])?;
+                self.out_current[0] = self.out_target[0];
+            }
+            if self.out_target[1] != self.out_current[1] {
+                self.write(i2c, self.registers.gpiob, self.out_target[1])?;
+                self.out_current[1] = self.out_target[1];
+            }
+        }
+
+        Ok(())
+    }
+}

src/libboard_artiq/src/lib.rs

@@ -0,0 +1,56 @@
+#![no_std]
+#![feature(never_type)]
+#![feature(naked_functions)]
+#![feature(asm)]
+
+extern crate core_io;
+extern crate crc;
+extern crate embedded_hal;
+extern crate io;
+extern crate libasync;
+extern crate libboard_zynq;
+extern crate libconfig;
+extern crate libcortex_a9;
+extern crate libregister;
+extern crate log;
+extern crate log_buffer;
+
+pub mod drtio_routing;
+#[cfg(has_drtio)]
+pub mod drtioaux;
+#[cfg(has_drtio)]
+pub mod drtioaux_async;
+pub mod drtioaux_proto;
+pub mod fiq;
+#[cfg(feature = "target_kasli_soc")]
+pub mod io_expander;
+pub mod logger;
+#[cfg(has_drtio)]
+#[rustfmt::skip]
+#[path = "../../../build/mem.rs"]
+pub mod mem;
+#[rustfmt::skip]
+#[path = "../../../build/pl.rs"]
+pub mod pl;
+#[cfg(has_drtio_eem)]
+pub mod drtio_eem;
+#[cfg(has_grabber)]
+pub mod grabber;
+#[cfg(has_si5324)]
+pub mod si5324;
+#[cfg(has_si549)]
+pub mod si549;
+use core::{cmp, str};
+
+pub fn identifier_read(buf: &mut [u8]) -> &str {
+    unsafe {
+        pl::csr::identifier::address_write(0);
+        let len = pl::csr::identifier::data_read();
+        let len = cmp::min(len, buf.len() as u8);
+        for i in 0..len {
+            pl::csr::identifier::address_write(1 + i);
+            buf[i as usize] = pl::csr::identifier::data_read();
+        }
+        str::from_utf8_unchecked(&buf[..len as usize])
+    }
+}

src/libboard_artiq/src/logger.rs

@@ -0,0 +1,131 @@
+use core::{cell::Cell, fmt::Write};
+
+use libboard_zynq::{println, timer::GlobalTimer};
+use libcortex_a9::mutex::{Mutex, MutexGuard};
+use log::{LevelFilter, Log};
+use log_buffer::LogBuffer;
+
+pub struct LogBufferRef<'a> {
+    buffer: MutexGuard<'a, LogBuffer<&'static mut [u8]>>,
+    old_log_level: LevelFilter,
+}
+
+impl<'a> LogBufferRef<'a> {
+    fn new(buffer: MutexGuard<'a, LogBuffer<&'static mut [u8]>>) -> LogBufferRef<'a> {
+        let old_log_level = log::max_level();
+        log::set_max_level(LevelFilter::Off);
+        LogBufferRef { buffer, old_log_level }
+    }
+
+    pub fn is_empty(&self) -> bool {
+        self.buffer.is_empty()
+    }
+
+    pub fn clear(&mut self) {
+        self.buffer.clear()
+    }
+
+    pub fn extract(&mut self) -> &str {
+        self.buffer.extract()
+    }
+}
+
+impl<'a> Drop for LogBufferRef<'a> {
+    fn drop(&mut self) {
+        log::set_max_level(self.old_log_level)
+    }
+}
+
+pub struct BufferLogger {
+    buffer: Mutex<LogBuffer<&'static mut [u8]>>,
+    uart_filter: Cell<LevelFilter>,
+    buffer_filter: Cell<LevelFilter>,
+}
+
+static mut LOGGER: Option<BufferLogger> = None;
+
+impl BufferLogger {
+    pub fn new(buffer: &'static mut [u8]) -> BufferLogger {
+        BufferLogger {
+            buffer: Mutex::new(LogBuffer::new(buffer)),
+            uart_filter: Cell::new(LevelFilter::Info),
+            buffer_filter: Cell::new(LevelFilter::Trace),
+        }
+    }
+
+    pub fn register(self) {
+        unsafe {
+            LOGGER = Some(self);
+            log::set_logger(LOGGER.as_ref().unwrap()).expect("global logger can only be initialized once");
+        }
+    }
+
+    pub unsafe fn get_logger() -> &'static mut Option<BufferLogger> {
+        &mut LOGGER
+    }
+
+    pub fn buffer<'a>(&'a self) -> Option<LogBufferRef<'a>> {
+        self.buffer.try_lock().map(LogBufferRef::new)
+    }
+
+    pub fn uart_log_level(&self) -> LevelFilter {
+        self.uart_filter.get()
+    }
+
+    pub fn set_uart_log_level(&self, max_level: LevelFilter) {
+        self.uart_filter.set(max_level)
+    }
+
+    pub fn buffer_log_level(&self) -> LevelFilter {
+        self.buffer_filter.get()
+    }
+
+    /// this should be reserved for mgmt module
+    pub fn set_buffer_log_level(&self, max_level: LevelFilter) {
+        self.buffer_filter.set(max_level)
+    }
+}
+
+// required for impl Log
+unsafe impl Sync for BufferLogger {}
+
+impl Log for BufferLogger {
+    fn enabled(&self, _metadata: &log::Metadata) -> bool {
+        true
+    }
+
+    fn log(&self, record: &log::Record) {
+        if self.enabled(record.metadata()) {
+            let timestamp = unsafe { GlobalTimer::get() }.get_us().0;
+            let seconds = timestamp / 1_000_000;
+            let micros = timestamp % 1_000_000;
+
+            if record.level() <= self.buffer_log_level() {
+                let mut buffer = self.buffer.lock();
+                writeln!(
+                    buffer,
+                    "[{:6}.{:06}s] {:>5}({}): {}",
+                    seconds,
+                    micros,
+                    record.level(),
+                    record.target(),
+                    record.args()
+                )
+                .unwrap();
+            }
+
+            if record.level() <= self.uart_log_level() {
+                println!(
+                    "[{:6}.{:06}s] {:>5}({}): {}",
+                    seconds,
+                    micros,
+                    record.level(),
+                    record.target(),
+                    record.args()
+                );
+            }
+        }
+    }
+
+    fn flush(&self) {}
+}

src/libboard_artiq/src/si5324.rs

@@ -0,0 +1,362 @@
+use core::result;
+
+use embedded_hal::blocking::delay::DelayUs;
+use libboard_zynq::{i2c::I2c, time::Milliseconds, timer::GlobalTimer};
+use log::info;
+
+#[cfg(not(si5324_soft_reset))]
+use crate::pl::csr;
+
+type Result<T> = result::Result<T, &'static str>;
+
+const ADDRESS: u8 = 0x68;
+
+#[cfg(not(si5324_soft_reset))]
+fn hard_reset(timer: &mut GlobalTimer) {
+    unsafe {
+        csr::si5324_rst_n::out_write(0);
+    }
+    timer.delay_us(1_000);
+    unsafe {
+        csr::si5324_rst_n::out_write(1);
+    }
+    timer.delay_us(10_000);
+}
+
+// NOTE: the logical parameters DO NOT MAP to physical values written
+// into registers. They have to be mapped; see the datasheet.
+// DSPLLsim reports the logical parameters in the design summary, not
+// the physical register values.
+pub struct FrequencySettings {
+    pub n1_hs: u8,
+    pub nc1_ls: u32,
+    pub n2_hs: u8,
+    pub n2_ls: u32,
+    pub n31: u32,
+    pub n32: u32,
+    pub bwsel: u8,
+    pub crystal_as_ckin2: bool,
+}
+
+pub enum Input {
+    Ckin1,
+    Ckin2,
+}
+
+fn map_frequency_settings(settings: &FrequencySettings) -> Result<FrequencySettings> {
+    if settings.nc1_ls != 0 && (settings.nc1_ls % 2) == 1 {
+        return Err("NC1_LS must be 0 or even");
+    }
+    if settings.nc1_ls > (1 << 20) {
+        return Err("NC1_LS is too high");
+    }
+    if (settings.n2_ls % 2) == 1 {
+        return Err("N2_LS must be even");
+    }
+    if settings.n2_ls > (1 << 20) {
+        return Err("N2_LS is too high");
+    }
+    if settings.n31 > (1 << 19) {
+        return Err("N31 is too high");
+    }
+    if settings.n32 > (1 << 19) {
+        return Err("N32 is too high");
+    }
+    let r = FrequencySettings {
+        n1_hs: match settings.n1_hs {
+            4 => 0b000,
+            5 => 0b001,
+            6 => 0b010,
+            7 => 0b011,
+            8 => 0b100,
+            9 => 0b101,
+            10 => 0b110,
+            11 => 0b111,
+            _ => return Err("N1_HS has an invalid value"),
+        },
+        nc1_ls: settings.nc1_ls - 1,
+        n2_hs: match settings.n2_hs {
+            4 => 0b000,
+            5 => 0b001,
+            6 => 0b010,
+            7 => 0b011,
+            8 => 0b100,
+            9 => 0b101,
+            10 => 0b110,
+            11 => 0b111,
+            _ => return Err("N2_HS has an invalid value"),
+        },
+        n2_ls: settings.n2_ls - 1,
+        n31: settings.n31 - 1,
+        n32: settings.n32 - 1,
+        bwsel: settings.bwsel,
+        crystal_as_ckin2: settings.crystal_as_ckin2,
+    };
+    Ok(r)
+}
+
+fn write(i2c: &mut I2c, reg: u8, val: u8) -> Result<()> {
+    i2c.start().unwrap();
+    if !i2c.write(ADDRESS << 1).unwrap() {
+        return Err("Si5324 failed to ack write address");
+    }
+    if !i2c.write(reg).unwrap() {
+        return Err("Si5324 failed to ack register");
+    }
+    if !i2c.write(val).unwrap() {
+        return Err("Si5324 failed to ack value");
+    }
+    i2c.stop().unwrap();
+    Ok(())
+}
+
+#[allow(dead_code)]
+fn write_no_ack_value(i2c: &mut I2c, reg: u8, val: u8) -> Result<()> {
+    i2c.start().unwrap();
+    if !i2c.write(ADDRESS << 1).unwrap() {
+        return Err("Si5324 failed to ack write address");
+    }
+    if !i2c.write(reg).unwrap() {
+        return Err("Si5324 failed to ack register");
+    }
+    i2c.write(val).unwrap();
+    i2c.stop().unwrap();
+    Ok(())
+}
+
+fn read(i2c: &mut I2c, reg: u8) -> Result<u8> {
+    i2c.start().unwrap();
+    if !i2c.write(ADDRESS << 1).unwrap() {
+        return Err("Si5324 failed to ack write address");
+    }
+    if !i2c.write(reg).unwrap() {
+        return Err("Si5324 failed to ack register");
+    }
+    i2c.restart().unwrap();
+    if !i2c.write((ADDRESS << 1) | 1).unwrap() {
+        return Err("Si5324 failed to ack read address");
+    }
+    let val = i2c.read(false).unwrap();
+    i2c.stop().unwrap();
+    Ok(val)
+}
+
+fn rmw<F>(i2c: &mut I2c, reg: u8, f: F) -> Result<()>
+where F: Fn(u8) -> u8 {
+    let value = read(i2c, reg)?;
+    write(i2c, reg, f(value))?;
+    Ok(())
+}
+
+fn ident(i2c: &mut I2c) -> Result<u16> {
+    Ok(((read(i2c, 134)? as u16) << 8) | (read(i2c, 135)? as u16))
+}
+
+#[cfg(si5324_soft_reset)]
+fn soft_reset(i2c: &mut I2c, timer: &mut GlobalTimer) -> Result<()> {
+    let val = read(i2c, 136)?;
+    write_no_ack_value(i2c, 136, val | 0x80)?;
+    timer.delay_us(10_000);
+    Ok(())
+}
+
+fn has_xtal(i2c: &mut I2c) -> Result<bool> {
+    Ok((read(i2c, 129)? & 0x01) == 0) // LOSX_INT=0
+}
+
+fn has_ckin(i2c: &mut I2c, input: Input) -> Result<bool> {
+    match input {
+        Input::Ckin1 => Ok((read(i2c, 129)? & 0x02) == 0), // LOS1_INT=0
+        Input::Ckin2 => Ok((read(i2c, 129)? & 0x04) == 0), // LOS2_INT=0
+    }
+}
+
+fn locked(i2c: &mut I2c) -> Result<bool> {
+    Ok((read(i2c, 130)? & 0x01) == 0) // LOL_INT=0
+}
+
+fn monitor_lock(i2c: &mut I2c, timer: &mut GlobalTimer) -> Result<()> {
+    info!("waiting for Si5324 lock...");
+    let timeout = timer.get_time() + Milliseconds(20_000);
+    while !locked(i2c)? {
+        // Yes, lock can be really slow.
+        if timer.get_time() > timeout {
+            return Err("Si5324 lock timeout");
+        }
+    }
+    info!("  ...locked");
+    Ok(())
+}
+
+fn init(i2c: &mut I2c, timer: &mut GlobalTimer) -> Result<()> {
+    #[cfg(not(si5324_soft_reset))]
+    hard_reset(timer);
+
+    #[cfg(feature = "target_kasli_soc")]
+    {
+        i2c.pca954x_select(0x70, None)?;
+        i2c.pca954x_select(0x71, Some(3))?;
+    }
+    #[cfg(feature = "target_zc706")]
+    {
+        i2c.pca954x_select(0x74, Some(4))?;
+    }
+
+    if ident(i2c)? != 0x0182 {
+        return Err("Si5324 does not have expected product number");
+    }
+
+    #[cfg(si5324_soft_reset)]
+    soft_reset(i2c, timer)?;
+    Ok(())
+}
+
+pub fn bypass(i2c: &mut I2c, input: Input, timer: &mut GlobalTimer) -> Result<()> {
+    let cksel_reg = match input {
+        Input::Ckin1 => 0b00,
+        Input::Ckin2 => 0b01,
+    };
+    init(i2c, timer)?;
+    rmw(i2c, 21, |v| v & 0xfe)?; // CKSEL_PIN=0
+    rmw(i2c, 3, |v| (v & 0x3f) | (cksel_reg << 6))?; // CKSEL_REG
+    rmw(i2c, 4, |v| (v & 0x3f) | (0b00 << 6))?; // AUTOSEL_REG=b00
+    rmw(i2c, 6, |v| (v & 0xc0) | 0b111111)?; // SFOUT2_REG=b111 SFOUT1_REG=b111
+    rmw(i2c, 0, |v| (v & 0xfd) | 0x02)?; // BYPASS_REG=1
+    Ok(())
+}
+
+pub fn setup(i2c: &mut I2c, settings: &FrequencySettings, input: Input, timer: &mut GlobalTimer) -> Result<()> {
+    let s = map_frequency_settings(settings)?;
+    let cksel_reg = match input {
+        Input::Ckin1 => 0b00,
+        Input::Ckin2 => 0b01,
+    };
+
+    init(i2c, timer)?;
+    if settings.crystal_as_ckin2 {
+        rmw(i2c, 0, |v| v | 0x40)?; // FREE_RUN=1
+    }
+    rmw(i2c, 2, |v| (v & 0x0f) | (s.bwsel << 4))?;
+    rmw(i2c, 21, |v| v & 0xfe)?; // CKSEL_PIN=0
+    rmw(i2c, 3, |v| (v & 0x2f) | (cksel_reg << 6) | 0x10)?; // CKSEL_REG, SQ_ICAL=1
+    rmw(i2c, 4, |v| (v & 0x3f) | (0b00 << 6))?; // AUTOSEL_REG=b00
+    rmw(i2c, 6, |v| (v & 0xc0) | 0b111111)?; // SFOUT2_REG=b111 SFOUT1_REG=b111
+    write(i2c, 25, (s.n1_hs << 5) as u8)?;
+    write(i2c, 31, (s.nc1_ls >> 16) as u8)?;
+    write(i2c, 32, (s.nc1_ls >> 8) as u8)?;
+    write(i2c, 33, (s.nc1_ls) as u8)?;
+    write(i2c, 34, (s.nc1_ls >> 16) as u8)?; // write to NC2_LS as well
+    write(i2c, 35, (s.nc1_ls >> 8) as u8)?;
+    write(i2c, 36, (s.nc1_ls) as u8)?;
+    write(i2c, 40, (s.n2_hs << 5) as u8 | (s.n2_ls >> 16) as u8)?;
+    write(i2c, 41, (s.n2_ls >> 8) as u8)?;
+    write(i2c, 42, (s.n2_ls) as u8)?;
+    write(i2c, 43, (s.n31 >> 16) as u8)?;
+    write(i2c, 44, (s.n31 >> 8) as u8)?;
+    write(i2c, 45, (s.n31) as u8)?;
+    write(i2c, 46, (s.n32 >> 16) as u8)?;
+    write(i2c, 47, (s.n32 >> 8) as u8)?;
+    write(i2c, 48, (s.n32) as u8)?;
+    rmw(i2c, 137, |v| v | 0x01)?; // FASTLOCK=1
+    rmw(i2c, 136, |v| v | 0x40)?; // ICAL=1
+
+    if !has_xtal(i2c)? {
+        return Err("Si5324 misses XA/XB signal");
+    }
+    if !has_ckin(i2c, input)? {
+        return Err("Si5324 misses clock input signal");
+    }
+
+    monitor_lock(i2c, timer)?;
+    Ok(())
+}
+
+pub fn select_input(i2c: &mut I2c, input: Input, timer: &mut GlobalTimer) -> Result<()> {
+    let cksel_reg = match input {
+        Input::Ckin1 => 0b00,
+        Input::Ckin2 => 0b01,
+    };
+    rmw(i2c, 3, |v| (v & 0x3f) | (cksel_reg << 6))?;
+    if !has_ckin(i2c, input)? {
+        return Err("Si5324 misses clock input signal");
+    }
+    monitor_lock(i2c, timer)?;
+    Ok(())
+}
+
+#[cfg(has_siphaser)]
+pub mod siphaser {
+    use super::*;
+    use crate::pl::csr;
+
+    pub fn select_recovered_clock(i2c: &mut I2c, rc: bool, timer: &mut GlobalTimer) -> Result<()> {
+        let val = read(i2c, 3)?;
+        write(i2c, 3, (val & 0xdf) | (1 << 5))?; // DHOLD=1
+        unsafe {
+            csr::siphaser::switch_clocks_write(if rc { 1 } else { 0 });
+        }
+        let val = read(i2c, 3)?;
+        write(i2c, 3, (val & 0xdf) | (0 << 5))?; // DHOLD=0
+        monitor_lock(i2c, timer)?;
+        Ok(())
+    }
+
+    fn phase_shift(direction: u8, timer: &mut GlobalTimer) {
+        unsafe {
+            csr::siphaser::phase_shift_write(direction);
+            while csr::siphaser::phase_shift_done_read() == 0 {}
+        }
+        // wait for the Si5324 loop to stabilize
+        timer.delay_us(500);
+    }
+
+    fn has_error(timer: &mut GlobalTimer) -> bool {
+        unsafe {
+            csr::siphaser::error_write(1);
+        }
+        timer.delay_us(5_000);
+        unsafe { csr::siphaser::error_read() != 0 }
+    }
+
+    fn find_edge(target: bool, timer: &mut GlobalTimer) -> Result<u32> {
+        let mut nshifts = 0;
+
+        let mut previous = has_error(timer);
+        loop {
+            phase_shift(1, timer);
+            nshifts += 1;
+            let current = has_error(timer);
+            if previous != target && current == target {
+                return Ok(nshifts);
+            }
+            if nshifts > 5000 {
+                return Err("failed to find timing error edge");
+            }
+            previous = current;
+        }
+    }
+
+    pub fn calibrate_skew(timer: &mut GlobalTimer) -> Result<()> {
+        let jitter_margin = 32;
+        let lead = find_edge(false, timer)?;
+        for _ in 0..jitter_margin {
+            phase_shift(1, timer);
+        }
+        let width = find_edge(true, timer)? + jitter_margin;
+        // width is 360 degrees (one full rotation of the phase between s/h limits) minus jitter
+        info!(
+            "calibration successful, lead: {}, width: {} ({}deg)",
+            lead,
+            width,
+            width * 360 / (56 * 8)
+        );
+
+        // Apply reverse phase shift for half the width to get into the
+        // middle of the working region.
+        for _ in 0..width / 2 {
+            phase_shift(0, timer);
+        }
+
+        Ok(())
+    }
+}

src/libboard_artiq/src/si549.rs

@@ -0,0 +1,854 @@
+use embedded_hal::prelude::_embedded_hal_blocking_delay_DelayUs;
+use libboard_zynq::timer::GlobalTimer;
+use log::info;
+
+use crate::pl::csr;
+
+#[cfg(feature = "target_kasli_soc")]
+const ADDRESS: u8 = 0x67;
+
+const ADPLL_MAX: i32 = (950.0 / 0.0001164) as i32;
+
+pub struct DividerConfig {
+    pub hsdiv: u16,
+    pub lsdiv: u8,
+    pub fbdiv: u64,
+}
+
+pub struct FrequencySetting {
+    pub main: DividerConfig,
+    pub helper: DividerConfig,
+}
+
+mod i2c {
+    use super::*;
+
+    #[derive(Clone, Copy)]
+    pub enum DCXO {
+        Main,
+        Helper,
+    }
+
+    fn half_period(timer: &mut GlobalTimer) {
+        timer.delay_us(1)
+    }
+
+    fn sda_i(dcxo: DCXO) -> bool {
+        match dcxo {
+            DCXO::Main => unsafe { csr::wrpll::main_dcxo_sda_in_read() == 1 },
+            DCXO::Helper => unsafe { csr::wrpll::helper_dcxo_sda_in_read() == 1 },
+        }
+    }
+
+    fn sda_oe(dcxo: DCXO, oe: bool) {
+        let val = if oe { 1 } else { 0 };
+        match dcxo {
+            DCXO::Main => unsafe { csr::wrpll::main_dcxo_sda_oe_write(val) },
+            DCXO::Helper => unsafe { csr::wrpll::helper_dcxo_sda_oe_write(val) },
+        };
+    }
+
+    fn sda_o(dcxo: DCXO, o: bool) {
+        let val = if o { 1 } else { 0 };
+        match dcxo {
+            DCXO::Main => unsafe { csr::wrpll::main_dcxo_sda_out_write(val) },
+            DCXO::Helper => unsafe { csr::wrpll::helper_dcxo_sda_out_write(val) },
+        };
+    }
+
+    fn scl_oe(dcxo: DCXO, oe: bool) {
+        let val = if oe { 1 } else { 0 };
+        match dcxo {
+            DCXO::Main => unsafe { csr::wrpll::main_dcxo_scl_oe_write(val) },
+            DCXO::Helper => unsafe { csr::wrpll::helper_dcxo_scl_oe_write(val) },
+        };
+    }
+
+    fn scl_o(dcxo: DCXO, o: bool) {
+        let val = if o { 1 } else { 0 };
+        match dcxo {
+            DCXO::Main => unsafe { csr::wrpll::main_dcxo_scl_out_write(val) },
+            DCXO::Helper => unsafe { csr::wrpll::helper_dcxo_scl_out_write(val) },
+        };
+    }
+
+    pub fn init(dcxo: DCXO, timer: &mut GlobalTimer) -> Result<(), &'static str> {
+        // Set SCL as output, and high level
+        scl_o(dcxo, true);
+        scl_oe(dcxo, true);
+        // Prepare a zero level on SDA so that sda_oe pulls it down
+        sda_o(dcxo, false);
+        // Release SDA
+        sda_oe(dcxo, false);
+
+        // Check the I2C bus is ready
+        half_period(timer);
+        half_period(timer);
+        if !sda_i(dcxo) {
+            // Try toggling SCL a few times
+            for _bit in 0..8 {
+                scl_o(dcxo, false);
+                half_period(timer);
+                scl_o(dcxo, true);
+                half_period(timer);
+            }
+        }
+
+        if !sda_i(dcxo) {
+            return Err("SDA is stuck low and doesn't get unstuck");
+        }
+        Ok(())
+    }
+
+    pub fn start(dcxo: DCXO, timer: &mut GlobalTimer) {
+        // Set SCL high then SDA low
+        scl_o(dcxo, true);
+        half_period(timer);
+        sda_oe(dcxo, true);
+        half_period(timer);
+    }
+
+    pub fn stop(dcxo: DCXO, timer: &mut GlobalTimer) {
+        // First, make sure SCL is low, so that the target releases the SDA line
+        scl_o(dcxo, false);
+        half_period(timer);
+        // Set SCL high then SDA high
+        sda_oe(dcxo, true);
+        scl_o(dcxo, true);
+        half_period(timer);
+        sda_oe(dcxo, false);
+        half_period(timer);
+    }
+
+    pub fn write(dcxo: DCXO, data: u8, timer: &mut GlobalTimer) -> bool {
+        // MSB first
+        for bit in (0..8).rev() {
+            // Set SCL low and set our bit on SDA
+            scl_o(dcxo, false);
+            sda_oe(dcxo, data & (1 << bit) == 0);
+            half_period(timer);
+            // Set SCL high ; data is shifted on the rising edge of SCL
+            scl_o(dcxo, true);
+            half_period(timer);
+        }
+        // Check ack
+        // Set SCL low, then release SDA so that the I2C target can respond
+        scl_o(dcxo, false);
+        half_period(timer);
+        sda_oe(dcxo, false);
+        // Set SCL high and check for ack
+        scl_o(dcxo, true);
+        half_period(timer);
+        // returns true if acked (I2C target pulled SDA low)
+        !sda_i(dcxo)
+    }
+
+    pub fn read(dcxo: DCXO, ack: bool, timer: &mut GlobalTimer) -> u8 {
+        // Set SCL low first, otherwise setting SDA as input may cause a transition
+        // on SDA with SCL high which will be interpreted as START/STOP condition.
+        scl_o(dcxo, false);
+        half_period(timer); // make sure SCL has settled low
+        sda_oe(dcxo, false);
+
+        let mut data: u8 = 0;
+
+        // MSB first
+        for bit in (0..8).rev() {
+            scl_o(dcxo, false);
+            half_period(timer);
+            // Set SCL high and shift data
+            scl_o(dcxo, true);
+            half_period(timer);
+            if sda_i(dcxo) {
+                data |= 1 << bit
+            }
+        }
+        // Send ack
+        // Set SCL low and pull SDA low when acking
+        scl_o(dcxo, false);
+        if ack {
+            sda_oe(dcxo, true)
+        }
+        half_period(timer);
+        // then set SCL high
+        scl_o(dcxo, true);
+        half_period(timer);
+
+        data
+    }
+}
+
+fn write(dcxo: i2c::DCXO, reg: u8, val: u8, timer: &mut GlobalTimer) -> Result<(), &'static str> {
+    i2c::start(dcxo, timer);
+    if !i2c::write(dcxo, ADDRESS << 1, timer) {
+        return Err("Si549 failed to ack write address");
+    }
+    if !i2c::write(dcxo, reg, timer) {
+        return Err("Si549 failed to ack register");
+    }
+    if !i2c::write(dcxo, val, timer) {
+        return Err("Si549 failed to ack value");
+    }
+    i2c::stop(dcxo, timer);
+    Ok(())
+}
+
+fn read(dcxo: i2c::DCXO, reg: u8, timer: &mut GlobalTimer) -> Result<u8, &'static str> {
+    i2c::start(dcxo, timer);
+    if !i2c::write(dcxo, ADDRESS << 1, timer) {
+        return Err("Si549 failed to ack write address");
+    }
+    if !i2c::write(dcxo, reg, timer) {
+        return Err("Si549 failed to ack register");
+    }
+    i2c::stop(dcxo, timer);
+
+    i2c::start(dcxo, timer);
+    if !i2c::write(dcxo, (ADDRESS << 1) | 1, timer) {
+        return Err("Si549 failed to ack read address");
+    }
+    let val = i2c::read(dcxo, false, timer);
+    i2c::stop(dcxo, timer);
+    Ok(val)
+}
+
+fn setup(dcxo: i2c::DCXO, config: &DividerConfig, timer: &mut GlobalTimer) -> Result<(), &'static str> {
+    i2c::init(dcxo, timer)?;
+
+    write(dcxo, 255, 0x00, timer)?; // PAGE
+    write(dcxo, 69, 0x00, timer)?; // Disable FCAL override.
+    write(dcxo, 17, 0x00, timer)?; // Synchronously disable output
+
+    // The Si549 has no ID register, so we check that it responds correctly
+    // by writing values to a RAM-like register and reading them back.
+    for test_value in 0..255 {
+        write(dcxo, 23, test_value, timer)?;
+        let readback = read(dcxo, 23, timer)?;
+        if readback != test_value {
+            return Err("Si549 detection failed");
+        }
+    }
+
+    write(dcxo, 23, config.hsdiv as u8, timer)?;
+    write(dcxo, 24, (config.hsdiv >> 8) as u8 | (config.lsdiv << 4), timer)?;
+    write(dcxo, 26, config.fbdiv as u8, timer)?;
+    write(dcxo, 27, (config.fbdiv >> 8) as u8, timer)?;
+    write(dcxo, 28, (config.fbdiv >> 16) as u8, timer)?;
+    write(dcxo, 29, (config.fbdiv >> 24) as u8, timer)?;
+    write(dcxo, 30, (config.fbdiv >> 32) as u8, timer)?;
+    write(dcxo, 31, (config.fbdiv >> 40) as u8, timer)?;
+
+    write(dcxo, 7, 0x08, timer)?; // Start FCAL
+    timer.delay_us(30_000); // Internal FCAL VCO calibration
+    write(dcxo, 17, 0x01, timer)?; // Synchronously enable output
+
+    Ok(())
+}
+
+pub fn main_setup(timer: &mut GlobalTimer, settings: &FrequencySetting) -> Result<(), &'static str> {
+    unsafe {
+        csr::wrpll::main_dcxo_bitbang_enable_write(1);
+        csr::wrpll::main_dcxo_i2c_address_write(ADDRESS);
+    }
+
+    setup(i2c::DCXO::Main, &settings.main, timer)?;
+
+    // Si549 maximum settling time for large frequency change.
+    timer.delay_us(40_000);
+
+    unsafe {
+        csr::wrpll::main_dcxo_bitbang_enable_write(0);
+    }
+
+    info!("Main Si549 started");
+    Ok(())
+}
+
+pub fn helper_setup(timer: &mut GlobalTimer, settings: &FrequencySetting) -> Result<(), &'static str> {
+    unsafe {
+        csr::wrpll::helper_reset_write(1);
+        csr::wrpll::helper_dcxo_bitbang_enable_write(1);
+        csr::wrpll::helper_dcxo_i2c_address_write(ADDRESS);
+    }
+
+    setup(i2c::DCXO::Helper, &settings.helper, timer)?;
+
+    // Si549 maximum settling time for large frequency change.
+    timer.delay_us(40_000);
+
+    unsafe {
+        csr::wrpll::helper_reset_write(0);
+        csr::wrpll::helper_dcxo_bitbang_enable_write(0);
+    }
+    info!("Helper Si549 started");
+    Ok(())
+}
+
+fn set_adpll(dcxo: i2c::DCXO, adpll: i32) -> Result<(), &'static str> {
+    if adpll.abs() > ADPLL_MAX {
+        return Err("adpll is too large");
+    }
+
+    match dcxo {
+        i2c::DCXO::Main => unsafe {
+            if csr::wrpll::main_dcxo_bitbang_enable_read() == 1 {
+                return Err("Main si549 bitbang mode is active when using gateware i2c");
+            }
+
+            while csr::wrpll::main_dcxo_adpll_busy_read() == 1 {}
+            if csr::wrpll::main_dcxo_nack_read() == 1 {
+                return Err("Main si549 failed to ack adpll write");
+            }
+
+            csr::wrpll::main_dcxo_i2c_address_write(ADDRESS);
+            csr::wrpll::main_dcxo_adpll_write(adpll as u32);
+
+            csr::wrpll::main_dcxo_adpll_stb_write(1);
+        },
+        i2c::DCXO::Helper => unsafe {
+            if csr::wrpll::helper_dcxo_bitbang_enable_read() == 1 {
+                return Err("Helper si549 bitbang mode is active when using gateware i2c");
+            }
+
+            while csr::wrpll::helper_dcxo_adpll_busy_read() == 1 {}
+            if csr::wrpll::helper_dcxo_nack_read() == 1 {
+                return Err("Helper si549 failed to ack adpll write");
+            }
+
+            csr::wrpll::helper_dcxo_i2c_address_write(ADDRESS);
+            csr::wrpll::helper_dcxo_adpll_write(adpll as u32);
+
+            csr::wrpll::helper_dcxo_adpll_stb_write(1);
+        },
+    };
+
+    Ok(())
+}
+
+#[cfg(has_wrpll)]
+pub mod wrpll {
+
+    use super::*;
+
+    const BEATING_PERIOD: i32 = 0x8000;
+    const BEATING_HALFPERIOD: i32 = 0x4000;
+    const COUNTER_WIDTH: u32 = 24;
+    const DIV_WIDTH: u32 = 2;
+
+    // y[n] = b0*x[n] + b1*x[n-1] + b2*x[n-2] - a1*y[n-1] - a2*y[n-2]
+    struct FilterParameters {
+        pub b0: f64,
+        pub b1: f64,
+        pub b2: f64,
+        pub a1: f64,
+        pub a2: f64,
+    }
+
+    #[cfg(rtio_frequency = "100.0")]
+    const LPF: FilterParameters = FilterParameters {
+        b0: 0.03967479060647884,
+        b1: 0.07934958121295768,
+        b2: 0.03967479060647884,
+        a1: -1.3865593741228928,
+        a2: 0.5452585365488082,
+    };
+
+    #[cfg(rtio_frequency = "125.0")]
+    const LPF: FilterParameters = FilterParameters {
+        b0: 0.07209205036273991,
+        b1: 0.14418410072547982,
+        b2: 0.07209205036273991,
+        a1: -0.6114078511562919,
+        a2: -0.10022394739274834,
+    };
+
+    static mut H_ADPLL1: i32 = 0;
+    static mut H_ADPLL2: i32 = 0;
+    static mut PERIOD_ERR1: i32 = 0;
+    static mut PERIOD_ERR2: i32 = 0;
+
+    static mut M_ADPLL1: i32 = 0;
+    static mut M_ADPLL2: i32 = 0;
+    static mut PHASE_ERR1: i32 = 0;
+    static mut PHASE_ERR2: i32 = 0;
+
+    static mut BASE_ADPLL: i32 = 0;
+
+    #[derive(Clone, Copy)]
+    pub enum ISR {
+        RefTag,
+        MainTag,
+    }
+
+    mod tag_collector {
+        use super::*;
+
+        #[cfg(wrpll_ref_clk = "GT_CDR")]
+        static mut TAG_OFFSET: u32 = 8382;
+        #[cfg(wrpll_ref_clk = "SMA_CLKIN")]
+        static mut TAG_OFFSET: u32 = 0;
+        static mut REF_TAG: u32 = 0;
+        static mut REF_TAG_READY: bool = false;
+        static mut MAIN_TAG: u32 = 0;
+        static mut MAIN_TAG_READY: bool = false;
+
+        pub fn reset() {
+            clear_phase_diff_ready();
+            unsafe {
+                REF_TAG = 0;
+                MAIN_TAG = 0;
+            }
+        }
+
+        pub fn clear_phase_diff_ready() {
+            unsafe {
+                REF_TAG_READY = false;
+                MAIN_TAG_READY = false;
+            }
+        }
+
+        pub fn collect_tags(interrupt: ISR) {
+            match interrupt {
+                ISR::RefTag => unsafe {
+                    REF_TAG = csr::wrpll::ref_tag_read();
+                    REF_TAG_READY = true;
+                },
+                ISR::MainTag => unsafe {
+                    MAIN_TAG = csr::wrpll::main_tag_read();
+                    MAIN_TAG_READY = true;
+                },
+            }
+        }
+
+        pub fn phase_diff_ready() -> bool {
+            unsafe { REF_TAG_READY && MAIN_TAG_READY }
+        }
+
+        #[cfg(feature = "calibrate_wrpll_skew")]
+        pub fn set_tag_offset(offset: u32) {
+            unsafe {
+                TAG_OFFSET = offset;
+            }
+        }
+
+        #[cfg(feature = "calibrate_wrpll_skew")]
+        pub fn get_tag_offset() -> u32 {
+            unsafe { TAG_OFFSET }
+        }
+
+        pub fn get_period_error() -> i32 {
+            // n * BEATING_PERIOD - REF_TAG(n) mod BEATING_PERIOD
+            let mut period_error = unsafe { REF_TAG.overflowing_neg().0.rem_euclid(BEATING_PERIOD as u32) as i32 };
+            // mapping tags from [0, 2π] -> [-π, π]
+            if period_error > BEATING_HALFPERIOD {
+                period_error -= BEATING_PERIOD
+            }
+            period_error
+        }
+
+        pub fn get_phase_error() -> i32 {
+            // MAIN_TAG(n) - REF_TAG(n) - TAG_OFFSET mod BEATING_PERIOD
+            let mut phase_error = unsafe {
+                MAIN_TAG
+                    .overflowing_sub(REF_TAG + TAG_OFFSET)
+                    .0
+                    .rem_euclid(BEATING_PERIOD as u32) as i32
+            };
+
+            // mapping tags from [0, 2π] -> [-π, π]
+            if phase_error > BEATING_HALFPERIOD {
+                phase_error -= BEATING_PERIOD
+            }
+            phase_error
+        }
+    }
+
+    fn set_isr(en: bool) {
+        let val = if en { 1 } else { 0 };
+        unsafe {
+            csr::wrpll::ref_tag_ev_enable_write(val);
+            csr::wrpll::main_tag_ev_enable_write(val);
+        }
+    }
+
+    fn set_base_adpll() -> Result<(), &'static str> {
+        let count2adpll =
+            |error: i32| ((error as f64 * 1e6) / (0.0001164 * (1 << (COUNTER_WIDTH - DIV_WIDTH)) as f64)) as i32;
+
+        let (ref_count, main_count) = get_freq_counts();
+        unsafe {
+            BASE_ADPLL = count2adpll(ref_count as i32 - main_count as i32);
+            set_adpll(i2c::DCXO::Main, BASE_ADPLL)?;
+            set_adpll(i2c::DCXO::Helper, BASE_ADPLL)?;
+        }
+        Ok(())
+    }
+
+    fn get_freq_counts() -> (u32, u32) {
+        unsafe {
+            csr::wrpll::frequency_counter_update_write(1);
+            while csr::wrpll::frequency_counter_busy_read() == 1 {}
+            #[cfg(wrpll_ref_clk = "GT_CDR")]
+            let ref_count = csr::wrpll::frequency_counter_counter_rtio_rx0_read();
+            #[cfg(wrpll_ref_clk = "SMA_CLKIN")]
+            let ref_count = csr::wrpll::frequency_counter_counter_ref_read();
+            let main_count = csr::wrpll::frequency_counter_counter_sys_read();
+
+            (ref_count, main_count)
+        }
+    }
+
+    fn reset_plls(timer: &mut GlobalTimer) -> Result<(), &'static str> {
+        unsafe {
+            H_ADPLL1 = 0;
+            H_ADPLL2 = 0;
+            PERIOD_ERR1 = 0;
+            PERIOD_ERR2 = 0;
+            M_ADPLL1 = 0;
+            M_ADPLL2 = 0;
+            PHASE_ERR1 = 0;
+            PHASE_ERR2 = 0;
+        }
+        set_adpll(i2c::DCXO::Main, 0)?;
+        set_adpll(i2c::DCXO::Helper, 0)?;
+        // wait for adpll to transfer and DCXO to settle
+        timer.delay_us(200);
+        Ok(())
+    }
+
+    fn clear_pending(interrupt: ISR) {
+        match interrupt {
+            ISR::RefTag => unsafe { csr::wrpll::ref_tag_ev_pending_write(1) },
+            ISR::MainTag => unsafe { csr::wrpll::main_tag_ev_pending_write(1) },
+        };
+    }
+
+    fn is_pending(interrupt: ISR) -> bool {
+        match interrupt {
+            ISR::RefTag => unsafe { csr::wrpll::ref_tag_ev_pending_read() == 1 },
+            ISR::MainTag => unsafe { csr::wrpll::main_tag_ev_pending_read() == 1 },
+        }
+    }
+
+    pub fn interrupt_handler() {
+        if is_pending(ISR::RefTag) {
+            tag_collector::collect_tags(ISR::RefTag);
+            clear_pending(ISR::RefTag);
+            helper_pll().expect("failed to run helper DCXO PLL");
+        }
+
+        if is_pending(ISR::MainTag) {
+            tag_collector::collect_tags(ISR::MainTag);
+            clear_pending(ISR::MainTag);
+        }
+
+        if tag_collector::phase_diff_ready() {
+            main_pll().expect("failed to run main DCXO PLL");
+            tag_collector::clear_phase_diff_ready();
+        }
+    }
+
+    fn helper_pll() -> Result<(), &'static str> {
+        let period_err = tag_collector::get_period_error();
+        unsafe {
+            let adpll = ((LPF.b0 * period_err as f64) + (LPF.b1 * PERIOD_ERR1 as f64) + (LPF.b2 * PERIOD_ERR2 as f64)
+                - (LPF.a1 * H_ADPLL1 as f64)
+                - (LPF.a2 * H_ADPLL2 as f64)) as i32;
+            set_adpll(i2c::DCXO::Helper, BASE_ADPLL + adpll)?;
+            H_ADPLL2 = H_ADPLL1;
+            PERIOD_ERR2 = PERIOD_ERR1;
+            H_ADPLL1 = adpll;
+            PERIOD_ERR1 = period_err;
+        };
+        Ok(())
+    }
+
+    fn main_pll() -> Result<(), &'static str> {
+        let phase_err = tag_collector::get_phase_error();
+        unsafe {
+            let adpll = ((LPF.b0 * phase_err as f64) + (LPF.b1 * PHASE_ERR1 as f64) + (LPF.b2 * PHASE_ERR2 as f64)
+                - (LPF.a1 * M_ADPLL1 as f64)
+                - (LPF.a2 * M_ADPLL2 as f64)) as i32;
+            set_adpll(i2c::DCXO::Main, BASE_ADPLL + adpll)?;
+            M_ADPLL2 = M_ADPLL1;
+            PHASE_ERR2 = PHASE_ERR1;
+            M_ADPLL1 = adpll;
+            PHASE_ERR1 = phase_err;
+        };
+        Ok(())
+    }
+
+    #[cfg(wrpll_ref_clk = "GT_CDR")]
+    fn test_skew(timer: &mut GlobalTimer) -> Result<(), &'static str> {
+        // wait for PLL to stabilize
+        timer.delay_us(20_000);
+
+        info!("testing the skew of SYS CLK...");
+        if has_timing_error(timer) {
+            return Err("the skew cannot satisfy setup/hold time constraint of RX synchronizer");
+        }
+        info!("the skew of SYS CLK met the timing constraint");
+        Ok(())
+    }
+
+    #[cfg(wrpll_ref_clk = "GT_CDR")]
+    fn has_timing_error(timer: &mut GlobalTimer) -> bool {
+        unsafe {
+            csr::wrpll_skewtester::error_write(1);
+        }
+        timer.delay_us(5_000);
+        unsafe { csr::wrpll_skewtester::error_read() == 1 }
+    }
+
+    #[cfg(feature = "calibrate_wrpll_skew")]
+    fn find_edge(target: bool, timer: &mut GlobalTimer) -> Result<u32, &'static str> {
+        const STEP: u32 = 8;
+        const STABLE_THRESHOLD: u32 = 10;
+
+        enum FSM {
+            Init,
+            WaitEdge,
+            GotEdge,
+        }
+
+        let mut state: FSM = FSM::Init;
+        let mut offset: u32 = tag_collector::get_tag_offset();
+        let mut median_edge: u32 = 0;
+        let mut stable_counter: u32 = 0;
+
+        for _ in 0..(BEATING_PERIOD as u32 / STEP) as usize {
+            tag_collector::set_tag_offset(offset);
+            offset += STEP;
+            // wait for PLL to stabilize
+            timer.delay_us(20_000);
+
+            let error = has_timing_error(timer);
+            // A median edge deglitcher
+            match state {
+                FSM::Init => {
+                    if error != target {
+                        stable_counter += 1;
+                    } else {
+                        stable_counter = 0;
+                    }
+
+                    if stable_counter >= STABLE_THRESHOLD {
+                        state = FSM::WaitEdge;
+                        stable_counter = 0;
+                    }
+                }
+                FSM::WaitEdge => {
+                    if error == target {
+                        state = FSM::GotEdge;
+                        median_edge = offset;
+                    }
+                }
+                FSM::GotEdge => {
+                    if error != target {
+                        median_edge += STEP;
+                        stable_counter = 0;
+                    } else {
+                        stable_counter += 1;
+                    }
+
+                    if stable_counter >= STABLE_THRESHOLD {
+                        return Ok(median_edge);
+                    }
+                }
+            }
+        }
+        return Err("failed to find timing error edge");
+    }
+
+    #[cfg(feature = "calibrate_wrpll_skew")]
+    fn calibrate_skew(timer: &mut GlobalTimer) -> Result<(), &'static str> {
+        info!("calibrating skew to meet timing constraint...");
+
+        // clear calibrated value
+        tag_collector::set_tag_offset(0);
+        let rising = find_edge(true, timer)? as i32;
+        let falling = find_edge(false, timer)? as i32;
+
+        let width = BEATING_PERIOD - (falling - rising);
+        let result = falling + width / 2;
+        tag_collector::set_tag_offset(result as u32);
+
+        info!(
+            "calibration successful, error zone: {} -> {}, width: {} ({}deg), middle of working region: {}",
+            rising,
+            falling,
+            width,
+            360 * width / BEATING_PERIOD,
+            result,
+        );
+
+        Ok(())
+    }
+
+    pub fn select_recovered_clock(rc: bool, timer: &mut GlobalTimer) {
+        set_isr(false);
+
+        if rc {
+            tag_collector::reset();
+            reset_plls(timer).expect("failed to reset main and helper PLL");
+
+            // get within capture range
+            set_base_adpll().expect("failed to set base adpll");
+
+            // clear gateware pending flag
+            clear_pending(ISR::RefTag);
+            clear_pending(ISR::MainTag);
+
+            // use nFIQ to avoid IRQ being disabled by mutex lock and mess up PLL
+            set_isr(true);
+            info!("WRPLL interrupt enabled");
+
+            #[cfg(feature = "calibrate_wrpll_skew")]
+            calibrate_skew(timer).expect("failed to set the correct skew");
+
+            #[cfg(wrpll_ref_clk = "GT_CDR")]
+            test_skew(timer).expect("skew test failed");
+        }
+    }
+}
+
+#[cfg(has_wrpll_refclk)]
+pub mod wrpll_refclk {
+    use super::*;
+
+    pub struct MmcmSetting {
+        pub clkout0_reg1: u16,  //0x08
+        pub clkout0_reg2: u16,  //0x09
+        pub clkfbout_reg1: u16, //0x14
+        pub clkfbout_reg2: u16, //0x15
+        pub div_reg: u16,       //0x16
+        pub lock_reg1: u16,     //0x18
+        pub lock_reg2: u16,     //0x19
+        pub lock_reg3: u16,     //0x1A
+        pub power_reg: u16,     //0x28
+        pub filt_reg1: u16,     //0x4E
+        pub filt_reg2: u16,     //0x4F
+    }
+
+    fn one_clock_cycle() {
+        unsafe {
+            csr::wrpll_refclk::mmcm_dclk_write(1);
+            csr::wrpll_refclk::mmcm_dclk_write(0);
+        }
+    }
+
+    fn set_addr(address: u8) {
+        unsafe {
+            csr::wrpll_refclk::mmcm_daddr_write(address);
+        }
+    }
+
+    fn set_data(value: u16) {
+        unsafe {
+            csr::wrpll_refclk::mmcm_din_write(value);
+        }
+    }
+
+    fn set_enable(en: bool) {
+        unsafe {
+            let val = if en { 1 } else { 0 };
+            csr::wrpll_refclk::mmcm_den_write(val);
+        }
+    }
+
+    fn set_write_enable(en: bool) {
+        unsafe {
+            let val = if en { 1 } else { 0 };
+            csr::wrpll_refclk::mmcm_dwen_write(val);
+        }
+    }
+
+    fn get_data() -> u16 {
+        unsafe { csr::wrpll_refclk::mmcm_dout_read() }
+    }
+
+    fn drp_ready() -> bool {
+        unsafe { csr::wrpll_refclk::mmcm_dready_read() == 1 }
+    }
+
+    #[allow(dead_code)]
+    fn read(address: u8) -> u16 {
+        set_addr(address);
+        set_enable(true);
+        // Set DADDR on the mmcm and assert DEN for one clock cycle
+        one_clock_cycle();
+
+        set_enable(false);
+        while !drp_ready() {
+            // keep the clock signal until data is ready
+            one_clock_cycle();
+        }
+        get_data()
+    }
+
+    fn write(address: u8, value: u16) {
+        set_addr(address);
+        set_data(value);
+        set_write_enable(true);
+        set_enable(true);
+        // Set DADDR, DI on the mmcm and assert DWE, DEN for one clock cycle
+        one_clock_cycle();
+
+        set_write_enable(false);
+        set_enable(false);
+        while !drp_ready() {
+            // keep the clock signal until write is finished
+            one_clock_cycle();
+        }
+    }
+
+    fn reset(rst: bool) {
+        unsafe {
+            let val = if rst { 1 } else { 0 };
+            csr::wrpll_refclk::mmcm_reset_write(val)
+        }
+    }
+
+    pub fn setup(timer: &mut GlobalTimer, settings: MmcmSetting, mmcm_bypass: bool) -> Result<(), &'static str> {
+        unsafe {
+            csr::wrpll_refclk::refclk_reset_write(1);
+        }
+
+        if mmcm_bypass {
+            info!("Bypassing mmcm");
+            unsafe {
+                csr::wrpll_refclk::mmcm_bypass_write(1);
+            }
+        } else {
+            // Based on "DRP State Machine" from XAPP888
+            // hold reset HIGH during mmcm config
+            reset(true);
+            write(0x08, settings.clkout0_reg1);
+            write(0x09, settings.clkout0_reg2);
+            write(0x14, settings.clkfbout_reg1);
+            write(0x15, settings.clkfbout_reg2);
+            write(0x16, settings.div_reg);
+            write(0x18, settings.lock_reg1);
+            write(0x19, settings.lock_reg2);
+            write(0x1A, settings.lock_reg3);
+            write(0x28, settings.power_reg);
+            write(0x4E, settings.filt_reg1);
+            write(0x4F, settings.filt_reg2);
+            reset(false);
+
+            // wait for the mmcm to lock
+            timer.delay_us(100);
+
+            let locked = unsafe { csr::wrpll_refclk::mmcm_locked_read() == 1 };
+            if !locked {
+                return Err("mmcm failed to generate 125MHz ref clock from SMA CLKIN");
+            }
+        }
+
+        unsafe {
+            csr::wrpll_refclk::refclk_reset_write(0);
+        }
+
+        Ok(())
+    }
+}

src/libbuild_zynq/Cargo.toml

@@ -0,0 +1,8 @@
+[package]
+authors = ["M-Labs"]
+name = "build_zynq"
+version = "0.0.0"
+
+[lib]
+name = "build_zynq"
+path = "lib.rs"

src/libbuild_zynq/lib.rs

@@ -0,0 +1,29 @@
+use std::{env,
+          fs::File,
+          io::{BufRead, BufReader, Write},
+          path::PathBuf};
+
+pub fn add_linker_script() {
+    // Put the linker script somewhere the linker can find it
+    let out = &PathBuf::from(env::var_os("OUT_DIR").unwrap());
+    File::create(out.join("link.x"))
+        .unwrap()
+        .write_all(include_bytes!("link.x"))
+        .unwrap();
+    println!("cargo:rustc-link-search={}", out.display());
+
+    // Only re-run the build script when link.x is changed,
+    // instead of when any part of the source code changes.
+    println!("cargo:rerun-if-changed=link.x");
+}
+
+pub fn cfg() {
+    // Handle rustc-cfg file
+    let cfg_path = "../../build/rustc-cfg";
+    println!("cargo:rerun-if-changed={}", cfg_path);
+
+    let f = BufReader::new(File::open(cfg_path).unwrap());
+    for line in f.lines() {
+        println!("cargo:rustc-cfg={}", line.unwrap());
+    }
+}

src/libbuild_zynq/link.x

@@ -10,7 +10,9 @@ SECTIONS
     __text_start = .;
     .text :
     {
+        __exceptions_start = .;
         KEEP(*(.text.exceptions));
+        __exceptions_end = .;
         *(.text.boot);
         *(.text .text.*);
     } > SDRAM
@@ -48,9 +50,12 @@ SECTIONS
 
     .heap (NOLOAD) : ALIGN(8)
     {
-        __heap_start = .;
-        . += 0x1000000;
-        __heap_end = .;
+        __heap0_start = .;
+        . += 0x8000000;
+        __heap0_end = .;
+        __heap1_start = .;
+        . += 0x8000000;
+        __heap1_end = .;
     } > SDRAM
 
     .stack1 (NOLOAD) : ALIGN(8)
@@ -66,4 +71,18 @@ SECTIONS
         . += 0x20000;
         __stack0_start = .;
     } > SDRAM
+
+    .irq_stack1 (NOLOAD) : ALIGN(8)
+    {
+        __irq_stack1_end = .;
+        . += 0x100;
+        __irq_stack1_start = .;
+    } > SDRAM
+
+    .irq_stack0 (NOLOAD) : ALIGN(8)
+    {
+        __irq_stack0_end = .;
+        . += 0x100;
+        __irq_stack0_start = .;
+    } > SDRAM
 }

src/libc/Cargo.toml

@@ -1,14 +0,0 @@
-[package]
-name = "libc"
-version = "0.1.0"
-authors = ["pca006132 <john.lck40@gmail.com>"]
-edition = "2018"
-build = "build.rs"
-
-# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
-
-[dependencies]
-libboard_zynq = { git = "https://git.m-labs.hk/M-Labs/zc706.git" }
-
-[build-dependencies]
-cc = { version = "1.0.1" }

src/libc/Cargo.toml.tpl

@@ -0,0 +1,12 @@
+[package]
+name = "libc"
+version = "0.1.0"
+authors = ["M-Labs"]
+edition = "2018"
+build = "build.rs"
+
+[dependencies]
+libboard_zynq = { path = "@@ZYNQ_RS@@/libboard_zynq" }
+
+[build-dependencies]
+cc = { version = "1.0.1" }

src/libc/build.rs

@@ -4,7 +4,8 @@ fn main() {
 }
 
 mod libc {
-    use std::path::Path;
+    use std::{env, path::Path};
+
     pub fn compile() {
         let cfg = &mut cc::Build::new();
         cfg.no_default_flags(true);
@@ -16,6 +17,9 @@ mod libc {
         cfg.flag("-ffreestanding");
         cfg.flag("-fno-PIC");
         cfg.flag("-isystem../include");
+        if let Ok(extra_include) = env::var("CLANG_EXTRA_INCLUDE_DIR") {
+            cfg.flag(&("-isystem".to_owned() + &extra_include));
+        }
         cfg.flag("-fno-stack-protector");
         cfg.flag("--target=armv7-none-eabihf");
         cfg.flag("-O2");
@@ -27,9 +31,7 @@ mod libc {
         cfg.flag("-U_FORTIFY_SOURCE");
         cfg.define("_FORTIFY_SOURCE", Some("0"));
 
-        let sources = vec![
-            "printf.c"
-        ];
+        let sources = vec!["printf.c"];
 
         let root = Path::new("./");
         for src in sources {

src/libc/src/lib.rs

@@ -4,7 +4,9 @@
 
 use libboard_zynq::stdio;
 
+pub type c_char = i8;
 pub type c_int = i32;
+pub type size_t = usize;
 pub type uintptr_t = usize;
 pub type c_void = core::ffi::c_void;
 

src/libcoreio/Cargo.toml

@@ -1,14 +0,0 @@
-[package]
-authors = ["M-Labs"]
-name = "core_io"
-version = "0.1.20200410"
-
-[lib]
-name = "core_io"
-
-[dependencies]
-memchr = { version = "2", default-features = false, optional = true }
-
-[features]
-alloc = []
-collections = ["alloc", "memchr"]

src/libcoreio/src/io/cursor.rs

@@ -1,896 +0,0 @@
-use crate::io::prelude::*;
-
-use core::cmp;
-use crate::io::{self, Error, ErrorKind, Initializer, SeekFrom};
-
-#[cfg(feature = "collections")]
-use core::convert::TryInto;
-
-#[cfg(feature="collections")] 
-use collections::vec::Vec;
-
-#[cfg(feature = "alloc")]
-use alloc::boxed::Box;
-
-/// A `Cursor` wraps an in-memory buffer and provides it with a
-/// [`Seek`] implementation.
-///
-/// `Cursor`s are used with in-memory buffers, anything implementing
-/// `AsRef<[u8]>`, to allow them to implement [`Read`] and/or [`Write`],
-/// allowing these buffers to be used anywhere you might use a reader or writer
-/// that does actual I/O.
-///
-/// The standard library implements some I/O traits on various types which
-/// are commonly used as a buffer, like `Cursor<`[`Vec`]`<u8>>` and
-/// `Cursor<`[`&[u8]`][bytes]`>`.
-///
-/// # Examples
-///
-/// We may want to write bytes to a [`File`] in our production
-/// code, but use an in-memory buffer in our tests. We can do this with
-/// `Cursor`:
-///
-/// [`Seek`]: trait.Seek.html
-/// [`Read`]: ../../std/io/trait.Read.html
-/// [`Write`]: ../../std/io/trait.Write.html
-/// [`Vec`]: ../../std/vec/struct.Vec.html
-/// [bytes]: ../../std/primitive.slice.html
-/// [`File`]: ../fs/struct.File.html
-///
-/// ```no_run
-/// use std::io::prelude::*;
-/// use std::io::{self, SeekFrom};
-/// use std::fs::File;
-///
-/// // a library function we've written
-/// fn write_ten_bytes_at_end<W: Write + Seek>(writer: &mut W) -> io::Result<()> {
-///     writer.seek(SeekFrom::End(-10))?;
-///
-///     for i in 0..10 {
-///         writer.write(&[i])?;
-///     }
-///
-///     // all went well
-///     Ok(())
-/// }
-///
-/// # fn foo() -> io::Result<()> {
-/// // Here's some code that uses this library function.
-/// //
-/// // We might want to use a BufReader here for efficiency, but let's
-/// // keep this example focused.
-/// let mut file = File::create("foo.txt")?;
-///
-/// write_ten_bytes_at_end(&mut file)?;
-/// # Ok(())
-/// # }
-///
-/// // now let's write a test
-/// #[test]
-/// fn test_writes_bytes() {
-///     // setting up a real File is much slower than an in-memory buffer,
-///     // let's use a cursor instead
-///     use std::io::Cursor;
-///     let mut buff = Cursor::new(vec![0; 15]);
-///
-///     write_ten_bytes_at_end(&mut buff).unwrap();
-///
-///     assert_eq!(&buff.get_ref()[5..15], &[0, 1, 2, 3, 4, 5, 6, 7, 8, 9]);
-/// }
-/// ```
-#[derive(Clone, Debug, Default, Eq, PartialEq)]
-pub struct Cursor<T> {
-    inner: T,
-    pos: u64,
-}
-
-impl<T> Cursor<T> {
-    /// Creates a new cursor wrapping the provided underlying in-memory buffer.
-    ///
-    /// Cursor initial position is `0` even if underlying buffer (e.g., `Vec`)
-    /// is not empty. So writing to cursor starts with overwriting `Vec`
-    /// content, not with appending to it.
-    ///
-    /// # Examples
-    ///
-    /// ```
-    /// use std::io::Cursor;
-    ///
-    /// let buff = Cursor::new(Vec::new());
-    /// # fn force_inference(_: &Cursor<Vec<u8>>) {}
-    /// # force_inference(&buff);
-    /// ```
-    pub fn new(inner: T) -> Cursor<T> {
-        Cursor { pos: 0, inner }
-    }
-
-    /// Consumes this cursor, returning the underlying value.
-    ///
-    /// # Examples
-    ///
-    /// ```
-    /// use std::io::Cursor;
-    ///
-    /// let buff = Cursor::new(Vec::new());
-    /// # fn force_inference(_: &Cursor<Vec<u8>>) {}
-    /// # force_inference(&buff);
-    ///
-    /// let vec = buff.into_inner();
-    /// ```
-    pub fn into_inner(self) -> T {
-        self.inner
-    }
-
-    /// Gets a reference to the underlying value in this cursor.
-    ///
-    /// # Examples
-    ///
-    /// ```
-    /// use std::io::Cursor;
-    ///
-    /// let buff = Cursor::new(Vec::new());
-    /// # fn force_inference(_: &Cursor<Vec<u8>>) {}
-    /// # force_inference(&buff);
-    ///
-    /// let reference = buff.get_ref();
-    /// ```
-    pub fn get_ref(&self) -> &T {
-        &self.inner
-    }
-
-    /// Gets a mutable reference to the underlying value in this cursor.
-    ///
-    /// Care should be taken to avoid modifying the internal I/O state of the
-    /// underlying value as it may corrupt this cursor's position.
-    ///
-    /// # Examples
-    ///
-    /// ```
-    /// use std::io::Cursor;
-    ///
-    /// let mut buff = Cursor::new(Vec::new());
-    /// # fn force_inference(_: &Cursor<Vec<u8>>) {}
-    /// # force_inference(&buff);
-    ///
-    /// let reference = buff.get_mut();
-    /// ```
-    pub fn get_mut(&mut self) -> &mut T {
-        &mut self.inner
-    }
-
-    /// Returns the current position of this cursor.
-    ///
-    /// # Examples
-    ///
-    /// ```
-    /// use std::io::Cursor;
-    /// use std::io::prelude::*;
-    /// use std::io::SeekFrom;
-    ///
-    /// let mut buff = Cursor::new(vec![1, 2, 3, 4, 5]);
-    ///
-    /// assert_eq!(buff.position(), 0);
-    ///
-    /// buff.seek(SeekFrom::Current(2)).unwrap();
-    /// assert_eq!(buff.position(), 2);
-    ///
-    /// buff.seek(SeekFrom::Current(-1)).unwrap();
-    /// assert_eq!(buff.position(), 1);
-    /// ```
-    pub fn position(&self) -> u64 {
-        self.pos
-    }
-
-    /// Sets the position of this cursor.
-    ///
-    /// # Examples
-    ///
-    /// ```
-    /// use std::io::Cursor;
-    ///
-    /// let mut buff = Cursor::new(vec![1, 2, 3, 4, 5]);
-    ///
-    /// assert_eq!(buff.position(), 0);
-    ///
-    /// buff.set_position(2);
-    /// assert_eq!(buff.position(), 2);
-    ///
-    /// buff.set_position(4);
-    /// assert_eq!(buff.position(), 4);
-    /// ```
-    pub fn set_position(&mut self, pos: u64) {
-        self.pos = pos;
-    }
-}
-
-impl<T> io::Seek for Cursor<T>
-where
-    T: AsRef<[u8]>,
-{
-    fn seek(&mut self, style: SeekFrom) -> io::Result<u64> {
-        let (base_pos, offset) = match style {
-            SeekFrom::Start(n) => {
-                self.pos = n;
-                return Ok(n);
-            }
-            SeekFrom::End(n) => (self.inner.as_ref().len() as u64, n),
-            SeekFrom::Current(n) => (self.pos, n),
-        };
-        let new_pos = if offset >= 0 {
-            base_pos.checked_add(offset as u64)
-        } else {
-            base_pos.checked_sub((offset.wrapping_neg()) as u64)
-        };
-        match new_pos {
-            Some(n) => {
-                self.pos = n;
-                Ok(self.pos)
-            }
-            None => Err(Error::new(
-                ErrorKind::InvalidInput,
-                "invalid seek to a negative or overflowing position",
-            )),
-        }
-    }
-
-    fn stream_len(&mut self) -> io::Result<u64> {
-        Ok(self.inner.as_ref().len() as u64)
-    }
-
-    fn stream_position(&mut self) -> io::Result<u64> {
-        Ok(self.pos)
-    }
-}
-
-impl<T> Read for Cursor<T>
-where
-    T: AsRef<[u8]>,
-{
-    fn read(&mut self, buf: &mut [u8]) -> io::Result<usize> {
-        let n = Read::read(&mut self.get_ref().as_ref(), buf)?;
-        self.pos += n as u64;
-        Ok(n)
-    }
-
-    fn read_exact(&mut self, buf: &mut [u8]) -> io::Result<()> {
-        let n = buf.len();
-        Read::read_exact(&mut self.get_ref().as_ref(), buf)?;
-        self.pos += n as u64;
-        Ok(())
-    }
-
-    #[inline]
-    unsafe fn initializer(&self) -> Initializer {
-        Initializer::nop()
-    }
-}
-
-#[cfg(feature = "collections")]
-impl<T> BufRead for Cursor<T>
-where
-    T: AsRef<[u8]>,
-{
-    fn fill_buf(&mut self) -> io::Result<&[u8]> {
-        let amt = cmp::min(self.pos, self.inner.as_ref().len() as u64);
-        Ok(&self.inner.as_ref()[(amt as usize)..])
-    }
-    fn consume(&mut self, amt: usize) {
-        self.pos += amt as u64;
-    }
-}
-
-// Non-resizing write implementation
-#[inline]
-fn slice_write(pos_mut: &mut u64, slice: &mut [u8], buf: &[u8]) -> io::Result<usize> {
-    let pos = cmp::min(*pos_mut, slice.len() as u64);
-    let amt = (&mut slice[(pos as usize)..]).write(buf)?;
-    *pos_mut += amt as u64;
-    Ok(amt)
-}
-
-// Resizing write implementation
-#[cfg(feature = "collections")]
-fn vec_write(pos_mut: &mut u64, vec: &mut Vec<u8>, buf: &[u8]) -> io::Result<usize> {
-    let pos: usize = (*pos_mut).try_into().map_err(|_| {
-        Error::new(
-            ErrorKind::InvalidInput,
-            "cursor position exceeds maximum possible vector length",
-        )
-    })?;
-    // Make sure the internal buffer is as least as big as where we
-    // currently are
-    let len = vec.len();
-    if len < pos {
-        // use `resize` so that the zero filling is as efficient as possible
-        vec.resize(pos, 0);
-    }
-    // Figure out what bytes will be used to overwrite what's currently
-    // there (left), and what will be appended on the end (right)
-    {
-        let space = vec.len() - pos;
-        let (left, right) = buf.split_at(cmp::min(space, buf.len()));
-        vec[pos..pos + left.len()].copy_from_slice(left);
-        vec.extend_from_slice(right);
-    }
-
-    // Bump us forward
-    *pos_mut = (pos + buf.len()) as u64;
-    Ok(buf.len())
-}
-
-impl Write for Cursor<&mut [u8]> {
-    #[inline]
-    fn write(&mut self, buf: &[u8]) -> io::Result<usize> {
-        slice_write(&mut self.pos, self.inner, buf)
-    }
-
-    #[inline]
-    fn flush(&mut self) -> io::Result<()> {
-        Ok(())
-    }
-}
-
-#[cfg(feature = "collections")]
-impl Write for Cursor<&mut Vec<u8>> {
-    fn write(&mut self, buf: &[u8]) -> io::Result<usize> {
-        vec_write(&mut self.pos, self.inner, buf)
-    }
-
-    #[inline]
-    fn flush(&mut self) -> io::Result<()> {
-        Ok(())
-    }
-}
-
-#[cfg(feature = "collections")]
-impl Write for Cursor<Vec<u8>> {
-    fn write(&mut self, buf: &[u8]) -> io::Result<usize> {
-        vec_write(&mut self.pos, &mut self.inner, buf)
-    }
-
-    #[inline]
-    fn flush(&mut self) -> io::Result<()> {
-        Ok(())
-    }
-}
-
-#[cfg(feature = "alloc")]
-impl Write for Cursor<Box<[u8]>> {
-    #[inline]
-    fn write(&mut self, buf: &[u8]) -> io::Result<usize> {
-        slice_write(&mut self.pos, &mut self.inner, buf)
-    }
-
-    #[inline]
-    fn flush(&mut self) -> io::Result<()> {
-        Ok(())
-    }
-}
-
-#[cfg(test)]
-mod tests {
-    use crate::io::prelude::*;
-    use crate::io::{Cursor, IoSlice, IoSliceMut, SeekFrom};
-
-    #[test]
-    fn test_vec_writer() {
-        let mut writer = Vec::new();
-        assert_eq!(writer.write(&[0]).unwrap(), 1);
-        assert_eq!(writer.write(&[1, 2, 3]).unwrap(), 3);
-        assert_eq!(writer.write(&[4, 5, 6, 7]).unwrap(), 4);
-        assert_eq!(
-            writer
-                .write_vectored(&[IoSlice::new(&[]), IoSlice::new(&[8, 9]), IoSlice::new(&[10])],)
-                .unwrap(),
-            3
-        );
-        let b: &[_] = &[0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10];
-        assert_eq!(writer, b);
-    }
-
-    #[test]
-    fn test_mem_writer() {
-        let mut writer = Cursor::new(Vec::new());
-        assert_eq!(writer.write(&[0]).unwrap(), 1);
-        assert_eq!(writer.write(&[1, 2, 3]).unwrap(), 3);
-        assert_eq!(writer.write(&[4, 5, 6, 7]).unwrap(), 4);
-        assert_eq!(
-            writer
-                .write_vectored(&[IoSlice::new(&[]), IoSlice::new(&[8, 9]), IoSlice::new(&[10])],)
-                .unwrap(),
-            3
-        );
-        let b: &[_] = &[0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10];
-        assert_eq!(&writer.get_ref()[..], b);
-    }
-
-    #[test]
-    fn test_mem_mut_writer() {
-        let mut vec = Vec::new();
-        let mut writer = Cursor::new(&mut vec);
-        assert_eq!(writer.write(&[0]).unwrap(), 1);
-        assert_eq!(writer.write(&[1, 2, 3]).unwrap(), 3);
-        assert_eq!(writer.write(&[4, 5, 6, 7]).unwrap(), 4);
-        assert_eq!(
-            writer
-                .write_vectored(&[IoSlice::new(&[]), IoSlice::new(&[8, 9]), IoSlice::new(&[10])],)
-                .unwrap(),
-            3
-        );
-        let b: &[_] = &[0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10];
-        assert_eq!(&writer.get_ref()[..], b);
-    }
-
-    #[test]
-    fn test_box_slice_writer() {
-        let mut writer = Cursor::new(vec![0u8; 9].into_boxed_slice());
-        assert_eq!(writer.position(), 0);
-        assert_eq!(writer.write(&[0]).unwrap(), 1);
-        assert_eq!(writer.position(), 1);
-        assert_eq!(writer.write(&[1, 2, 3]).unwrap(), 3);
-        assert_eq!(writer.write(&[4, 5, 6, 7]).unwrap(), 4);
-        assert_eq!(writer.position(), 8);
-        assert_eq!(writer.write(&[]).unwrap(), 0);
-        assert_eq!(writer.position(), 8);
-
-        assert_eq!(writer.write(&[8, 9]).unwrap(), 1);
-        assert_eq!(writer.write(&[10]).unwrap(), 0);
-        let b: &[_] = &[0, 1, 2, 3, 4, 5, 6, 7, 8];
-        assert_eq!(&**writer.get_ref(), b);
-    }
-
-    #[test]
-    fn test_box_slice_writer_vectored() {
-        let mut writer = Cursor::new(vec![0u8; 9].into_boxed_slice());
-        assert_eq!(writer.position(), 0);
-        assert_eq!(writer.write_vectored(&[IoSlice::new(&[0])]).unwrap(), 1);
-        assert_eq!(writer.position(), 1);
-        assert_eq!(
-            writer
-                .write_vectored(&[IoSlice::new(&[1, 2, 3]), IoSlice::new(&[4, 5, 6, 7]),])
-                .unwrap(),
-            7,
-        );
-        assert_eq!(writer.position(), 8);
-        assert_eq!(writer.write_vectored(&[]).unwrap(), 0);
-        assert_eq!(writer.position(), 8);
-
-        assert_eq!(writer.write_vectored(&[IoSlice::new(&[8, 9])]).unwrap(), 1);
-        assert_eq!(writer.write_vectored(&[IoSlice::new(&[10])]).unwrap(), 0);
-        let b: &[_] = &[0, 1, 2, 3, 4, 5, 6, 7, 8];
-        assert_eq!(&**writer.get_ref(), b);
-    }
-
-    #[test]
-    fn test_buf_writer() {
-        let mut buf = [0 as u8; 9];
-        {
-            let mut writer = Cursor::new(&mut buf[..]);
-            assert_eq!(writer.position(), 0);
-            assert_eq!(writer.write(&[0]).unwrap(), 1);
-            assert_eq!(writer.position(), 1);
-            assert_eq!(writer.write(&[1, 2, 3]).unwrap(), 3);
-            assert_eq!(writer.write(&[4, 5, 6, 7]).unwrap(), 4);
-            assert_eq!(writer.position(), 8);
-            assert_eq!(writer.write(&[]).unwrap(), 0);
-            assert_eq!(writer.position(), 8);
-
-            assert_eq!(writer.write(&[8, 9]).unwrap(), 1);
-            assert_eq!(writer.write(&[10]).unwrap(), 0);
-        }
-        let b: &[_] = &[0, 1, 2, 3, 4, 5, 6, 7, 8];
-        assert_eq!(buf, b);
-    }
-
-    #[test]
-    fn test_buf_writer_vectored() {
-        let mut buf = [0 as u8; 9];
-        {
-            let mut writer = Cursor::new(&mut buf[..]);
-            assert_eq!(writer.position(), 0);
-            assert_eq!(writer.write_vectored(&[IoSlice::new(&[0])]).unwrap(), 1);
-            assert_eq!(writer.position(), 1);
-            assert_eq!(
-                writer
-                    .write_vectored(&[IoSlice::new(&[1, 2, 3]), IoSlice::new(&[4, 5, 6, 7])],)
-                    .unwrap(),
-                7,
-            );
-            assert_eq!(writer.position(), 8);
-            assert_eq!(writer.write_vectored(&[]).unwrap(), 0);
-            assert_eq!(writer.position(), 8);
-
-            assert_eq!(writer.write_vectored(&[IoSlice::new(&[8, 9])]).unwrap(), 1);
-            assert_eq!(writer.write_vectored(&[IoSlice::new(&[10])]).unwrap(), 0);
-        }
-        let b: &[_] = &[0, 1, 2, 3, 4, 5, 6, 7, 8];
-        assert_eq!(buf, b);
-    }
-
-    #[test]
-    fn test_buf_writer_seek() {
-        let mut buf = [0 as u8; 8];
-        {
-            let mut writer = Cursor::new(&mut buf[..]);
-            assert_eq!(writer.position(), 0);
-            assert_eq!(writer.write(&[1]).unwrap(), 1);
-            assert_eq!(writer.position(), 1);
-
-            assert_eq!(writer.seek(SeekFrom::Start(2)).unwrap(), 2);
-            assert_eq!(writer.position(), 2);
-            assert_eq!(writer.write(&[2]).unwrap(), 1);
-            assert_eq!(writer.position(), 3);
-
-            assert_eq!(writer.seek(SeekFrom::Current(-2)).unwrap(), 1);
-            assert_eq!(writer.position(), 1);
-            assert_eq!(writer.write(&[3]).unwrap(), 1);
-            assert_eq!(writer.position(), 2);
-
-            assert_eq!(writer.seek(SeekFrom::End(-1)).unwrap(), 7);
-            assert_eq!(writer.position(), 7);
-            assert_eq!(writer.write(&[4]).unwrap(), 1);
-            assert_eq!(writer.position(), 8);
-        }
-        let b: &[_] = &[1, 3, 2, 0, 0, 0, 0, 4];
-        assert_eq!(buf, b);
-    }
-
-    #[test]
-    fn test_buf_writer_error() {
-        let mut buf = [0 as u8; 2];
-        let mut writer = Cursor::new(&mut buf[..]);
-        assert_eq!(writer.write(&[0]).unwrap(), 1);
-        assert_eq!(writer.write(&[0, 0]).unwrap(), 1);
-        assert_eq!(writer.write(&[0, 0]).unwrap(), 0);
-    }
-
-    #[test]
-    fn test_mem_reader() {
-        let mut reader = Cursor::new(vec![0, 1, 2, 3, 4, 5, 6, 7]);
-        let mut buf = [];
-        assert_eq!(reader.read(&mut buf).unwrap(), 0);
-        assert_eq!(reader.position(), 0);
-        let mut buf = [0];
-        assert_eq!(reader.read(&mut buf).unwrap(), 1);
-        assert_eq!(reader.position(), 1);
-        let b: &[_] = &[0];
-        assert_eq!(buf, b);
-        let mut buf = [0; 4];
-        assert_eq!(reader.read(&mut buf).unwrap(), 4);
-        assert_eq!(reader.position(), 5);
-        let b: &[_] = &[1, 2, 3, 4];
-        assert_eq!(buf, b);
-        assert_eq!(reader.read(&mut buf).unwrap(), 3);
-        let b: &[_] = &[5, 6, 7];
-        assert_eq!(&buf[..3], b);
-        assert_eq!(reader.read(&mut buf).unwrap(), 0);
-    }
-
-    #[test]
-    fn test_mem_reader_vectored() {
-        let mut reader = Cursor::new(vec![0, 1, 2, 3, 4, 5, 6, 7]);
-        let mut buf = [];
-        assert_eq!(reader.read_vectored(&mut [IoSliceMut::new(&mut buf)]).unwrap(), 0);
-        assert_eq!(reader.position(), 0);
-        let mut buf = [0];
-        assert_eq!(
-            reader
-                .read_vectored(&mut [IoSliceMut::new(&mut []), IoSliceMut::new(&mut buf),])
-                .unwrap(),
-            1,
-        );
-        assert_eq!(reader.position(), 1);
-        let b: &[_] = &[0];
-        assert_eq!(buf, b);
-        let mut buf1 = [0; 4];
-        let mut buf2 = [0; 4];
-        assert_eq!(
-            reader
-                .read_vectored(&mut [IoSliceMut::new(&mut buf1), IoSliceMut::new(&mut buf2),])
-                .unwrap(),
-            7,
-        );
-        let b1: &[_] = &[1, 2, 3, 4];
-        let b2: &[_] = &[5, 6, 7];
-        assert_eq!(buf1, b1);
-        assert_eq!(&buf2[..3], b2);
-        assert_eq!(reader.read(&mut buf).unwrap(), 0);
-    }
-
-    #[test]
-    fn test_boxed_slice_reader() {
-        let mut reader = Cursor::new(vec![0, 1, 2, 3, 4, 5, 6, 7].into_boxed_slice());
-        let mut buf = [];
-        assert_eq!(reader.read(&mut buf).unwrap(), 0);
-        assert_eq!(reader.position(), 0);
-        let mut buf = [0];
-        assert_eq!(reader.read(&mut buf).unwrap(), 1);
-        assert_eq!(reader.position(), 1);
-        let b: &[_] = &[0];
-        assert_eq!(buf, b);
-        let mut buf = [0; 4];
-        assert_eq!(reader.read(&mut buf).unwrap(), 4);
-        assert_eq!(reader.position(), 5);
-        let b: &[_] = &[1, 2, 3, 4];
-        assert_eq!(buf, b);
-        assert_eq!(reader.read(&mut buf).unwrap(), 3);
-        let b: &[_] = &[5, 6, 7];
-        assert_eq!(&buf[..3], b);
-        assert_eq!(reader.read(&mut buf).unwrap(), 0);
-    }
-
-    #[test]
-    fn test_boxed_slice_reader_vectored() {
-        let mut reader = Cursor::new(vec![0, 1, 2, 3, 4, 5, 6, 7].into_boxed_slice());
-        let mut buf = [];
-        assert_eq!(reader.read_vectored(&mut [IoSliceMut::new(&mut buf)]).unwrap(), 0);
-        assert_eq!(reader.position(), 0);
-        let mut buf = [0];
-        assert_eq!(
-            reader
-                .read_vectored(&mut [IoSliceMut::new(&mut []), IoSliceMut::new(&mut buf),])
-                .unwrap(),
-            1,
-        );
-        assert_eq!(reader.position(), 1);
-        let b: &[_] = &[0];
-        assert_eq!(buf, b);
-        let mut buf1 = [0; 4];
-        let mut buf2 = [0; 4];
-        assert_eq!(
-            reader
-                .read_vectored(&mut [IoSliceMut::new(&mut buf1), IoSliceMut::new(&mut buf2)],)
-                .unwrap(),
-            7,
-        );
-        let b1: &[_] = &[1, 2, 3, 4];
-        let b2: &[_] = &[5, 6, 7];
-        assert_eq!(buf1, b1);
-        assert_eq!(&buf2[..3], b2);
-        assert_eq!(reader.read(&mut buf).unwrap(), 0);
-    }
-
-    #[test]
-    fn read_to_end() {
-        let mut reader = Cursor::new(vec![0, 1, 2, 3, 4, 5, 6, 7]);
-        let mut v = Vec::new();
-        reader.read_to_end(&mut v).unwrap();
-        assert_eq!(v, [0, 1, 2, 3, 4, 5, 6, 7]);
-    }
-
-    #[test]
-    fn test_slice_reader() {
-        let in_buf = vec![0, 1, 2, 3, 4, 5, 6, 7];
-        let reader = &mut &in_buf[..];
-        let mut buf = [];
-        assert_eq!(reader.read(&mut buf).unwrap(), 0);
-        let mut buf = [0];
-        assert_eq!(reader.read(&mut buf).unwrap(), 1);
-        assert_eq!(reader.len(), 7);
-        let b: &[_] = &[0];
-        assert_eq!(&buf[..], b);
-        let mut buf = [0; 4];
-        assert_eq!(reader.read(&mut buf).unwrap(), 4);
-        assert_eq!(reader.len(), 3);
-        let b: &[_] = &[1, 2, 3, 4];
-        assert_eq!(&buf[..], b);
-        assert_eq!(reader.read(&mut buf).unwrap(), 3);
-        let b: &[_] = &[5, 6, 7];
-        assert_eq!(&buf[..3], b);
-        assert_eq!(reader.read(&mut buf).unwrap(), 0);
-    }
-
-    #[test]
-    fn test_slice_reader_vectored() {
-        let in_buf = vec![0, 1, 2, 3, 4, 5, 6, 7];
-        let reader = &mut &in_buf[..];
-        let mut buf = [];
-        assert_eq!(reader.read_vectored(&mut [IoSliceMut::new(&mut buf)]).unwrap(), 0);
-        let mut buf = [0];
-        assert_eq!(
-            reader
-                .read_vectored(&mut [IoSliceMut::new(&mut []), IoSliceMut::new(&mut buf),])
-                .unwrap(),
-            1,
-        );
-        assert_eq!(reader.len(), 7);
-        let b: &[_] = &[0];
-        assert_eq!(buf, b);
-        let mut buf1 = [0; 4];
-        let mut buf2 = [0; 4];
-        assert_eq!(
-            reader
-                .read_vectored(&mut [IoSliceMut::new(&mut buf1), IoSliceMut::new(&mut buf2)],)
-                .unwrap(),
-            7,
-        );
-        let b1: &[_] = &[1, 2, 3, 4];
-        let b2: &[_] = &[5, 6, 7];
-        assert_eq!(buf1, b1);
-        assert_eq!(&buf2[..3], b2);
-        assert_eq!(reader.read(&mut buf).unwrap(), 0);
-    }
-
-    #[test]
-    fn test_read_exact() {
-        let in_buf = vec![0, 1, 2, 3, 4, 5, 6, 7];
-        let reader = &mut &in_buf[..];
-        let mut buf = [];
-        assert!(reader.read_exact(&mut buf).is_ok());
-        let mut buf = [8];
-        assert!(reader.read_exact(&mut buf).is_ok());
-        assert_eq!(buf[0], 0);
-        assert_eq!(reader.len(), 7);
-        let mut buf = [0, 0, 0, 0, 0, 0, 0];
-        assert!(reader.read_exact(&mut buf).is_ok());
-        assert_eq!(buf, [1, 2, 3, 4, 5, 6, 7]);
-        assert_eq!(reader.len(), 0);
-        let mut buf = [0];
-        assert!(reader.read_exact(&mut buf).is_err());
-    }
-
-    #[test]
-    fn test_buf_reader() {
-        let in_buf = vec![0, 1, 2, 3, 4, 5, 6, 7];
-        let mut reader = Cursor::new(&in_buf[..]);
-        let mut buf = [];
-        assert_eq!(reader.read(&mut buf).unwrap(), 0);
-        assert_eq!(reader.position(), 0);
-        let mut buf = [0];
-        assert_eq!(reader.read(&mut buf).unwrap(), 1);
-        assert_eq!(reader.position(), 1);
-        let b: &[_] = &[0];
-        assert_eq!(buf, b);
-        let mut buf = [0; 4];
-        assert_eq!(reader.read(&mut buf).unwrap(), 4);
-        assert_eq!(reader.position(), 5);
-        let b: &[_] = &[1, 2, 3, 4];
-        assert_eq!(buf, b);
-        assert_eq!(reader.read(&mut buf).unwrap(), 3);
-        let b: &[_] = &[5, 6, 7];
-        assert_eq!(&buf[..3], b);
-        assert_eq!(reader.read(&mut buf).unwrap(), 0);
-    }
-
-    #[test]
-    fn seek_past_end() {
-        let buf = [0xff];
-        let mut r = Cursor::new(&buf[..]);
-        assert_eq!(r.seek(SeekFrom::Start(10)).unwrap(), 10);
-        assert_eq!(r.read(&mut [0]).unwrap(), 0);
-
-        let mut r = Cursor::new(vec![10]);
-        assert_eq!(r.seek(SeekFrom::Start(10)).unwrap(), 10);
-        assert_eq!(r.read(&mut [0]).unwrap(), 0);
-
-        let mut buf = [0];
-        let mut r = Cursor::new(&mut buf[..]);
-        assert_eq!(r.seek(SeekFrom::Start(10)).unwrap(), 10);
-        assert_eq!(r.write(&[3]).unwrap(), 0);
-
-        let mut r = Cursor::new(vec![10].into_boxed_slice());
-        assert_eq!(r.seek(SeekFrom::Start(10)).unwrap(), 10);
-        assert_eq!(r.write(&[3]).unwrap(), 0);
-    }
-
-    #[test]
-    fn seek_past_i64() {
-        let buf = [0xff];
-        let mut r = Cursor::new(&buf[..]);
-        assert_eq!(r.seek(SeekFrom::Start(6)).unwrap(), 6);
-        assert_eq!(r.seek(SeekFrom::Current(0x7ffffffffffffff0)).unwrap(), 0x7ffffffffffffff6);
-        assert_eq!(r.seek(SeekFrom::Current(0x10)).unwrap(), 0x8000000000000006);
-        assert_eq!(r.seek(SeekFrom::Current(0)).unwrap(), 0x8000000000000006);
-        assert!(r.seek(SeekFrom::Current(0x7ffffffffffffffd)).is_err());
-        assert_eq!(r.seek(SeekFrom::Current(-0x8000000000000000)).unwrap(), 6);
-
-        let mut r = Cursor::new(vec![10]);
-        assert_eq!(r.seek(SeekFrom::Start(6)).unwrap(), 6);
-        assert_eq!(r.seek(SeekFrom::Current(0x7ffffffffffffff0)).unwrap(), 0x7ffffffffffffff6);
-        assert_eq!(r.seek(SeekFrom::Current(0x10)).unwrap(), 0x8000000000000006);
-        assert_eq!(r.seek(SeekFrom::Current(0)).unwrap(), 0x8000000000000006);
-        assert!(r.seek(SeekFrom::Current(0x7ffffffffffffffd)).is_err());
-        assert_eq!(r.seek(SeekFrom::Current(-0x8000000000000000)).unwrap(), 6);
-
-        let mut buf = [0];
-        let mut r = Cursor::new(&mut buf[..]);
-        assert_eq!(r.seek(SeekFrom::Start(6)).unwrap(), 6);
-        assert_eq!(r.seek(SeekFrom::Current(0x7ffffffffffffff0)).unwrap(), 0x7ffffffffffffff6);
-        assert_eq!(r.seek(SeekFrom::Current(0x10)).unwrap(), 0x8000000000000006);
-        assert_eq!(r.seek(SeekFrom::Current(0)).unwrap(), 0x8000000000000006);
-        assert!(r.seek(SeekFrom::Current(0x7ffffffffffffffd)).is_err());
-        assert_eq!(r.seek(SeekFrom::Current(-0x8000000000000000)).unwrap(), 6);
-
-        let mut r = Cursor::new(vec![10].into_boxed_slice());
-        assert_eq!(r.seek(SeekFrom::Start(6)).unwrap(), 6);
-        assert_eq!(r.seek(SeekFrom::Current(0x7ffffffffffffff0)).unwrap(), 0x7ffffffffffffff6);
-        assert_eq!(r.seek(SeekFrom::Current(0x10)).unwrap(), 0x8000000000000006);
-        assert_eq!(r.seek(SeekFrom::Current(0)).unwrap(), 0x8000000000000006);
-        assert!(r.seek(SeekFrom::Current(0x7ffffffffffffffd)).is_err());
-        assert_eq!(r.seek(SeekFrom::Current(-0x8000000000000000)).unwrap(), 6);
-    }
-
-    #[test]
-    fn seek_before_0() {
-        let buf = [0xff];
-        let mut r = Cursor::new(&buf[..]);
-        assert!(r.seek(SeekFrom::End(-2)).is_err());
-
-        let mut r = Cursor::new(vec![10]);
-        assert!(r.seek(SeekFrom::End(-2)).is_err());
-
-        let mut buf = [0];
-        let mut r = Cursor::new(&mut buf[..]);
-        assert!(r.seek(SeekFrom::End(-2)).is_err());
-
-        let mut r = Cursor::new(vec![10].into_boxed_slice());
-        assert!(r.seek(SeekFrom::End(-2)).is_err());
-    }
-
-    #[test]
-    fn test_seekable_mem_writer() {
-        let mut writer = Cursor::new(Vec::<u8>::new());
-        assert_eq!(writer.position(), 0);
-        assert_eq!(writer.write(&[0]).unwrap(), 1);
-        assert_eq!(writer.position(), 1);
-        assert_eq!(writer.write(&[1, 2, 3]).unwrap(), 3);
-        assert_eq!(writer.write(&[4, 5, 6, 7]).unwrap(), 4);
-        assert_eq!(writer.position(), 8);
-        let b: &[_] = &[0, 1, 2, 3, 4, 5, 6, 7];
-        assert_eq!(&writer.get_ref()[..], b);
-
-        assert_eq!(writer.seek(SeekFrom::Start(0)).unwrap(), 0);
-        assert_eq!(writer.position(), 0);
-        assert_eq!(writer.write(&[3, 4]).unwrap(), 2);
-        let b: &[_] = &[3, 4, 2, 3, 4, 5, 6, 7];
-        assert_eq!(&writer.get_ref()[..], b);
-
-        assert_eq!(writer.seek(SeekFrom::Current(1)).unwrap(), 3);
-        assert_eq!(writer.write(&[0, 1]).unwrap(), 2);
-        let b: &[_] = &[3, 4, 2, 0, 1, 5, 6, 7];
-        assert_eq!(&writer.get_ref()[..], b);
-
-        assert_eq!(writer.seek(SeekFrom::End(-1)).unwrap(), 7);
-        assert_eq!(writer.write(&[1, 2]).unwrap(), 2);
-        let b: &[_] = &[3, 4, 2, 0, 1, 5, 6, 1, 2];
-        assert_eq!(&writer.get_ref()[..], b);
-
-        assert_eq!(writer.seek(SeekFrom::End(1)).unwrap(), 10);
-        assert_eq!(writer.write(&[1]).unwrap(), 1);
-        let b: &[_] = &[3, 4, 2, 0, 1, 5, 6, 1, 2, 0, 1];
-        assert_eq!(&writer.get_ref()[..], b);
-    }
-
-    #[test]
-    fn vec_seek_past_end() {
-        let mut r = Cursor::new(Vec::new());
-        assert_eq!(r.seek(SeekFrom::Start(10)).unwrap(), 10);
-        assert_eq!(r.write(&[3]).unwrap(), 1);
-    }
-
-    #[test]
-    fn vec_seek_before_0() {
-        let mut r = Cursor::new(Vec::new());
-        assert!(r.seek(SeekFrom::End(-2)).is_err());
-    }
-
-    #[test]
-    #[cfg(target_pointer_width = "32")]
-    fn vec_seek_and_write_past_usize_max() {
-        let mut c = Cursor::new(Vec::new());
-        c.set_position(<usize>::max_value() as u64 + 1);
-        assert!(c.write_all(&[1, 2, 3]).is_err());
-    }
-
-    #[test]
-    fn test_partial_eq() {
-        assert_eq!(Cursor::new(Vec::<u8>::new()), Cursor::new(Vec::<u8>::new()));
-    }
-
-    #[test]
-    fn test_eq() {
-        struct AssertEq<T: Eq>(pub T);
-
-        let _: AssertEq<Cursor<Vec<u8>>> = AssertEq(Cursor::new(Vec::new()));
-    }
-}

src/libcoreio/src/io/error.rs

@@ -1,551 +0,0 @@
-#[cfg(feature="alloc")] use alloc::boxed::Box;
-#[cfg(not(feature="alloc"))] use ::FakeBox as Box;
-use core::convert::Into;
-use core::fmt;
-use core::marker::{Send, Sync};
-use core::option::Option::{self, Some, None};
-use core::result;
-#[cfg(feature="collections")] use collections::string::String;
-#[cfg(not(feature="collections"))] use ::ErrorString as String;
-use core::convert::From;
-
-/// A specialized [`Result`](../result/enum.Result.html) type for I/O
-/// operations.
-///
-/// This type is broadly used across [`std::io`] for any operation which may
-/// produce an error.
-///
-/// This typedef is generally used to avoid writing out [`io::Error`] directly and
-/// is otherwise a direct mapping to [`Result`].
-///
-/// While usual Rust style is to import types directly, aliases of [`Result`]
-/// often are not, to make it easier to distinguish between them. [`Result`] is
-/// generally assumed to be [`std::result::Result`][`Result`], and so users of this alias
-/// will generally use `io::Result` instead of shadowing the prelude's import
-/// of [`std::result::Result`][`Result`].
-///
-/// [`std::io`]: ../io/index.html
-/// [`io::Error`]: ../io/struct.Error.html
-/// [`Result`]: ../result/enum.Result.html
-///
-/// # Examples
-///
-/// A convenience function that bubbles an `io::Result` to its caller:
-///
-/// ```
-/// use std::io;
-///
-/// fn get_string() -> io::Result<String> {
-///     let mut buffer = String::new();
-///
-///     io::stdin().read_line(&mut buffer)?;
-///
-///     Ok(buffer)
-/// }
-/// ```
-pub type Result<T> = result::Result<T, Error>;
-
-/// The error type for I/O operations of the [`Read`], [`Write`], [`Seek`], and
-/// associated traits.
-///
-/// Errors mostly originate from the underlying OS, but custom instances of
-/// `Error` can be created with crafted error messages and a particular value of
-/// [`ErrorKind`].
-///
-/// [`Read`]: ../io/trait.Read.html
-/// [`Write`]: ../io/trait.Write.html
-/// [`Seek`]: ../io/trait.Seek.html
-/// [`ErrorKind`]: enum.ErrorKind.html
-pub struct Error {
-    repr: Repr,
-}
-
-impl fmt::Debug for Error {
-    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
-        fmt::Debug::fmt(&self.repr, f)
-    }
-}
-
-enum Repr {
-    Os(i32),
-    Simple(ErrorKind),
-    #[cfg(feature="alloc")]
-    Custom(Box<Custom>),
-    #[cfg(not(feature="alloc"))]
-    Custom(Custom),
-}
-
-#[derive(Debug)]
-struct Custom {
-    kind: ErrorKind,
-    error: String,
-}
-
-/// A list specifying general categories of I/O error.
-///
-/// This list is intended to grow over time and it is not recommended to
-/// exhaustively match against it.
-///
-/// It is used with the [`io::Error`] type.
-///
-/// [`io::Error`]: struct.Error.html
-#[derive(Clone, Copy, Debug, Eq, Hash, Ord, PartialEq, PartialOrd)]
-#[allow(deprecated)]
-#[non_exhaustive]
-pub enum ErrorKind {
-    /// An entity was not found, often a file.
-    NotFound,
-    /// The operation lacked the necessary privileges to complete.
-    PermissionDenied,
-    /// The connection was refused by the remote server.
-    ConnectionRefused,
-    /// The connection was reset by the remote server.
-    ConnectionReset,
-    /// The connection was aborted (terminated) by the remote server.
-    ConnectionAborted,
-    /// The network operation failed because it was not connected yet.
-    NotConnected,
-    /// A socket address could not be bound because the address is already in
-    /// use elsewhere.
-    AddrInUse,
-    /// A nonexistent interface was requested or the requested address was not
-    /// local.
-    AddrNotAvailable,
-    /// The operation failed because a pipe was closed.
-    BrokenPipe,
-    /// An entity already exists, often a file.
-    AlreadyExists,
-    /// The operation needs to block to complete, but the blocking operation was
-    /// requested to not occur.
-    WouldBlock,
-    /// A parameter was incorrect.
-    InvalidInput,
-    /// Data not valid for the operation were encountered.
-    ///
-    /// Unlike [`InvalidInput`], this typically means that the operation
-    /// parameters were valid, however the error was caused by malformed
-    /// input data.
-    ///
-    /// For example, a function that reads a file into a string will error with
-    /// `InvalidData` if the file's contents are not valid UTF-8.
-    ///
-    /// [`InvalidInput`]: #variant.InvalidInput
-    InvalidData,
-    /// The I/O operation's timeout expired, causing it to be canceled.
-    TimedOut,
-    /// An error returned when an operation could not be completed because a
-    /// call to [`write`] returned [`Ok(0)`].
-    ///
-    /// This typically means that an operation could only succeed if it wrote a
-    /// particular number of bytes but only a smaller number of bytes could be
-    /// written.
-    ///
-    /// [`write`]: ../../std/io/trait.Write.html#tymethod.write
-    /// [`Ok(0)`]: ../../std/io/type.Result.html
-    WriteZero,
-    /// This operation was interrupted.
-    ///
-    /// Interrupted operations can typically be retried.
-    Interrupted,
-    /// Any I/O error not part of this list.
-    Other,
-
-    /// An error returned when an operation could not be completed because an
-    /// "end of file" was reached prematurely.
-    ///
-    /// This typically means that an operation could only succeed if it read a
-    /// particular number of bytes but only a smaller number of bytes could be
-    /// read.
-    UnexpectedEof,
-}
-
-impl ErrorKind {
-    pub(crate) fn as_str(&self) -> &'static str {
-        match *self {
-            ErrorKind::NotFound => "entity not found",
-            ErrorKind::PermissionDenied => "permission denied",
-            ErrorKind::ConnectionRefused => "connection refused",
-            ErrorKind::ConnectionReset => "connection reset",
-            ErrorKind::ConnectionAborted => "connection aborted",
-            ErrorKind::NotConnected => "not connected",
-            ErrorKind::AddrInUse => "address in use",
-            ErrorKind::AddrNotAvailable => "address not available",
-            ErrorKind::BrokenPipe => "broken pipe",
-            ErrorKind::AlreadyExists => "entity already exists",
-            ErrorKind::WouldBlock => "operation would block",
-            ErrorKind::InvalidInput => "invalid input parameter",
-            ErrorKind::InvalidData => "invalid data",
-            ErrorKind::TimedOut => "timed out",
-            ErrorKind::WriteZero => "write zero",
-            ErrorKind::Interrupted => "operation interrupted",
-            ErrorKind::Other => "other os error",
-            ErrorKind::UnexpectedEof => "unexpected end of file",
-        }
-    }
-}
-
-/// Intended for use for errors not exposed to the user, where allocating onto
-/// the heap (for normal construction via Error::new) is too costly.
-impl From<ErrorKind> for Error {
-    /// Converts an [`ErrorKind`] into an [`Error`].
-    ///
-    /// This conversion allocates a new error with a simple representation of error kind.
-    ///
-    /// # Examples
-    ///
-    /// ```
-    /// use std::io::{Error, ErrorKind};
-    ///
-    /// let not_found = ErrorKind::NotFound;
-    /// let error = Error::from(not_found);
-    /// assert_eq!("entity not found", format!("{}", error));
-    /// ```
-    ///
-    /// [`ErrorKind`]: ../../std/io/enum.ErrorKind.html
-    /// [`Error`]: ../../std/io/struct.Error.html
-    #[inline]
-    fn from(kind: ErrorKind) -> Error {
-        Error { repr: Repr::Simple(kind) }
-    }
-}
-
-impl Error {
-    /// Creates a new I/O error from a known kind of error as well as an
-    /// arbitrary error payload.
-    ///
-    /// This function is used to generically create I/O errors which do not
-    /// originate from the OS itself. The `error` argument is an arbitrary
-    /// payload which will be contained in this `Error`.
-    ///
-    /// # Examples
-    ///
-    /// ```
-    /// use std::io::{Error, ErrorKind};
-    ///
-    /// // errors can be created from strings
-    /// let custom_error = Error::new(ErrorKind::Other, "oh no!");
-    ///
-    /// // errors can also be created from other errors
-    /// let custom_error2 = Error::new(ErrorKind::Interrupted, custom_error);
-    /// ```
-    pub fn new<E>(kind: ErrorKind, error: E) -> Error
-    where
-        E: Into<String>,
-    {
-        Self::_new(kind, error.into())
-    }
-
-    fn _new(kind: ErrorKind, error: String) -> Error {
-        Error { repr: Repr::Custom(Box::new(Custom { kind, error })) }
-    }
-
-    /// Creates a new instance of an `Error` from a particular OS error code.
-    ///
-    /// # Examples
-    ///
-    /// On Linux:
-    ///
-    /// ```
-    /// # if cfg!(target_os = "linux") {
-    /// use std::io;
-    ///
-    /// let error = io::Error::from_raw_os_error(22);
-    /// assert_eq!(error.kind(), io::ErrorKind::InvalidInput);
-    /// # }
-    /// ```
-    ///
-    /// On Windows:
-    ///
-    /// ```
-    /// # if cfg!(windows) {
-    /// use std::io;
-    ///
-    /// let error = io::Error::from_raw_os_error(10022);
-    /// assert_eq!(error.kind(), io::ErrorKind::InvalidInput);
-    /// # }
-    /// ```
-    pub fn from_raw_os_error(code: i32) -> Error {
-        Error { repr: Repr::Os(code) }
-    }
-
-    /// Returns the OS error that this error represents (if any).
-    ///
-    /// If this `Error` was constructed via `last_os_error` or
-    /// `from_raw_os_error`, then this function will return `Some`, otherwise
-    /// it will return `None`.
-    ///
-    /// # Examples
-    ///
-    /// ```
-    /// use std::io::{Error, ErrorKind};
-    ///
-    /// fn print_os_error(err: &Error) {
-    ///     if let Some(raw_os_err) = err.raw_os_error() {
-    ///         println!("raw OS error: {:?}", raw_os_err);
-    ///     } else {
-    ///         println!("Not an OS error");
-    ///     }
-    /// }
-    ///
-    /// fn main() {
-    ///     // Will print "raw OS error: ...".
-    ///     print_os_error(&Error::last_os_error());
-    ///     // Will print "Not an OS error".
-    ///     print_os_error(&Error::new(ErrorKind::Other, "oh no!"));
-    /// }
-    /// ```
-    pub fn raw_os_error(&self) -> Option<i32> {
-        match self.repr {
-            Repr::Os(i) => Some(i),
-            Repr::Custom(..) => None,
-            Repr::Simple(..) => None,
-        }
-    }
-
-    /// Returns a reference to the inner error wrapped by this error (if any).
-    ///
-    /// If this `Error` was constructed via `new` then this function will
-    /// return `Some`, otherwise it will return `None`.
-    ///
-    /// # Examples
-    ///
-    /// ```
-    /// use std::io::{Error, ErrorKind};
-    ///
-    /// fn print_error(err: &Error) {
-    ///     if let Some(inner_err) = err.get_ref() {
-    ///         println!("Inner error: {:?}", inner_err);
-    ///     } else {
-    ///         println!("No inner error");
-    ///     }
-    /// }
-    ///
-    /// fn main() {
-    ///     // Will print "No inner error".
-    ///     print_error(&Error::last_os_error());
-    ///     // Will print "Inner error: ...".
-    ///     print_error(&Error::new(ErrorKind::Other, "oh no!"));
-    /// }
-    /// ```
-    pub fn get_ref(&self) -> Option<&String> {
-        match self.repr {
-            Repr::Os(..) => None,
-            Repr::Simple(..) => None,
-            Repr::Custom(ref c) => Some(&c.error),
-        }
-    }
-
-    /// Returns a mutable reference to the inner error wrapped by this error
-    /// (if any).
-    ///
-    /// If this `Error` was constructed via `new` then this function will
-    /// return `Some`, otherwise it will return `None`.
-    ///
-    /// # Examples
-    ///
-    /// ```
-    /// use std::io::{Error, ErrorKind};
-    /// use std::{error, fmt};
-    /// use std::fmt::Display;
-    ///
-    /// #[derive(Debug)]
-    /// struct MyError {
-    ///     v: String,
-    /// }
-    ///
-    /// impl MyError {
-    ///     fn new() -> MyError {
-    ///         MyError {
-    ///             v: "oh no!".to_string()
-    ///         }
-    ///     }
-    ///
-    ///     fn change_message(&mut self, new_message: &str) {
-    ///         self.v = new_message.to_string();
-    ///     }
-    /// }
-    ///
-    /// impl error::Error for MyError {}
-    ///
-    /// impl Display for MyError {
-    ///     fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
-    ///         write!(f, "MyError: {}", &self.v)
-    ///     }
-    /// }
-    ///
-    /// fn change_error(mut err: Error) -> Error {
-    ///     if let Some(inner_err) = err.get_mut() {
-    ///         inner_err.downcast_mut::<MyError>().unwrap().change_message("I've been changed!");
-    ///     }
-    ///     err
-    /// }
-    ///
-    /// fn print_error(err: &Error) {
-    ///     if let Some(inner_err) = err.get_ref() {
-    ///         println!("Inner error: {}", inner_err);
-    ///     } else {
-    ///         println!("No inner error");
-    ///     }
-    /// }
-    ///
-    /// fn main() {
-    ///     // Will print "No inner error".
-    ///     print_error(&change_error(Error::last_os_error()));
-    ///     // Will print "Inner error: ...".
-    ///     print_error(&change_error(Error::new(ErrorKind::Other, MyError::new())));
-    /// }
-    /// ```
-    pub fn get_mut(&mut self) -> Option<&mut String> {
-        match self.repr {
-            Repr::Os(..) => None,
-            Repr::Simple(..) => None,
-            Repr::Custom(ref mut c) => Some(&mut c.error),
-        }
-    }
-
-    /// Consumes the `Error`, returning its inner error (if any).
-    ///
-    /// If this `Error` was constructed via `new` then this function will
-    /// return `Some`, otherwise it will return `None`.
-    ///
-    /// # Examples
-    ///
-    /// ```
-    /// use std::io::{Error, ErrorKind};
-    ///
-    /// fn print_error(err: Error) {
-    ///     if let Some(inner_err) = err.into_inner() {
-    ///         println!("Inner error: {}", inner_err);
-    ///     } else {
-    ///         println!("No inner error");
-    ///     }
-    /// }
-    ///
-    /// fn main() {
-    ///     // Will print "No inner error".
-    ///     print_error(Error::last_os_error());
-    ///     // Will print "Inner error: ...".
-    ///     print_error(Error::new(ErrorKind::Other, "oh no!"));
-    /// }
-    /// ```
-    pub fn into_inner(self) -> Option<String> {
-        match self.repr {
-            Repr::Os(..) => None,
-            Repr::Simple(..) => None,
-            Repr::Custom(c) => Some(c.error),
-        }
-    }
-
-    /// Returns the corresponding `ErrorKind` for this error.
-    ///
-    /// # Examples
-    ///
-    /// ```
-    /// use std::io::{Error, ErrorKind};
-    ///
-    /// fn print_error(err: Error) {
-    ///     println!("{:?}", err.kind());
-    /// }
-    ///
-    /// fn main() {
-    ///     // Will print "No inner error".
-    ///     print_error(Error::last_os_error());
-    ///     // Will print "Inner error: ...".
-    ///     print_error(Error::new(ErrorKind::AddrInUse, "oh no!"));
-    /// }
-    /// ```
-    pub fn kind(&self) -> ErrorKind {
-        match self.repr {
-            Repr::Os(_code) => ErrorKind::Other,
-            Repr::Custom(ref c) => c.kind,
-            Repr::Simple(kind) => kind,
-        }
-    }
-}
-
-impl fmt::Debug for Repr {
-    fn fmt(&self, fmt: &mut fmt::Formatter<'_>) -> fmt::Result {
-        match *self {
-            Repr::Os(code) => fmt
-                .debug_struct("Os")
-                .field("code", &code)
-                .finish(),
-            Repr::Custom(ref c) => fmt::Debug::fmt(&c, fmt),
-            Repr::Simple(kind) => fmt.debug_tuple("Kind").field(&kind).finish(),
-        }
-    }
-}
-
-impl fmt::Display for Error {
-    fn fmt(&self, fmt: &mut fmt::Formatter<'_>) -> fmt::Result {
-        match self.repr {
-            Repr::Os(code) => {
-                write!(fmt, "os error {}", code)
-            }
-            Repr::Custom(ref c) => c.error.fmt(fmt),
-            Repr::Simple(kind) => write!(fmt, "{}", kind.as_str()),
-        }
-    }
-}
-
-fn _assert_error_is_sync_send() {
-    fn _is_sync_send<T: Sync + Send>() {}
-    _is_sync_send::<Error>();
-}
-
-#[cfg(test)]
-mod test {
-    use super::{Custom, Error, ErrorKind, Repr};
-    use crate::error;
-    use crate::fmt;
-    use crate::sys::decode_error_kind;
-    use crate::sys::os::error_string;
-
-    #[test]
-    fn test_debug_error() {
-        let code = 6;
-        let msg = error_string(code);
-        let kind = decode_error_kind(code);
-        let err = Error {
-            repr: Repr::Custom(box Custom {
-                kind: ErrorKind::InvalidInput,
-                error: box Error { repr: super::Repr::Os(code) },
-            }),
-        };
-        let expected = format!(
-            "Custom {{ \
-             kind: InvalidInput, \
-             error: Os {{ \
-             code: {:?}, \
-             kind: {:?}, \
-             message: {:?} \
-             }} \
-             }}",
-            code, kind, msg
-        );
-        assert_eq!(format!("{:?}", err), expected);
-    }
-
-    #[test]
-    fn test_downcasting() {
-        #[derive(Debug)]
-        struct TestError;
-
-        impl fmt::Display for TestError {
-            fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
-                f.write_str("asdf")
-            }
-        }
-
-        impl error::Error for TestError {}
-
-        // we have to call all of these UFCS style right now since method
-        // resolution won't implicitly drop the Send+Sync bounds
-        let mut err = Error::new(ErrorKind::Other, TestError);
-        assert!(err.get_ref().unwrap().is::<TestError>());
-        assert_eq!("asdf", err.get_ref().unwrap().to_string());
-        assert!(err.get_mut().unwrap().is::<TestError>());
-        let extracted = err.into_inner().unwrap();
-        extracted.downcast::<TestError>().unwrap();
-    }
-}

src/libcoreio/src/io/impls.rs

@@ -1,378 +0,0 @@
-use core::cmp;
-use core::fmt;
-use crate::io::{
-    self, Error, ErrorKind, Initializer, Read, Seek, SeekFrom, Write,
-};
-#[cfg(feature = "collections")] use crate::io::BufRead;
-use core::mem;
-
-#[cfg(feature="collections")] 
-use collections::{
-    vec::Vec,
-    string::String,
-};
-#[cfg(feature = "alloc")]
-use alloc::boxed::Box;
-
-// =============================================================================
-// Forwarding implementations
-
-impl<R: Read + ?Sized> Read for &mut R {
-    #[inline]
-    fn read(&mut self, buf: &mut [u8]) -> io::Result<usize> {
-        (**self).read(buf)
-    }
-
-    #[inline]
-    unsafe fn initializer(&self) -> Initializer {
-        (**self).initializer()
-    }
-
-    #[cfg(feature="collections")]
-    #[inline]
-    fn read_to_end(&mut self, buf: &mut Vec<u8>) -> io::Result<usize> {
-        (**self).read_to_end(buf)
-    }
-
-    #[cfg(feature="collections")]
-    #[inline]
-    fn read_to_string(&mut self, buf: &mut String) -> io::Result<usize> {
-        (**self).read_to_string(buf)
-    }
-
-    #[inline]
-    fn read_exact(&mut self, buf: &mut [u8]) -> io::Result<()> {
-        (**self).read_exact(buf)
-    }
-}
-impl<W: Write + ?Sized> Write for &mut W {
-    #[inline]
-    fn write(&mut self, buf: &[u8]) -> io::Result<usize> {
-        (**self).write(buf)
-    }
-
-    #[inline]
-    fn flush(&mut self) -> io::Result<()> {
-        (**self).flush()
-    }
-
-    #[inline]
-    fn write_all(&mut self, buf: &[u8]) -> io::Result<()> {
-        (**self).write_all(buf)
-    }
-
-    #[inline]
-    fn write_fmt(&mut self, fmt: fmt::Arguments<'_>) -> io::Result<()> {
-        (**self).write_fmt(fmt)
-    }
-}
-impl<S: Seek + ?Sized> Seek for &mut S {
-    #[inline]
-    fn seek(&mut self, pos: SeekFrom) -> io::Result<u64> {
-        (**self).seek(pos)
-    }
-}
-#[cfg(feature = "collections")]
-impl<B: BufRead + ?Sized> BufRead for &mut B {
-    #[inline]
-    fn fill_buf(&mut self) -> io::Result<&[u8]> {
-        (**self).fill_buf()
-    }
-
-    #[inline]
-    fn consume(&mut self, amt: usize) {
-        (**self).consume(amt)
-    }
-
-    #[cfg(feature="collections")]
-    #[inline]
-    fn read_until(&mut self, byte: u8, buf: &mut Vec<u8>) -> io::Result<usize> {
-        (**self).read_until(byte, buf)
-    }
-
-    #[cfg(feature="collections")]
-    #[inline]
-    fn read_line(&mut self, buf: &mut String) -> io::Result<usize> {
-        (**self).read_line(buf)
-    }
-}
-
-#[cfg(feature="alloc")]
-#[cfg(feature="collections")]
-impl<R: Read + ?Sized> Read for Box<R> {
-    #[inline]
-    fn read(&mut self, buf: &mut [u8]) -> io::Result<usize> {
-        (**self).read(buf)
-    }
-
-    #[cfg(feature="collections")]
-    #[inline]
-    fn read_to_end(&mut self, buf: &mut Vec<u8>) -> io::Result<usize> {
-        (**self).read_to_end(buf)
-    }
-
-    #[cfg(feature="collections")]
-    #[inline]
-    fn read_to_string(&mut self, buf: &mut String) -> io::Result<usize> {
-        (**self).read_to_string(buf)
-    }
-
-    #[inline]
-    fn read_exact(&mut self, buf: &mut [u8]) -> io::Result<()> {
-        (**self).read_exact(buf)
-    }
-}
-#[cfg(feature="alloc")]
-#[cfg(feature="collections")]
-impl<W: Write + ?Sized> Write for Box<W> {
-    #[inline]
-    fn write(&mut self, buf: &[u8]) -> io::Result<usize> {
-        (**self).write(buf)
-    }
-
-    #[inline]
-    fn flush(&mut self) -> io::Result<()> {
-        (**self).flush()
-    }
-
-    #[inline]
-    fn write_all(&mut self, buf: &[u8]) -> io::Result<()> {
-        (**self).write_all(buf)
-    }
-
-    #[inline]
-    fn write_fmt(&mut self, fmt: fmt::Arguments<'_>) -> io::Result<()> {
-        (**self).write_fmt(fmt)
-    }
-}
-#[cfg(feature="collections")]
-impl<S: Seek + ?Sized> Seek for Box<S> {
-    #[inline]
-    fn seek(&mut self, pos: SeekFrom) -> io::Result<u64> {
-        (**self).seek(pos)
-    }
-}
-#[cfg(feature="collections")]
-impl<B: BufRead + ?Sized> BufRead for Box<B> {
-    #[inline]
-    fn fill_buf(&mut self) -> io::Result<&[u8]> {
-        (**self).fill_buf()
-    }
-
-    #[inline]
-    fn consume(&mut self, amt: usize) {
-        (**self).consume(amt)
-    }
-
-    #[inline]
-    fn read_until(&mut self, byte: u8, buf: &mut Vec<u8>) -> io::Result<usize> {
-        (**self).read_until(byte, buf)
-    }
-
-    #[inline]
-    fn read_line(&mut self, buf: &mut String) -> io::Result<usize> {
-        (**self).read_line(buf)
-    }
-}
-
-// Used by panicking::default_hook
-#[cfg(test)]
-/// This impl is only used by printing logic, so any error returned is always
-/// of kind `Other`, and should be ignored.
-#[cfg(feature="collections")]
-impl Write for Box<dyn (::realstd::io::Write) + Send> {
-    fn write(&mut self, buf: &[u8]) -> io::Result<usize> {
-        (**self).write(buf).map_err(|_| ErrorKind::Other.into())
-    }
-
-    fn flush(&mut self) -> io::Result<()> {
-        (**self).flush().map_err(|_| ErrorKind::Other.into())
-    }
-}
-
-// =============================================================================
-// In-memory buffer implementations
-
-/// Read is implemented for `&[u8]` by copying from the slice.
-///
-/// Note that reading updates the slice to point to the yet unread part.
-/// The slice will be empty when EOF is reached.
-impl Read for &[u8] {
-    #[inline]
-    fn read(&mut self, buf: &mut [u8]) -> io::Result<usize> {
-        let amt = cmp::min(buf.len(), self.len());
-        let (a, b) = self.split_at(amt);
-
-        // First check if the amount of bytes we want to read is small:
-        // `copy_from_slice` will generally expand to a call to `memcpy`, and
-        // for a single byte the overhead is significant.
-        if amt == 1 {
-            buf[0] = a[0];
-        } else {
-            buf[..amt].copy_from_slice(a);
-        }
-
-        *self = b;
-        Ok(amt)
-    }
-
-    #[inline]
-    unsafe fn initializer(&self) -> Initializer {
-        Initializer::nop()
-    }
-
-    #[inline]
-    fn read_exact(&mut self, buf: &mut [u8]) -> io::Result<()> {
-        if buf.len() > self.len() {
-            return Err(Error::new(ErrorKind::UnexpectedEof, "failed to fill whole buffer"));
-        }
-        let (a, b) = self.split_at(buf.len());
-
-        // First check if the amount of bytes we want to read is small:
-        // `copy_from_slice` will generally expand to a call to `memcpy`, and
-        // for a single byte the overhead is significant.
-        if buf.len() == 1 {
-            buf[0] = a[0];
-        } else {
-            buf.copy_from_slice(a);
-        }
-
-        *self = b;
-        Ok(())
-    }
-
-    #[cfg(feature="collections")]
-    #[inline]
-    fn read_to_end(&mut self, buf: &mut Vec<u8>) -> io::Result<usize> {
-        buf.extend_from_slice(*self);
-        let len = self.len();
-        *self = &self[len..];
-        Ok(len)
-    }
-}
-
-#[cfg(feature="collections")]
-impl BufRead for &[u8] {
-    #[inline]
-    fn fill_buf(&mut self) -> io::Result<&[u8]> {
-        Ok(*self)
-    }
-
-    #[inline]
-    fn consume(&mut self, amt: usize) {
-        *self = &self[amt..];
-    }
-}
-
-/// Write is implemented for `&mut [u8]` by copying into the slice, overwriting
-/// its data.
-///
-/// Note that writing updates the slice to point to the yet unwritten part.
-/// The slice will be empty when it has been completely overwritten.
-impl Write for &mut [u8] {
-    #[inline]
-    fn write(&mut self, data: &[u8]) -> io::Result<usize> {
-        let amt = cmp::min(data.len(), self.len());
-        let (a, b) = mem::replace(self, &mut []).split_at_mut(amt);
-        a.copy_from_slice(&data[..amt]);
-        *self = b;
-        Ok(amt)
-    }
-
-    #[inline]
-    fn write_all(&mut self, data: &[u8]) -> io::Result<()> {
-        if self.write(data)? == data.len() {
-            Ok(())
-        } else {
-            Err(Error::new(ErrorKind::WriteZero, "failed to write whole buffer"))
-        }
-    }
-
-    #[inline]
-    fn flush(&mut self) -> io::Result<()> {
-        Ok(())
-    }
-}
-
-/// Write is implemented for `Vec<u8>` by appending to the vector.
-/// The vector will grow as needed.
-#[cfg(feature="collections")]
-impl Write for Vec<u8> {
-    #[inline]
-    fn write(&mut self, buf: &[u8]) -> io::Result<usize> {
-        self.extend_from_slice(buf);
-        Ok(buf.len())
-    }
-
-    #[inline]
-    fn write_all(&mut self, buf: &[u8]) -> io::Result<()> {
-        self.extend_from_slice(buf);
-        Ok(())
-    }
-
-    #[inline]
-    fn flush(&mut self) -> io::Result<()> {
-        Ok(())
-    }
-}
-
-#[cfg(test)]
-mod tests {
-    use crate::io::prelude::*;
-
-    #[bench]
-    fn bench_read_slice(b: &mut test::Bencher) {
-        let buf = [5; 1024];
-        let mut dst = [0; 128];
-
-        b.iter(|| {
-            let mut rd = &buf[..];
-            for _ in 0..8 {
-                let _ = rd.read(&mut dst);
-                test::black_box(&dst);
-            }
-        })
-    }
-
-    #[bench]
-    fn bench_write_slice(b: &mut test::Bencher) {
-        let mut buf = [0; 1024];
-        let src = [5; 128];
-
-        b.iter(|| {
-            let mut wr = &mut buf[..];
-            for _ in 0..8 {
-                let _ = wr.write_all(&src);
-                test::black_box(&wr);
-            }
-        })
-    }
-
-    #[bench]
-    fn bench_read_vec(b: &mut test::Bencher) {
-        let buf = vec![5; 1024];
-        let mut dst = [0; 128];
-
-        b.iter(|| {
-            let mut rd = &buf[..];
-            for _ in 0..8 {
-                let _ = rd.read(&mut dst);
-                test::black_box(&dst);
-            }
-        })
-    }
-
-    #[bench]
-    fn bench_write_vec(b: &mut test::Bencher) {
-        let mut buf = Vec::with_capacity(1024);
-        let src = [5; 128];
-
-        b.iter(|| {
-            let mut wr = &mut buf[..];
-            for _ in 0..8 {
-                let _ = wr.write_all(&src);
-                test::black_box(&wr);
-            }
-        })
-    }
-}

src/libcoreio/src/io/prelude.rs

@@ -1,13 +0,0 @@
-//! The I/O Prelude
-//!
-//! The purpose of this module is to alleviate imports of many common I/O traits
-//! by adding a glob import to the top of I/O heavy modules:
-//!
-//! ```
-//! # #![allow(unused_imports)]
-//! use std::io::prelude::*;
-//! ```
-
-
-pub use super::{Read, Seek, Write};
-#[cfg(feature = "collections")] pub use super::BufRead;

src/libcoreio/src/io/util.rs

@@ -1,269 +0,0 @@
-#![allow(missing_copy_implementations)]
-
-use core::fmt;
-use core::mem;
-use crate::io::{self, ErrorKind, Initializer, Read, Write};
-#[cfg(feature = "collections")] use crate::io::BufRead;
-
-/// Copies the entire contents of a reader into a writer.
-///
-/// This function will continuously read data from `reader` and then
-/// write it into `writer` in a streaming fashion until `reader`
-/// returns EOF.
-///
-/// On success, the total number of bytes that were copied from
-/// `reader` to `writer` is returned.
-///
-/// If you’re wanting to copy the contents of one file to another and you’re
-/// working with filesystem paths, see the [`fs::copy`] function.
-///
-/// [`fs::copy`]: ../fs/fn.copy.html
-///
-/// # Errors
-///
-/// This function will return an error immediately if any call to `read` or
-/// `write` returns an error. All instances of `ErrorKind::Interrupted` are
-/// handled by this function and the underlying operation is retried.
-///
-/// # Examples
-///
-/// ```
-/// use std::io;
-///
-/// fn main() -> io::Result<()> {
-///     let mut reader: &[u8] = b"hello";
-///     let mut writer: Vec<u8> = vec![];
-///
-///     io::copy(&mut reader, &mut writer)?;
-///
-///     assert_eq!(&b"hello"[..], &writer[..]);
-///     Ok(())
-/// }
-/// ```
-pub fn copy<R: ?Sized, W: ?Sized>(reader: &mut R, writer: &mut W) -> io::Result<u64>
-where
-    R: Read,
-    W: Write,
-{
-    let mut buf = unsafe {
-        #[allow(deprecated)]
-        let mut buf: [u8; super::DEFAULT_BUF_SIZE] = mem::uninitialized();
-        reader.initializer().initialize(&mut buf);
-        buf
-    };
-
-    let mut written = 0;
-    loop {
-        let len = match reader.read(&mut buf) {
-            Ok(0) => return Ok(written),
-            Ok(len) => len,
-            Err(ref e) if e.kind() == ErrorKind::Interrupted => continue,
-            Err(e) => return Err(e),
-        };
-        writer.write_all(&buf[..len])?;
-        written += len as u64;
-    }
-}
-
-/// A reader which is always at EOF.
-///
-/// This struct is generally created by calling [`empty`]. Please see
-/// the documentation of [`empty()`][`empty`] for more details.
-///
-/// [`empty`]: fn.empty.html
-pub struct Empty {
-    _priv: (),
-}
-
-/// Constructs a new handle to an empty reader.
-///
-/// All reads from the returned reader will return [`Ok`]`(0)`.
-///
-/// [`Ok`]: ../result/enum.Result.html#variant.Ok
-///
-/// # Examples
-///
-/// A slightly sad example of not reading anything into a buffer:
-///
-/// ```
-/// use std::io::{self, Read};
-///
-/// let mut buffer = String::new();
-/// io::empty().read_to_string(&mut buffer).unwrap();
-/// assert!(buffer.is_empty());
-/// ```
-pub fn empty() -> Empty {
-    Empty { _priv: () }
-}
-
-impl Read for Empty {
-    #[inline]
-    fn read(&mut self, _buf: &mut [u8]) -> io::Result<usize> {
-        Ok(0)
-    }
-
-    #[inline]
-    unsafe fn initializer(&self) -> Initializer {
-        Initializer::nop()
-    }
-}
-
-#[cfg(feature="collections")]
-impl BufRead for Empty {
-    #[inline]
-    fn fill_buf(&mut self) -> io::Result<&[u8]> {
-        Ok(&[])
-    }
-    #[inline]
-    fn consume(&mut self, _n: usize) {}
-}
-
-impl fmt::Debug for Empty {
-    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
-        f.pad("Empty { .. }")
-    }
-}
-
-/// A reader which yields one byte over and over and over and over and over and...
-///
-/// This struct is generally created by calling [`repeat`][repeat]. Please
-/// see the documentation of `repeat()` for more details.
-///
-/// [repeat]: fn.repeat.html
-pub struct Repeat {
-    byte: u8,
-}
-
-/// Creates an instance of a reader that infinitely repeats one byte.
-///
-/// All reads from this reader will succeed by filling the specified buffer with
-/// the given byte.
-///
-/// # Examples
-///
-/// ```
-/// use std::io::{self, Read};
-///
-/// let mut buffer = [0; 3];
-/// io::repeat(0b101).read_exact(&mut buffer).unwrap();
-/// assert_eq!(buffer, [0b101, 0b101, 0b101]);
-/// ```
-pub fn repeat(byte: u8) -> Repeat {
-    Repeat { byte }
-}
-
-impl Read for Repeat {
-    #[inline]
-    fn read(&mut self, buf: &mut [u8]) -> io::Result<usize> {
-        for slot in &mut *buf {
-            *slot = self.byte;
-        }
-        Ok(buf.len())
-    }
-
-    #[inline]
-    unsafe fn initializer(&self) -> Initializer {
-        Initializer::nop()
-    }
-}
-
-impl fmt::Debug for Repeat {
-    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
-        f.pad("Repeat { .. }")
-    }
-}
-
-/// A writer which will move data into the void.
-///
-/// This struct is generally created by calling [`sink`][sink]. Please
-/// see the documentation of `sink()` for more details.
-///
-/// [sink]: fn.sink.html
-pub struct Sink {
-    _priv: (),
-}
-
-/// Creates an instance of a writer which will successfully consume all data.
-///
-/// All calls to `write` on the returned instance will return `Ok(buf.len())`
-/// and the contents of the buffer will not be inspected.
-///
-/// # Examples
-///
-/// ```rust
-/// use std::io::{self, Write};
-///
-/// let buffer = vec![1, 2, 3, 5, 8];
-/// let num_bytes = io::sink().write(&buffer).unwrap();
-/// assert_eq!(num_bytes, 5);
-/// ```
-pub fn sink() -> Sink {
-    Sink { _priv: () }
-}
-
-impl Write for Sink {
-    #[inline]
-    fn write(&mut self, buf: &[u8]) -> io::Result<usize> {
-        Ok(buf.len())
-    }
-
-    #[inline]
-    fn flush(&mut self) -> io::Result<()> {
-        Ok(())
-    }
-}
-
-impl fmt::Debug for Sink {
-    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
-        f.pad("Sink { .. }")
-    }
-}
-
-#[cfg(test)]
-mod tests {
-    use crate::io::prelude::*;
-    use crate::io::{copy, empty, repeat, sink};
-
-    #[test]
-    fn copy_copies() {
-        let mut r = repeat(0).take(4);
-        let mut w = sink();
-        assert_eq!(copy(&mut r, &mut w).unwrap(), 4);
-
-        let mut r = repeat(0).take(1 << 17);
-        assert_eq!(copy(&mut r as &mut dyn Read, &mut w as &mut dyn Write).unwrap(), 1 << 17);
-    }
-
-    #[test]
-    fn sink_sinks() {
-        let mut s = sink();
-        assert_eq!(s.write(&[]).unwrap(), 0);
-        assert_eq!(s.write(&[0]).unwrap(), 1);
-        assert_eq!(s.write(&[0; 1024]).unwrap(), 1024);
-        assert_eq!(s.by_ref().write(&[0; 1024]).unwrap(), 1024);
-    }
-
-    #[test]
-    fn empty_reads() {
-        let mut e = empty();
-        assert_eq!(e.read(&mut []).unwrap(), 0);
-        assert_eq!(e.read(&mut [0]).unwrap(), 0);
-        assert_eq!(e.read(&mut [0; 1024]).unwrap(), 0);
-        assert_eq!(e.by_ref().read(&mut [0; 1024]).unwrap(), 0);
-    }
-
-    #[test]
-    fn repeat_repeats() {
-        let mut r = repeat(4);
-        let mut b = [0; 1024];
-        assert_eq!(r.read(&mut b).unwrap(), 1024);
-        assert!(b.iter().all(|b| *b == 4));
-    }
-
-    #[test]
-    fn take_some_bytes() {
-        assert_eq!(repeat(4).take(100).bytes().count(), 100);
-        assert_eq!(repeat(4).take(100).bytes().next().unwrap().unwrap(), 4);
-        assert_eq!(repeat(1).take(10).chain(repeat(2).take(10)).bytes().count(), 20);
-    }
-}

src/libcoreio/src/lib.rs

@@ -1,51 +0,0 @@
-//! <p id="core_io-show-docblock"></p>
-//! This is just a listing of the functionality available in this crate. See
-//! the [std documentation](https://doc.rust-lang.org/nightly/std/io/index.html)
-//! for a full description of the functionality.
-#![allow(stable_features,unused_features)]
-#![feature(question_mark,const_fn,copy_from_slice,try_from,str_internals,align_offset,slice_internals)]
-#![cfg_attr(any(feature="alloc",feature="collections"),feature(alloc))]
-#![cfg_attr(pattern_guards,feature(bind_by_move_pattern_guards,nll))]
-#![cfg_attr(non_exhaustive,feature(non_exhaustive))]
-#![cfg_attr(unicode,feature(str_char))]
-#![cfg_attr(unicode,feature(unicode))]
-#![no_std]
-
-#[cfg_attr(feature="collections",macro_use)]
-#[cfg_attr(feature="collections",allow(unused_imports))]
-#[cfg(feature="collections")] extern crate alloc as collections;
-#[cfg(feature="alloc")] extern crate alloc;
-#[cfg(rustc_unicode)]
-extern crate rustc_unicode;
-#[cfg(std_unicode)]
-extern crate std_unicode;
-
-#[cfg(not(feature="collections"))]
-pub type ErrorString = &'static str;
-
-// Provide Box::new wrapper
-#[cfg(not(feature="alloc"))]
-struct FakeBox<T>(core::marker::PhantomData<T>);
-#[cfg(not(feature="alloc"))]
-impl<T> FakeBox<T> {
-	fn new(val: T) -> T {
-		val
-	}
-}
-
-// Needed for older compilers, to ignore vec!/format! macros in tests
-#[cfg(not(feature="collections"))]
-#[allow(unused)]
-macro_rules! vec (
-	( $ elem : expr ; $ n : expr ) => { () };
-	( $ ( $ x : expr ) , * ) => { () };
-	( $ ( $ x : expr , ) * ) => { () };
-);
-#[cfg(not(feature="collections"))]
-#[allow(unused)]
-macro_rules! format {
-	( $ ( $ arg : tt ) * ) => { () };
-}
-
-mod io;
-pub use io::*;

src/libdwarf/Cargo.toml

@@ -15,3 +15,4 @@ libc = { path = "../libc" }
 unwind = { path = "../libunwind" }
 compiler_builtins = "0.1.0"
 cfg-if = "0.1.8"
+cslice = "0.3"

src/libdwarf/src/eh.rs

@@ -11,9 +11,12 @@
 #![allow(non_upper_case_globals)]
 #![allow(unused)]
 
-use crate::DwarfReader;
 use core::mem;
 
+use cslice::CSlice;
+
+use crate::DwarfReader;
+
 pub const DW_EH_PE_omit: u8 = 0xFF;
 pub const DW_EH_PE_absptr: u8 = 0x00;
 
@@ -51,10 +54,48 @@ pub enum EHAction {
 
 pub const USING_SJLJ_EXCEPTIONS: bool = cfg!(all(target_os = "ios", target_arch = "arm"));
 
+fn size_of_encoded_value(encoding: u8) -> usize {
+    if encoding == DW_EH_PE_omit {
+        0
+    } else {
+        let encoding = encoding & 0x07;
+        match encoding {
+            DW_EH_PE_absptr => core::mem::size_of::<*const ()>(),
+            DW_EH_PE_udata2 => 2,
+            DW_EH_PE_udata4 => 4,
+            DW_EH_PE_udata8 => 8,
+            _ => unreachable!(),
+        }
+    }
+}
+
+unsafe fn get_ttype_entry(
+    offset: usize,
+    encoding: u8,
+    ttype_base: usize,
+    ttype: *const u8,
+) -> Result<Option<*const u8>, ()> {
+    let i = (offset * size_of_encoded_value(encoding)) as isize;
+    read_encoded_pointer_with_base(
+        &mut DwarfReader::new(ttype.offset(-i)),
+        // the DW_EH_PE_pcrel is a hack.
+        // It seems that the default encoding is absolute, but we have to take reallocation into
+        // account. Unsure if we can fix this in the compiler setting or if this would be affected
+        // by updating the compiler
+        encoding | DW_EH_PE_pcrel,
+        ttype_base,
+    )
+    .map(|v| match v {
+        ttype_base => None,
+        ttype_entry => Some(ttype_entry as *const u8),
+    })
+}
+
 pub unsafe fn find_eh_action(
     lsda: *const u8,
     context: &EHContext<'_>,
     foreign_exception: bool,
+    id: u32,
 ) -> Result<EHAction, ()> {
     if lsda.is_null() {
         return Ok(EHAction::None);
@@ -72,10 +113,17 @@ pub unsafe fn find_eh_action(
     };
 
     let ttype_encoding = reader.read::<u8>();
-    if ttype_encoding != DW_EH_PE_omit {
-        // Rust doesn't analyze exception types, so we don't care about the type table
-        reader.read_uleb128();
-    }
+    // we do care about the type table
+    let ttype_offset = if ttype_encoding != DW_EH_PE_omit {
+        reader.read_uleb128()
+    } else {
+        0
+    };
+    // for rust functions, it seems that there is no type table, so I just put whatever value here.
+    // we should not return an error, otherwise we would abort unwinding and cannot unwind through
+    // rust functions
+    let ttype_base = get_base(ttype_encoding, context).unwrap_or(1);
+    let ttype_table = reader.ptr.offset(ttype_offset as isize);
 
     let call_site_encoding = reader.read::<u8>();
     let call_site_table_length = reader.read_uleb128();
@@ -94,11 +142,49 @@ pub unsafe fn find_eh_action(
                 break;
             }
             if ip < func_start + cs_start + cs_len {
+                // https://github.com/gcc-mirror/gcc/blob/master/libstdc%2B%2B-v3/libsupc%2B%2B/eh_personality.cc#L528
+                let lpad = lpad_base + cs_lpad;
                 if cs_lpad == 0 {
+                    // no cleanups/handler
+                    return Ok(EHAction::None);
+                } else if cs_action == 0 {
+                    return Ok(EHAction::Cleanup(lpad));
+                } else if foreign_exception {
                     return Ok(EHAction::None);
                 } else {
-                    let lpad = lpad_base + cs_lpad;
-                    return Ok(interpret_cs_action(cs_action, lpad, foreign_exception));
+                    let mut saw_cleanup = false;
+                    let mut action_record = action_table.offset(cs_action as isize - 1);
+                    loop {
+                        let mut reader = DwarfReader::new(action_record);
+                        let ar_filter = reader.read_sleb128();
+                        action_record = reader.ptr;
+                        let ar_disp = reader.read_sleb128();
+                        if ar_filter == 0 {
+                            saw_cleanup = true;
+                        } else if ar_filter > 0 {
+                            let catch_type =
+                                get_ttype_entry(ar_filter as usize, ttype_encoding, ttype_base, ttype_table)?;
+                            match catch_type {
+                                Some(clause_ptr) if *(clause_ptr as *const u32) == id => {
+                                    return Ok(EHAction::Catch(lpad));
+                                }
+                                None => return Ok(EHAction::Catch(lpad)),
+                                _ => {}
+                            }
+                        } else if ar_filter < 0 {
+                            // FIXME: how to handle this?
+                            break;
+                        }
+                        if ar_disp == 0 {
+                            break;
+                        }
+                        action_record = action_record.offset((ar_disp as usize) as isize);
+                    }
+                    if saw_cleanup {
+                        return Ok(EHAction::Cleanup(lpad));
+                    } else {
+                        return Ok(EHAction::None);
+                    }
                 }
             }
         }
@@ -106,7 +192,7 @@ pub unsafe fn find_eh_action(
         // So rather than returning EHAction::Terminate, we do this.
         Ok(EHAction::None)
     } else {
-        // SjLj version:
+        // SjLj version: (not yet modified)
         // The "IP" is an index into the call-site table, with two exceptions:
         // -1 means 'no-action', and 0 means 'terminate'.
         match ip as isize {
@@ -146,18 +232,33 @@ fn interpret_cs_action(cs_action: u64, lpad: usize, foreign_exception: bool) ->
 
 #[inline]
 fn round_up(unrounded: usize, align: usize) -> Result<usize, ()> {
-    if align.is_power_of_two() { Ok((unrounded + align - 1) & !(align - 1)) } else { Err(()) }
+    if align.is_power_of_two() {
+        Ok((unrounded + align - 1) & !(align - 1))
+    } else {
+        Err(())
+    }
 }
 
-unsafe fn read_encoded_pointer(
-    reader: &mut DwarfReader,
-    context: &EHContext<'_>,
-    encoding: u8,
-) -> Result<usize, ()> {
+fn get_base(encoding: u8, context: &EHContext<'_>) -> Result<usize, ()> {
+    match encoding & 0x70 {
+        DW_EH_PE_absptr | DW_EH_PE_pcrel | DW_EH_PE_aligned => Ok(0),
+        DW_EH_PE_textrel => Ok((*context.get_text_start)()),
+        DW_EH_PE_datarel => Ok((*context.get_data_start)()),
+        DW_EH_PE_funcrel if context.func_start != 0 => Ok(context.func_start),
+        _ => return Err(()),
+    }
+}
+
+unsafe fn read_encoded_pointer(reader: &mut DwarfReader, context: &EHContext<'_>, encoding: u8) -> Result<usize, ()> {
+    read_encoded_pointer_with_base(reader, encoding, get_base(encoding, context)?)
+}
+
+unsafe fn read_encoded_pointer_with_base(reader: &mut DwarfReader, encoding: u8, base: usize) -> Result<usize, ()> {
     if encoding == DW_EH_PE_omit {
         return Err(());
     }
 
+    let original_ptr = reader.ptr;
     // DW_EH_PE_aligned implies it's an absolute pointer value
     if encoding == DW_EH_PE_aligned {
         reader.ptr = round_up(reader.ptr as usize, mem::size_of::<usize>())? as *const u8;
@@ -177,19 +278,10 @@ unsafe fn read_encoded_pointer(
         _ => return Err(()),
     };
 
-    result += match encoding & 0x70 {
-        DW_EH_PE_absptr => 0,
-        // relative to address of the encoded value, despite the name
-        DW_EH_PE_pcrel => reader.ptr as usize,
-        DW_EH_PE_funcrel => {
-            if context.func_start == 0 {
-                return Err(());
-            }
-            context.func_start
-        }
-        DW_EH_PE_textrel => (*context.get_text_start)(),
-        DW_EH_PE_datarel => (*context.get_data_start)(),
-        _ => return Err(()),
+    result += if (encoding & 0x70) == DW_EH_PE_pcrel {
+        original_ptr as usize
+    } else {
+        base
     };
 
     if encoding & DW_EH_PE_indirect != 0 {

src/libdwarf/src/lib.rs

@@ -26,6 +26,10 @@ impl DwarfReader {
         DwarfReader { ptr }
     }
 
+    pub unsafe fn offset(&mut self, offset: isize) {
+        self.ptr = self.ptr.offset(offset);
+    }
+
     // DWARF streams are packed, so e.g., a u32 would not necessarily be aligned
     // on a 4-byte boundary. This may cause problems on platforms with strict
     // alignment requirements. By wrapping data in a "packed" struct, we are

src/libdyld/Cargo.toml.tpl

@@ -8,3 +8,4 @@ name = "dyld"
 
 [dependencies]
 log = "0.4"
+libcortex_a9 = { path = "@@ZYNQ_RS@@/libcortex_a9" }

src/libdyld/src/elf.rs

@@ -1451,8 +1451,7 @@ pub const R_AARCH64_TLSDESC_CALL: usize = 569;
 pub const R_AARCH64_TLSLE_LDST128_TPREL_LO12: usize = 570;
 pub const R_AARCH64_TLSLE_LDST128_TPREL_LO12_NC: usize = 571;
 pub const R_AARCH64_TLSLD_LDST128_DTPREL_LO12: usize = 572;
-pub const R_AARCH64_TLSLD_LDST128_DTPREL_LO12_NC: usize =
-    573;
+pub const R_AARCH64_TLSLD_LDST128_DTPREL_LO12_NC: usize = 573;
 pub const R_AARCH64_COPY: usize = 1024;
 pub const R_AARCH64_GLOB_DAT: usize = 1025;
 pub const R_AARCH64_JUMP_SLOT: usize = 1026;
@@ -2267,7 +2266,9 @@ pub struct Elf32_Ehdr {
     pub e_shstrndx: Elf32_Half,
 }
 impl Clone for Elf32_Ehdr {
-    fn clone(&self) -> Self { *self }
+    fn clone(&self) -> Self {
+        *self
+    }
 }
 #[repr(C)]
 #[derive(Debug, Copy)]
@@ -2288,7 +2289,9 @@ pub struct Elf64_Ehdr {
     pub e_shstrndx: Elf64_Half,
 }
 impl Clone for Elf64_Ehdr {
-    fn clone(&self) -> Self { *self }
+    fn clone(&self) -> Self {
+        *self
+    }
 }
 #[repr(C)]
 #[derive(Debug, Copy)]
@@ -2305,7 +2308,9 @@ pub struct Elf32_Shdr {
     pub sh_entsize: Elf32_Word,
 }
 impl Clone for Elf32_Shdr {
-    fn clone(&self) -> Self { *self }
+    fn clone(&self) -> Self {
+        *self
+    }
 }
 #[repr(C)]
 #[derive(Debug, Copy)]
@@ -2322,7 +2327,9 @@ pub struct Elf64_Shdr {
     pub sh_entsize: Elf64_Xword,
 }
 impl Clone for Elf64_Shdr {
-    fn clone(&self) -> Self { *self }
+    fn clone(&self) -> Self {
+        *self
+    }
 }
 #[repr(C)]
 #[derive(Debug, Copy)]
@@ -2335,7 +2342,9 @@ pub struct Elf32_Sym {
     pub st_shndx: Elf32_Section,
 }
 impl Clone for Elf32_Sym {
-    fn clone(&self) -> Self { *self }
+    fn clone(&self) -> Self {
+        *self
+    }
 }
 #[repr(C)]
 #[derive(Debug, Copy)]
@@ -2348,7 +2357,9 @@ pub struct Elf64_Sym {
     pub st_size: Elf64_Xword,
 }
 impl Clone for Elf64_Sym {
-    fn clone(&self) -> Self { *self }
+    fn clone(&self) -> Self {
+        *self
+    }
 }
 #[repr(C)]
 #[derive(Debug, Copy)]
@@ -2357,7 +2368,9 @@ pub struct Elf32_Syminfo {
     pub si_flags: Elf32_Half,
 }
 impl Clone for Elf32_Syminfo {
-    fn clone(&self) -> Self { *self }
+    fn clone(&self) -> Self {
+        *self
+    }
 }
 #[repr(C)]
 #[derive(Debug, Copy)]
@@ -2366,7 +2379,9 @@ pub struct Elf64_Syminfo {
     pub si_flags: Elf64_Half,
 }
 impl Clone for Elf64_Syminfo {
-    fn clone(&self) -> Self { *self }
+    fn clone(&self) -> Self {
+        *self
+    }
 }
 #[repr(C)]
 #[derive(Debug, Copy)]
@@ -2375,7 +2390,9 @@ pub struct Elf32_Rel {
     pub r_info: Elf32_Word,
 }
 impl Clone for Elf32_Rel {
-    fn clone(&self) -> Self { *self }
+    fn clone(&self) -> Self {
+        *self
+    }
 }
 #[repr(C)]
 #[derive(Debug, Copy)]
@@ -2384,7 +2401,9 @@ pub struct Elf64_Rel {
     pub r_info: Elf64_Xword,
 }
 impl Clone for Elf64_Rel {
-    fn clone(&self) -> Self { *self }
+    fn clone(&self) -> Self {
+        *self
+    }
 }
 #[repr(C)]
 #[derive(Debug, Copy)]
@@ -2394,7 +2413,9 @@ pub struct Elf32_Rela {
     pub r_addend: Elf32_Sword,
 }
 impl Clone for Elf32_Rela {
-    fn clone(&self) -> Self { *self }
+    fn clone(&self) -> Self {
+        *self
+    }
 }
 #[repr(C)]
 #[derive(Debug, Copy)]
@@ -2404,7 +2425,9 @@ pub struct Elf64_Rela {
     pub r_addend: Elf64_Sxword,
 }
 impl Clone for Elf64_Rela {
-    fn clone(&self) -> Self { *self }
+    fn clone(&self) -> Self {
+        *self
+    }
 }
 #[repr(C)]
 #[derive(Debug, Copy)]
@@ -2419,7 +2442,9 @@ pub struct Elf32_Phdr {
     pub p_align: Elf32_Word,
 }
 impl Clone for Elf32_Phdr {
-    fn clone(&self) -> Self { *self }
+    fn clone(&self) -> Self {
+        *self
+    }
 }
 #[repr(C)]
 #[derive(Debug, Copy)]
@@ -2434,7 +2459,9 @@ pub struct Elf64_Phdr {
     pub p_align: Elf64_Xword,
 }
 impl Clone for Elf64_Phdr {
-    fn clone(&self) -> Self { *self }
+    fn clone(&self) -> Self {
+        *self
+    }
 }
 #[repr(C)]
 #[derive(Copy)]
@@ -2449,10 +2476,14 @@ pub union Elf32_Dyn__bindgen_ty_1 {
     pub d_ptr: Elf32_Addr,
 }
 impl Clone for Elf32_Dyn__bindgen_ty_1 {
-    fn clone(&self) -> Self { *self }
+    fn clone(&self) -> Self {
+        *self
+    }
 }
 impl Clone for Elf32_Dyn {
-    fn clone(&self) -> Self { *self }
+    fn clone(&self) -> Self {
+        *self
+    }
 }
 #[repr(C)]
 #[derive(Copy)]
@@ -2467,10 +2498,14 @@ pub union Elf64_Dyn__bindgen_ty_1 {
     pub d_ptr: Elf64_Addr,
 }
 impl Clone for Elf64_Dyn__bindgen_ty_1 {
-    fn clone(&self) -> Self { *self }
+    fn clone(&self) -> Self {
+        *self
+    }
 }
 impl Clone for Elf64_Dyn {
-    fn clone(&self) -> Self { *self }
+    fn clone(&self) -> Self {
+        *self
+    }
 }
 #[repr(C)]
 #[derive(Debug, Copy)]
@@ -2484,7 +2519,9 @@ pub struct Elf32_Verdef {
     pub vd_next: Elf32_Word,
 }
 impl Clone for Elf32_Verdef {
-    fn clone(&self) -> Self { *self }
+    fn clone(&self) -> Self {
+        *self
+    }
 }
 #[repr(C)]
 #[derive(Debug, Copy)]
@@ -2498,7 +2535,9 @@ pub struct Elf64_Verdef {
     pub vd_next: Elf64_Word,
 }
 impl Clone for Elf64_Verdef {
-    fn clone(&self) -> Self { *self }
+    fn clone(&self) -> Self {
+        *self
+    }
 }
 #[repr(C)]
 #[derive(Debug, Copy)]
@@ -2507,7 +2546,9 @@ pub struct Elf32_Verdaux {
     pub vda_next: Elf32_Word,
 }
 impl Clone for Elf32_Verdaux {
-    fn clone(&self) -> Self { *self }
+    fn clone(&self) -> Self {
+        *self
+    }
 }
 #[repr(C)]
 #[derive(Debug, Copy)]
@@ -2516,7 +2557,9 @@ pub struct Elf64_Verdaux {
     pub vda_next: Elf64_Word,
 }
 impl Clone for Elf64_Verdaux {
-    fn clone(&self) -> Self { *self }
+    fn clone(&self) -> Self {
+        *self
+    }
 }
 #[repr(C)]
 #[derive(Debug, Copy)]
@@ -2528,7 +2571,9 @@ pub struct Elf32_Verneed {
     pub vn_next: Elf32_Word,
 }
 impl Clone for Elf32_Verneed {
-    fn clone(&self) -> Self { *self }
+    fn clone(&self) -> Self {
+        *self
+    }
 }
 #[repr(C)]
 #[derive(Debug, Copy)]
@@ -2540,7 +2585,9 @@ pub struct Elf64_Verneed {
     pub vn_next: Elf64_Word,
 }
 impl Clone for Elf64_Verneed {
-    fn clone(&self) -> Self { *self }
+    fn clone(&self) -> Self {
+        *self
+    }
 }
 #[repr(C)]
 #[derive(Debug, Copy)]
@@ -2552,7 +2599,9 @@ pub struct Elf32_Vernaux {
     pub vna_next: Elf32_Word,
 }
 impl Clone for Elf32_Vernaux {
-    fn clone(&self) -> Self { *self }
+    fn clone(&self) -> Self {
+        *self
+    }
 }
 #[repr(C)]
 #[derive(Debug, Copy)]
@@ -2564,7 +2613,9 @@ pub struct Elf64_Vernaux {
     pub vna_next: Elf64_Word,
 }
 impl Clone for Elf64_Vernaux {
-    fn clone(&self) -> Self { *self }
+    fn clone(&self) -> Self {
+        *self
+    }
 }
 #[repr(C)]
 #[derive(Copy)]
@@ -2578,10 +2629,14 @@ pub union Elf32_auxv_t__bindgen_ty_1 {
     pub a_val: u32,
 }
 impl Clone for Elf32_auxv_t__bindgen_ty_1 {
-    fn clone(&self) -> Self { *self }
+    fn clone(&self) -> Self {
+        *self
+    }
 }
 impl Clone for Elf32_auxv_t {
-    fn clone(&self) -> Self { *self }
+    fn clone(&self) -> Self {
+        *self
+    }
 }
 #[repr(C)]
 #[derive(Copy)]
@@ -2595,10 +2650,14 @@ pub union Elf64_auxv_t__bindgen_ty_1 {
     pub a_val: u64,
 }
 impl Clone for Elf64_auxv_t__bindgen_ty_1 {
-    fn clone(&self) -> Self { *self }
+    fn clone(&self) -> Self {
+        *self
+    }
 }
 impl Clone for Elf64_auxv_t {
-    fn clone(&self) -> Self { *self }
+    fn clone(&self) -> Self {
+        *self
+    }
 }
 #[repr(C)]
 #[derive(Debug, Copy)]
@@ -2608,7 +2667,9 @@ pub struct Elf32_Nhdr {
     pub n_type: Elf32_Word,
 }
 impl Clone for Elf32_Nhdr {
-    fn clone(&self) -> Self { *self }
+    fn clone(&self) -> Self {
+        *self
+    }
 }
 #[repr(C)]
 #[derive(Debug, Copy)]
@@ -2618,7 +2679,9 @@ pub struct Elf64_Nhdr {
     pub n_type: Elf64_Word,
 }
 impl Clone for Elf64_Nhdr {
-    fn clone(&self) -> Self { *self }
+    fn clone(&self) -> Self {
+        *self
+    }
 }
 #[repr(C)]
 #[derive(Debug, Copy)]
@@ -2630,7 +2693,9 @@ pub struct Elf32_Move {
     pub m_stride: Elf32_Half,
 }
 impl Clone for Elf32_Move {
-    fn clone(&self) -> Self { *self }
+    fn clone(&self) -> Self {
+        *self
+    }
 }
 #[repr(C)]
 #[derive(Debug, Copy)]
@@ -2642,7 +2707,9 @@ pub struct Elf64_Move {
     pub m_stride: Elf64_Half,
 }
 impl Clone for Elf64_Move {
-    fn clone(&self) -> Self { *self }
+    fn clone(&self) -> Self {
+        *self
+    }
 }
 #[repr(C)]
 #[derive(Copy)]
@@ -2657,7 +2724,9 @@ pub struct Elf32_gptab__bindgen_ty_1 {
     pub gt_unused: Elf32_Word,
 }
 impl Clone for Elf32_gptab__bindgen_ty_1 {
-    fn clone(&self) -> Self { *self }
+    fn clone(&self) -> Self {
+        *self
+    }
 }
 #[repr(C)]
 #[derive(Debug, Copy)]
@@ -2666,10 +2735,14 @@ pub struct Elf32_gptab__bindgen_ty_2 {
     pub gt_bytes: Elf32_Word,
 }
 impl Clone for Elf32_gptab__bindgen_ty_2 {
-    fn clone(&self) -> Self { *self }
+    fn clone(&self) -> Self {
+        *self
+    }
 }
 impl Clone for Elf32_gptab {
-    fn clone(&self) -> Self { *self }
+    fn clone(&self) -> Self {
+        *self
+    }
 }
 #[repr(C)]
 #[derive(Debug, Copy)]
@@ -2679,7 +2752,9 @@ pub struct Elf32_RegInfo {
     pub ri_gp_value: Elf32_Sword,
 }
 impl Clone for Elf32_RegInfo {
-    fn clone(&self) -> Self { *self }
+    fn clone(&self) -> Self {
+        *self
+    }
 }
 #[repr(C)]
 #[derive(Debug, Copy)]
@@ -2690,7 +2765,9 @@ pub struct Elf_Options {
     pub info: Elf32_Word,
 }
 impl Clone for Elf_Options {
-    fn clone(&self) -> Self { *self }
+    fn clone(&self) -> Self {
+        *self
+    }
 }
 #[repr(C)]
 #[derive(Debug, Copy)]
@@ -2699,7 +2776,9 @@ pub struct Elf_Options_Hw {
     pub hwp_flags2: Elf32_Word,
 }
 impl Clone for Elf_Options_Hw {
-    fn clone(&self) -> Self { *self }
+    fn clone(&self) -> Self {
+        *self
+    }
 }
 #[repr(C)]
 #[derive(Debug, Copy)]
@@ -2711,7 +2790,9 @@ pub struct Elf32_Lib {
     pub l_flags: Elf32_Word,
 }
 impl Clone for Elf32_Lib {
-    fn clone(&self) -> Self { *self }
+    fn clone(&self) -> Self {
+        *self
+    }
 }
 #[repr(C)]
 #[derive(Debug, Copy)]
@@ -2723,14 +2804,31 @@ pub struct Elf64_Lib {
     pub l_flags: Elf64_Word,
 }
 impl Clone for Elf64_Lib {
-    fn clone(&self) -> Self { *self }
+    fn clone(&self) -> Self {
+        *self
+    }
 }
 pub type Elf32_Conflict = Elf32_Addr;
+#[repr(C)]
+#[derive(Clone, Copy)]
+pub struct EXIDX_Entry(u32, u32);
 
-pub fn ELF32_R_SYM(info: Elf32_Word) -> Elf32_Word { info >> 8 }
-pub fn ELF32_R_TYPE(info: Elf32_Word) -> u8 { info as u8 }
-pub fn ELF32_R_INFO(sym: Elf32_Word, ty: u8) -> Elf32_Word { sym << 8 | ty as Elf32_Word }
+pub fn ELF32_R_SYM(info: Elf32_Word) -> Elf32_Word {
+    info >> 8
+}
+pub fn ELF32_R_TYPE(info: Elf32_Word) -> u8 {
+    info as u8
+}
+pub fn ELF32_R_INFO(sym: Elf32_Word, ty: u8) -> Elf32_Word {
+    sym << 8 | ty as Elf32_Word
+}
 
-pub fn ELF32_ST_BIND(info: u8) -> u8 { info >> 4 }
-pub fn ELF32_ST_TYPE(info: u8) -> u8 { info & 0xf }
-pub fn ELF32_ST_INFO(bind: u8, ty: u8) -> u8 { (bind << 4) | (ty & 0xf) }
+pub fn ELF32_ST_BIND(info: u8) -> u8 {
+    info >> 4
+}
+pub fn ELF32_ST_TYPE(info: u8) -> u8 {
+    info & 0xf
+}
+pub fn ELF32_ST_INFO(bind: u8, ty: u8) -> u8 {
+    (bind << 4) | (ty & 0xf)
+}

src/libdyld/src/file.rs

@@ -1,8 +1,8 @@
-use core::{mem, ptr, ops::{Deref, Range}};
-use super::{
-    Arch,
-    elf::*,
-};
+use core::{mem,
+           ops::{Deref, Range},
+           ptr};
+
+use super::{elf::*, Arch};
 
 fn read_unaligned<T: Copy>(data: &[u8], offset: usize) -> Option<T> {
     if data.len() < offset + mem::size_of::<T>() {
@@ -31,14 +31,40 @@ impl<'a> File<'a> {
 
     pub fn arch(&self) -> Option<Arch> {
         const IDENT_OPENRISC: [u8; EI_NIDENT] = [
-            ELFMAG0,    ELFMAG1,     ELFMAG2,    ELFMAG3,
-            ELFCLASS32, ELFDATA2MSB, EV_CURRENT, ELFOSABI_NONE,
-            /* ABI version */ 0, /* padding */ 0, 0, 0, 0, 0, 0, 0
+            ELFMAG0,
+            ELFMAG1,
+            ELFMAG2,
+            ELFMAG3,
+            ELFCLASS32,
+            ELFDATA2MSB,
+            EV_CURRENT,
+            ELFOSABI_NONE,
+            /* ABI version */ 0,
+            /* padding */ 0,
+            0,
+            0,
+            0,
+            0,
+            0,
+            0,
         ];
         const IDENT_ARM: [u8; EI_NIDENT] = [
-            ELFMAG0,    ELFMAG1,     ELFMAG2,    ELFMAG3,
-            ELFCLASS32, ELFDATA2LSB, EV_CURRENT, ELFOSABI_NONE,
-            /* ABI version */ 0, /* padding */ 0, 0, 0, 0, 0, 0, 0
+            ELFMAG0,
+            ELFMAG1,
+            ELFMAG2,
+            ELFMAG3,
+            ELFCLASS32,
+            ELFDATA2LSB,
+            EV_CURRENT,
+            ELFOSABI_NONE,
+            /* ABI version */ 0,
+            /* padding */ 0,
+            0,
+            0,
+            0,
+            0,
+            0,
+            0,
         ];
 
         match (self.ehdr.e_ident, self.ehdr.e_machine) {
@@ -48,14 +74,20 @@ impl<'a> File<'a> {
         }
     }
 
-    pub fn program_headers<'b>(&'b self) -> impl Iterator<Item = Option<Elf32_Phdr>> + 'b
-    {
+    pub fn program_headers<'b>(&'b self) -> impl Iterator<Item = Option<Elf32_Phdr>> + 'b {
         (0..self.ehdr.e_phnum).map(move |i| {
             let phdr_off = self.ehdr.e_phoff as usize + mem::size_of::<Elf32_Phdr>() * i as usize;
             self.read_unaligned::<Elf32_Phdr>(phdr_off)
         })
     }
 
+    pub fn section_headers<'b>(&'b self) -> impl Iterator<Item = Option<Elf32_Shdr>> + 'b {
+        (0..self.ehdr.e_shnum).map(move |i| {
+            let shdr_off = self.ehdr.e_shoff as usize + mem::size_of::<Elf32_Shdr>() * i as usize;
+            self.read_unaligned::<Elf32_Shdr>(shdr_off)
+        })
+    }
+
     pub fn dyn_header_vaddr(&self) -> Option<Range<usize>> {
         self.program_headers()
             .filter_map(|phdr| phdr)

src/libdyld/src/image.rs

@@ -1,13 +1,9 @@
-use core::{
-    ops::{Deref, DerefMut, Range},
-    mem,
-    slice,
-};
-use alloc::alloc::{alloc_zeroed, dealloc, Layout, LayoutErr};
-use super::{
-    elf::*,
-    Error,
-};
+use alloc::alloc::{alloc_zeroed, dealloc, Layout, LayoutError};
+use core::{mem,
+           ops::{Deref, DerefMut, Range},
+           slice};
+
+use super::{elf::*, Error};
 
 pub struct DynamicSection {
     pub strtab: Range<usize>,
@@ -27,24 +23,19 @@ pub struct Image {
 }
 
 impl Image {
-    pub fn new(size: usize, align: usize) -> Result<Self, LayoutErr> {
+    pub fn new(size: usize, align: usize) -> Result<Self, LayoutError> {
         let layout = Layout::from_size_align(size, align)?;
         let data = unsafe {
             let ptr = alloc_zeroed(layout);
             slice::from_raw_parts_mut(ptr, size)
         };
 
-        Ok(Image {
-            layout,
-            data,
-        })
+        Ok(Image { layout, data })
     }
 
     /// assumes that self.data is properly aligned
     pub(crate) fn get_ref<T>(&self, offset: usize) -> Option<&T>
-    where
-        T: Copy,
-    {
+    where T: Copy {
         if self.data.len() < offset + mem::size_of::<T>() {
             None
         } else if (self.data.as_ptr() as usize + offset) & (mem::align_of::<T>() - 1) != 0 {
@@ -66,55 +57,53 @@ impl Image {
         unsafe { slice::from_raw_parts(ptr, len) }
     }
 
-    fn dyn_headers<'a>(&'a self, range: Range<usize>) ->
-        impl Iterator<Item = &'a Elf32_Dyn> + 'a
-    {
+    fn dyn_headers<'a>(&'a self, range: Range<usize>) -> impl Iterator<Item = &'a Elf32_Dyn> + 'a {
         range
             .step_by(mem::size_of::<Elf32_Dyn>())
-            .filter_map(move |offset| {
-                self.get_ref::<Elf32_Dyn>(offset)
-            })
+            .filter_map(move |offset| self.get_ref::<Elf32_Dyn>(offset))
             .take_while(|d| unsafe { d.d_un.d_val } as i32 != DT_NULL)
     }
 
     pub fn dyn_section(&self, range: Range<usize>) -> Result<DynamicSection, Error> {
         let (mut strtab_off, mut strtab_sz) = (0, 0);
-        let (mut rel_off,    mut rel_sz)    = (0, 0);
-        let (mut rela_off,   mut rela_sz)   = (0, 0);
+        let (mut rel_off, mut rel_sz) = (0, 0);
+        let (mut rela_off, mut rela_sz) = (0, 0);
         let (mut pltrel_off, mut pltrel_sz) = (0, 0);
-        let (mut hash_off,   mut hash_sz)   = (0, 0);
+        let (mut hash_off, mut hash_sz) = (0, 0);
         let mut symtab_off = 0;
-        let mut sym_ent    = 0;
-        let mut rel_ent    = 0;
-        let mut rela_ent   = 0;
-        let mut nbucket    = 0;
-        let mut nchain     = 0;
+        let mut sym_ent = 0;
+        let mut rel_ent = 0;
+        let mut rela_ent = 0;
+        let mut nbucket = 0;
+        let mut nchain = 0;
 
         for dyn_header in self.dyn_headers(range) {
             let val = unsafe { dyn_header.d_un.d_val } as usize;
             match dyn_header.d_tag {
-                DT_NULL     => break,
-                DT_STRTAB   => strtab_off = val,
-                DT_STRSZ    => strtab_sz  = val,
-                DT_SYMTAB   => symtab_off = val,
-                DT_SYMENT   => sym_ent    = val,
-                DT_REL      => rel_off    = val,
-                DT_RELSZ    => rel_sz     = val,
-                DT_RELENT   => rel_ent    = val,
-                DT_RELA     => rela_off   = val,
-                DT_RELASZ   => rela_sz    = val,
-                DT_RELAENT  => rela_ent   = val,
-                DT_JMPREL   => pltrel_off = val,
-                DT_PLTRELSZ => pltrel_sz  = val,
-                DT_HASH     => {
-                    nbucket  = *self.get_ref::<Elf32_Word>(val + 0)
+                DT_NULL => break,
+                DT_STRTAB => strtab_off = val,
+                DT_STRSZ => strtab_sz = val,
+                DT_SYMTAB => symtab_off = val,
+                DT_SYMENT => sym_ent = val,
+                DT_REL => rel_off = val,
+                DT_RELSZ => rel_sz = val,
+                DT_RELENT => rel_ent = val,
+                DT_RELA => rela_off = val,
+                DT_RELASZ => rela_sz = val,
+                DT_RELAENT => rela_ent = val,
+                DT_JMPREL => pltrel_off = val,
+                DT_PLTRELSZ => pltrel_sz = val,
+                DT_HASH => {
+                    nbucket = *self
+                        .get_ref::<Elf32_Word>(val + 0)
                         .ok_or("cannot read hash bucket count")? as usize;
-                    nchain   = *self.get_ref::<Elf32_Word>(val + 4)
+                    nchain = *self
+                        .get_ref::<Elf32_Word>(val + 4)
                         .ok_or("cannot read hash chain count")? as usize;
                     hash_off = val + 8;
-                    hash_sz  = (nbucket + nchain) * mem::size_of::<Elf32_Word>();
+                    hash_sz = (nbucket + nchain) * mem::size_of::<Elf32_Word>();
                 }
-                _ => ()
+                _ => (),
             }
         }
 
@@ -123,28 +112,28 @@ impl Image {
         let symtab_sz = nchain * mem::size_of::<Elf32_Sym>();
 
         if strtab_off + strtab_sz > self.data.len() {
-            return Err("invalid strtab offset/size")?
+            return Err("invalid strtab offset/size")?;
         }
         if symtab_off + symtab_sz > self.data.len() {
-            return Err("invalid symtab offset/size")?
+            return Err("invalid symtab offset/size")?;
         }
         if sym_ent != mem::size_of::<Elf32_Sym>() {
-            return Err("incorrect symbol entry size")?
+            return Err("incorrect symbol entry size")?;
         }
         if rel_off + rel_sz > self.data.len() {
-            return Err("invalid rel offset/size")?
+            return Err("invalid rel offset/size")?;
         }
         if rel_ent != 0 && rel_ent != mem::size_of::<Elf32_Rel>() {
-            return Err("incorrect relocation entry size")?
+            return Err("incorrect relocation entry size")?;
         }
         if rela_off + rela_sz > self.data.len() {
-            return Err("invalid rela offset/size")?
+            return Err("invalid rela offset/size")?;
         }
         if rela_ent != 0 && rela_ent != mem::size_of::<Elf32_Rela>() {
-            return Err("incorrect relocation entry size")?
+            return Err("incorrect relocation entry size")?;
         }
         if pltrel_off + pltrel_sz > self.data.len() {
-            return Err("invalid pltrel offset/size")?
+            return Err("invalid pltrel offset/size")?;
         }
 
         Ok(DynamicSection {
@@ -165,7 +154,7 @@ impl Image {
 
     pub fn write(&self, offset: usize, value: Elf32_Word) -> Result<(), Error> {
         if offset + mem::size_of::<Elf32_Addr>() > self.data.len() {
-            return Err("relocation out of image bounds")?
+            return Err("relocation out of image bounds")?;
         }
 
         let ptr = (self.data.as_ptr() as usize + offset) as *mut Elf32_Addr;

src/libdyld/src/lib.rs

@@ -1,12 +1,14 @@
 #![no_std]
 
 extern crate alloc;
+extern crate libcortex_a9;
 extern crate log;
 
-use core::{convert, fmt, str};
 use alloc::string::String;
-use log::{debug, trace};
+use core::{convert, fmt, ops::Range, str};
+
 use elf::*;
+use log::{debug, trace};
 
 pub mod elf;
 mod file;
@@ -20,11 +22,10 @@ pub enum Arch {
     OpenRisc,
 }
 
-
 #[derive(Debug)]
 pub enum Error {
     Parsing(&'static str),
-    Lookup(String)
+    Lookup(String),
 }
 
 impl convert::From<&'static str> for Error {
@@ -36,10 +37,8 @@ impl convert::From<&'static str> for Error {
 impl fmt::Display for Error {
     fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
         match self {
-            &Error::Parsing(desc) =>
-                write!(f, "parse error: {}", desc),
-            &Error::Lookup(ref sym) =>
-                write!(f, "symbol lookup error: {}", sym),
+            &Error::Parsing(desc) => write!(f, "parse error: {}", desc),
+            &Error::Lookup(ref sym) => write!(f, "symbol lookup error: {}", sym),
         }
     }
 }
@@ -57,43 +56,11 @@ fn elf_hash(name: &[u8]) -> u32 {
     h
 }
 
-// linker symbols
-extern "C" {
-    #[no_mangle]
-    static __text_start: u32;
-    #[no_mangle]
-    static __text_end: u32;
-    #[no_mangle]
-    static __exidx_start: u32;
-    #[no_mangle]
-    static __exidx_end: u32;
-}
-
-static mut KERNEL_EXIDX_START: u32 = 0;
-static mut KERNEL_EXIDX_END: u32 = 0;
-
-#[no_mangle]
-extern fn dl_unwind_find_exidx(pc: u32, len_ptr: *mut u32) -> u32 {
-    let length: u32;
-    let start: u32;
-    unsafe {
-        if (&__text_start as *const u32 as u32) <= pc && pc < (&__text_end as *const u32 as u32) {
-            length = (&__exidx_end - &__exidx_start) as u32;
-            start = &__exidx_start as *const u32 as u32;
-        } else {
-            // make sure that the kernel is loaded
-            assert_ne!(KERNEL_EXIDX_START, 0);
-            length = (KERNEL_EXIDX_END - KERNEL_EXIDX_START) / core::mem::size_of::<u32>() as u32;
-            start = KERNEL_EXIDX_START;
-        }
-        *len_ptr = length;
-    }
-    start
-}
-
 pub struct Library {
     pub image: Image,
+    pub arch: Arch,
     dyn_section: DynamicSection,
+    exidx: Range<usize>,
 }
 
 impl Library {
@@ -134,20 +101,22 @@ impl Library {
         let mut index = self.hash_bucket()[hash as usize % self.hash_bucket().len()] as usize;
 
         loop {
-            if index == STN_UNDEF { return None }
+            if index == STN_UNDEF {
+                return None;
+            }
 
             let sym = &self.symtab()[index];
             let sym_name_off = sym.st_name as usize;
             match self.strtab().get(sym_name_off..sym_name_off + name.len()) {
                 Some(sym_name) if sym_name == name => {
                     if ELF32_ST_BIND(sym.st_info) & STB_GLOBAL == 0 {
-                        return None
+                        return None;
                     }
 
                     match sym.st_shndx {
                         SHN_UNDEF => return None,
                         SHN_ABS => return Some(self.image.ptr() as u32 + sym.st_value),
-                        _ => return Some(self.image.ptr() as u32 + sym.st_value)
+                        _ => return Some(self.image.ptr() as u32 + sym.st_value),
                     }
                 }
                 _ => (),
@@ -158,85 +127,118 @@ impl Library {
     }
 
     pub fn name_starting_at(&self, offset: usize) -> Result<&[u8], Error> {
-        let size = self.strtab().iter().skip(offset).position(|&x| x == 0)
-                              .ok_or("symbol in symbol table not null-terminated")?;
-        Ok(self.strtab().get(offset..offset + size)
-           .ok_or("cannot read symbol name")?)
+        let size = self
+            .strtab()
+            .iter()
+            .skip(offset)
+            .position(|&x| x == 0)
+            .ok_or("symbol in symbol table not null-terminated")?;
+        Ok(self
+            .strtab()
+            .get(offset..offset + size)
+            .ok_or("cannot read symbol name")?)
+    }
+
+    /// Rebind Rela by `name` to a new `addr`
+    pub fn rebind(&self, name: &[u8], addr: *const ()) -> Result<(), Error> {
+        reloc::rebind(self.arch, self, name, addr as Elf32_Word)
+    }
+
+    pub fn exidx(&self) -> &[EXIDX_Entry] {
+        self.image.get_ref_slice_unchecked(&self.exidx)
     }
 }
 
-pub fn load(
-    data: &[u8],
-    resolve: &dyn Fn(&[u8]) -> Option<Elf32_Word>
-) -> Result<Library, Error> {
+pub fn load(data: &[u8], resolve: &dyn Fn(&[u8]) -> Option<Elf32_Word>) -> Result<Library, Error> {
     // validate ELF file
-    let file = file::File::new(data)
-        .ok_or("cannot read ELF header")?;
+    let file = file::File::new(data).ok_or("cannot read ELF header")?;
     if file.ehdr.e_type != ET_DYN {
-        return Err("not a shared library")?
+        return Err("not a shared library")?;
     }
-    let arch = file.arch()
-        .ok_or("not for a supported architecture")?;
+    let arch = file.arch().ok_or("not for a supported architecture")?;
 
     // prepare target memory
-    let image_size = file.program_headers()
+    let image_size = file
+        .program_headers()
         .filter_map(|phdr| phdr.map(|phdr| phdr.p_vaddr + phdr.p_memsz))
         .max()
         .unwrap_or(0) as usize;
-    let image_align = file.program_headers()
-        .filter_map(|phdr| phdr.and_then(|phdr| {
-            if phdr.p_type == PT_LOAD {
-                Some(phdr.p_align)
-            } else {
-                None
-            }
-        }))
+    let image_align = file
+        .program_headers()
+        .filter_map(|phdr| {
+            phdr.and_then(|phdr| {
+                if phdr.p_type == PT_LOAD {
+                    Some(phdr.p_align)
+                } else {
+                    None
+                }
+            })
+        })
         .max()
         .unwrap_or(4) as usize;
     // 1 image for all segments
-    let mut image = image::Image::new(image_size, image_align)
-        .map_err(|_| "cannot allocate target image")?;
-    debug!("ELF target: {} bytes, align to {:X}, allocated at {:08X}", image_size, image_align, image.ptr() as usize);
+    let mut image = image::Image::new(image_size, image_align).map_err(|_| "cannot allocate target image")?;
+    debug!(
+        "ELF target: {} bytes, align to {:X}, allocated at {:08X}",
+        image_size,
+        image_align,
+        image.ptr() as usize
+    );
 
     // LOAD
     for phdr in file.program_headers() {
         let phdr = phdr.ok_or("cannot read program header")?;
-        trace!("Program header: {:08X}+{:08X} to {:08X}",
-              phdr.p_offset, phdr.p_filesz,
-              image.ptr() as u32
+        trace!(
+            "Program header: {:08X}+{:08X} to {:08X}",
+            phdr.p_offset,
+            phdr.p_filesz,
+            image.ptr() as u32
         );
         let file_range = phdr.p_offset as usize..(phdr.p_offset + phdr.p_filesz) as usize;
         match phdr.p_type {
             PT_LOAD => {
-                let src = file.get(file_range)
+                let src = file
+                    .get(file_range)
                     .ok_or("program header requests an out of bounds load (in file)")?;
-                let dst = image.get_mut(phdr.p_vaddr as usize..
-                                        (phdr.p_vaddr + phdr.p_filesz) as usize)
+                let dst = image
+                    .get_mut(phdr.p_vaddr as usize..(phdr.p_vaddr + phdr.p_filesz) as usize)
                     .ok_or("program header requests an out of bounds load (in target)")?;
                 dst.copy_from_slice(src);
             }
-            PT_ARM_EXIDX => {
-                let range = image.get(phdr.p_vaddr as usize..
-                    (phdr.p_vaddr + phdr.p_filesz) as usize)
-                    .ok_or("program header requests and out of bounds load (in target)")?;
-                unsafe {
-                    KERNEL_EXIDX_START = range.as_ptr() as u32;
-                    KERNEL_EXIDX_END = range.as_ptr().add(range.len()) as u32;
-                }
+            _ => {}
+        }
+    }
+
+    let mut exidx = None;
+    // Obtain EXIDX
+    for shdr in file.section_headers() {
+        let shdr = shdr.ok_or("cannot read section header")?;
+        match shdr.sh_type as usize {
+            SHT_ARM_EXIDX => {
+                let range = shdr.sh_addr as usize..(shdr.sh_addr + shdr.sh_size) as usize;
+                let _ = image
+                    .get(range.clone())
+                    .ok_or("section header specifies EXIDX outside of image (in target)")?;
+                exidx = Some(range);
             }
             _ => {}
         }
     }
 
     // relocate DYNAMIC
-    let dyn_range = file.dyn_header_vaddr()
-        .ok_or("cannot find a dynamic header")?;
+    let dyn_range = file.dyn_header_vaddr().ok_or("cannot find a dynamic header")?;
     let dyn_section = image.dyn_section(dyn_range.clone())?;
-    debug!("Relocating {} rela, {} rel, {} pltrel",
-           dyn_section.rela.len(), dyn_section.rel.len(), dyn_section.pltrel.len());
+    debug!(
+        "Relocating {} rela, {} rel, {} pltrel",
+        dyn_section.rela.len(),
+        dyn_section.rel.len(),
+        dyn_section.pltrel.len()
+    );
     let lib = Library {
+        arch,
         image,
-        dyn_section
+        dyn_section,
+        exidx: exidx.ok_or("no EXIDX section")?,
     };
 
     for rela in lib.rela() {

src/libdyld/src/reloc.rs

@@ -1,12 +1,10 @@
 use alloc::string::String;
+
+use libcortex_a9::{asm::{dsb, isb},
+                   cache::{bpiall, dcci_slice, iciallu}};
 use log::trace;
-use super::{
-    Arch,
-    elf::*,
-    Error,
-    image::Image,
-    Library,
-};
+
+use super::{elf::*, image::Image, Arch, Error, Library};
 
 pub trait Relocatable {
     fn offset(&self) -> usize;
@@ -55,29 +53,25 @@ impl Relocatable for Elf32_Rela {
 enum RelType {
     None,
     Relative,
-    Lookup,
+    LookupAbs,
+    LookupRel,
 }
 
 impl RelType {
     pub fn new(arch: Arch, type_info: u8) -> Option<Self> {
         match type_info {
-            R_OR1K_NONE if arch == Arch::OpenRisc =>
-                Some(RelType::None),
-            R_ARM_NONE if arch == Arch::Arm =>
-                Some(RelType::None),
+            R_OR1K_NONE if arch == Arch::OpenRisc => Some(RelType::None),
+            R_ARM_NONE if arch == Arch::Arm => Some(RelType::None),
 
-            R_OR1K_RELATIVE if arch == Arch::OpenRisc =>
-                Some(RelType::Relative),
-            R_ARM_RELATIVE if arch == Arch::Arm =>
-                Some(RelType::Relative),
+            R_OR1K_RELATIVE if arch == Arch::OpenRisc => Some(RelType::Relative),
+            R_ARM_RELATIVE if arch == Arch::Arm => Some(RelType::Relative),
 
-            R_OR1K_32 | R_OR1K_GLOB_DAT | R_OR1K_JMP_SLOT
-                if arch == Arch::OpenRisc => Some(RelType::Lookup),
-            R_ARM_GLOB_DAT | R_ARM_JUMP_SLOT
-                if arch == Arch::Arm => Some(RelType::Lookup),
+            R_OR1K_32 | R_OR1K_GLOB_DAT | R_OR1K_JMP_SLOT if arch == Arch::OpenRisc => Some(RelType::LookupAbs),
+            R_ARM_GLOB_DAT | R_ARM_JUMP_SLOT | R_ARM_ABS32 if arch == Arch::Arm => Some(RelType::LookupAbs),
 
-            _ =>
-                None
+            R_ARM_PREL31 if arch == Arch::Arm => Some(RelType::LookupRel),
+
+            _ => None,
         }
     }
 }
@@ -89,47 +83,115 @@ fn format_sym_name(sym_name: &[u8]) -> String {
 }
 
 pub fn relocate<R: Relocatable>(
-    arch: Arch, lib: &Library,
-    rel: &R, resolve: &dyn Fn(&[u8]) -> Option<Elf32_Word>
+    arch: Arch,
+    lib: &Library,
+    rel: &R,
+    resolve: &dyn Fn(&[u8]) -> Option<Elf32_Word>,
 ) -> Result<(), Error> {
     let sym;
     if rel.sym_info() == 0 {
         sym = None;
     } else {
-        sym = Some(lib.symtab().get(rel.sym_info() as usize)
-                   .ok_or("symbol out of bounds of symbol table")?)
+        sym = Some(
+            lib.symtab()
+                .get(rel.sym_info() as usize)
+                .ok_or("symbol out of bounds of symbol table")?,
+        )
     }
 
-    let rel_type = RelType::new(arch, rel.type_info())
-        .ok_or("unsupported relocation type")?;
-    let value;
-    match rel_type {
-        RelType::None =>
-            return Ok(()),
+    let rel_type = RelType::new(arch, rel.type_info()).ok_or("unsupported relocation type")?;
+    let value = match rel_type {
+        RelType::None => return Ok(()),
 
         RelType::Relative => {
             let addend = rel.addend(&lib.image);
-            value = lib.image.ptr().wrapping_offset(addend as isize) as Elf32_Word;
+            lib.image.ptr().wrapping_offset(addend as isize) as Elf32_Word
         }
 
-        RelType::Lookup => {
+        RelType::LookupAbs | RelType::LookupRel => {
             let sym = sym.ok_or("relocation requires an associated symbol")?;
             let sym_name = lib.name_starting_at(sym.st_name as usize)?;
 
-            if let Some(addr) = lib.lookup(sym_name) {
+            let sym_addr = if let Some(addr) = lib.lookup(sym_name) {
                 // First, try to resolve against itself.
                 trace!("looked up symbol {} in image", format_sym_name(sym_name));
-                value = lib.image.ptr() as u32 + addr;
+                addr
             } else if let Some(addr) = resolve(sym_name) {
                 // Second, call the user-provided function.
                 trace!("resolved symbol {:?}", format_sym_name(sym_name));
-                value = addr;
+                addr
             } else {
                 // We couldn't find it anywhere.
-                return Err(Error::Lookup(format_sym_name(sym_name)))
+                return Err(Error::Lookup(format_sym_name(sym_name)));
+            };
+
+            match rel_type {
+                RelType::LookupAbs => sym_addr,
+                RelType::LookupRel => {
+                    sym_addr.wrapping_sub(lib.image.ptr().wrapping_offset(rel.offset() as isize) as Elf32_Addr)
+                }
+                _ => unreachable!(),
             }
         }
+    };
+
+    match rel.type_info() {
+        R_ARM_PREL31 => {
+            let reloc_word = lib
+                .image
+                .get_ref::<Elf32_Word>(rel.offset())
+                .ok_or("relocation offset cannot be read")?;
+            lib.image
+                .write(rel.offset(), (reloc_word & 0x80000000) | (value & 0x7FFFFFFF))
+        }
+
+        _ => lib.image.write(rel.offset(), value),
+    }
+}
+
+pub fn rebind(arch: Arch, lib: &Library, name: &[u8], value: Elf32_Word) -> Result<(), Error> {
+    fn rebind_symbol_to_value<R: Relocatable>(
+        arch: Arch,
+        lib: &Library,
+        name: &[u8],
+        value: Elf32_Word,
+        relocs: &[R],
+    ) -> Result<(), Error> {
+        for reloc in relocs {
+            let rel_type = RelType::new(arch, reloc.type_info()).ok_or("unsupported relocation type")?;
+            match rel_type {
+                RelType::LookupAbs => {
+                    let sym = lib
+                        .symtab()
+                        .get(reloc.sym_info() as usize)
+                        .ok_or("symbol out of bounds of symbol table")?;
+                    let sym_name = lib.name_starting_at(sym.st_name as usize)?;
+
+                    if sym_name == name {
+                        lib.image.write(reloc.offset(), value)?
+                    }
+                }
+                // No associated symbols for other relocation types.
+                _ => {}
+            }
+        }
+
+        Ok(())
     }
 
-    lib.image.write(rel.offset(), value)
+    if lib.pltrel().is_empty() {
+        rebind_symbol_to_value(arch, lib, name, value, lib.rela())?;
+    } else {
+        rebind_symbol_to_value(arch, lib, name, value, lib.pltrel())?;
+    }
+
+    // FIXME: the cache maintainance operations may be more than enough,
+    // may cause performance degradation.
+    dcci_slice(lib.image.data);
+    iciallu();
+    bpiall();
+    dsb();
+    isb();
+
+    Ok(())
 }

src/libio/Cargo.toml.tpl

@@ -0,0 +1,17 @@
+[package]
+authors = ["M-Labs"]
+name = "io"
+version = "0.0.0"
+
+[lib]
+name = "io"
+path = "lib.rs"
+
+[dependencies]
+core_io = { version = "0.1", features = ["collections"] }
+byteorder = { version = "1.0", default-features = false, optional = true }
+
+libsupport_zynq = { path = "@@ZYNQ_RS@@/libsupport_zynq", default-features = false, features = ["alloc_core"] }
+
+[features]
+alloc = []

src/libio/cursor.rs

@@ -0,0 +1,85 @@
+#[cfg(feature = "alloc")]
+use alloc::vec::Vec;
+
+use core_io::{Error as IoError, Read, Write};
+
+#[derive(Debug, Clone)]
+pub struct Cursor<T> {
+    inner: T,
+    pos: usize,
+}
+
+impl<T> Cursor<T> {
+    #[inline]
+    pub fn new(inner: T) -> Cursor<T> {
+        Cursor { inner, pos: 0 }
+    }
+
+    #[inline]
+    pub fn into_inner(self) -> T {
+        self.inner
+    }
+
+    #[inline]
+    pub fn get_ref(&self) -> &T {
+        &self.inner
+    }
+
+    #[inline]
+    pub fn get_mut(&mut self) -> &mut T {
+        &mut self.inner
+    }
+
+    #[inline]
+    pub fn position(&self) -> usize {
+        self.pos
+    }
+
+    #[inline]
+    pub fn set_position(&mut self, pos: usize) {
+        self.pos = pos
+    }
+}
+
+impl<T: AsRef<[u8]>> Read for Cursor<T> {
+    fn read(&mut self, buf: &mut [u8]) -> Result<usize, IoError> {
+        let data = &self.inner.as_ref()[self.pos..];
+        let len = buf.len().min(data.len());
+        // ``copy_from_slice`` generates AXI bursts, use a regular loop instead
+        for i in 0..len {
+            buf[i] = data[i];
+        }
+        self.pos += len;
+        Ok(len)
+    }
+}
+
+impl Write for Cursor<&mut [u8]> {
+    fn write(&mut self, buf: &[u8]) -> Result<usize, IoError> {
+        let data = &mut self.inner[self.pos..];
+        let len = buf.len().min(data.len());
+        for i in 0..len {
+            data[i] = buf[i];
+        }
+        self.pos += len;
+        Ok(len)
+    }
+
+    #[inline]
+    fn flush(&mut self) -> Result<(), IoError> {
+        Ok(())
+    }
+}
+
+#[cfg(feature = "alloc")]
+impl Write for Cursor<Vec<u8>> {
+    fn write(&mut self, buf: &[u8]) -> Result<usize, IoError> {
+        self.inner.extend_from_slice(buf);
+        Ok(buf.len())
+    }
+
+    #[inline]
+    fn flush(&mut self) -> Result<(), IoError> {
+        Ok(())
+    }
+}

src/libio/lib.rs

@@ -0,0 +1,19 @@
+#![no_std]
+#![feature(never_type)]
+
+#[cfg(feature = "alloc")]
+extern crate alloc;
+extern crate core_io;
+
+#[cfg(feature = "byteorder")]
+extern crate byteorder;
+
+pub mod cursor;
+#[cfg(feature = "byteorder")]
+pub mod proto;
+
+pub use cursor::Cursor;
+#[cfg(all(feature = "byteorder", feature = "alloc"))]
+pub use proto::ReadStringError;
+#[cfg(feature = "byteorder")]
+pub use proto::{ProtoRead, ProtoWrite};

src/libio/proto.rs

@@ -1,15 +1,15 @@
+#[cfg(feature = "alloc")]
+use alloc::{string::String, vec};
 use core::str::Utf8Error;
-use byteorder::{ByteOrder, NetworkEndian};
-use alloc::vec;
-use alloc::string::String;
 
-use core_io::{Read, Write, Error as IoError};
+use byteorder::{ByteOrder, NativeEndian};
+use core_io::{Error as IoError, Read, Write};
 
 #[allow(dead_code)]
 #[derive(Debug, Clone, PartialEq)]
 pub enum ReadStringError<T> {
     Utf8(Utf8Error),
-    Other(T)
+    Other(T),
 }
 
 pub trait ProtoRead {
@@ -28,21 +28,21 @@ pub trait ProtoRead {
     fn read_u16(&mut self) -> Result<u16, Self::ReadError> {
         let mut bytes = [0; 2];
         self.read_exact(&mut bytes)?;
-        Ok(NetworkEndian::read_u16(&bytes))
+        Ok(NativeEndian::read_u16(&bytes))
     }
 
     #[inline]
     fn read_u32(&mut self) -> Result<u32, Self::ReadError> {
         let mut bytes = [0; 4];
         self.read_exact(&mut bytes)?;
-        Ok(NetworkEndian::read_u32(&bytes))
+        Ok(NativeEndian::read_u32(&bytes))
     }
 
     #[inline]
     fn read_u64(&mut self) -> Result<u64, Self::ReadError> {
         let mut bytes = [0; 8];
         self.read_exact(&mut bytes)?;
-        Ok(NetworkEndian::read_u64(&bytes))
+        Ok(NativeEndian::read_u64(&bytes))
     }
 
     #[inline]
@@ -51,7 +51,8 @@ pub trait ProtoRead {
     }
 
     #[inline]
-    fn read_bytes(&mut self) -> Result<::alloc::vec::Vec<u8>, Self::ReadError> {
+    #[cfg(feature = "alloc")]
+    fn read_bytes(&mut self) -> Result<vec::Vec<u8>, Self::ReadError> {
         let length = self.read_u32()?;
         let mut value = vec![0; length as usize];
         self.read_exact(&mut value)?;
@@ -59,7 +60,8 @@ pub trait ProtoRead {
     }
 
     #[inline]
-    fn read_string(&mut self) -> Result<::alloc::string::String, ReadStringError<Self::ReadError>> {
+    #[cfg(feature = "alloc")]
+    fn read_string(&mut self) -> Result<String, ReadStringError<Self::ReadError>> {
         let bytes = self.read_bytes().map_err(ReadStringError::Other)?;
         String::from_utf8(bytes).map_err(|err| ReadStringError::Utf8(err.utf8_error()))
     }
@@ -85,42 +87,42 @@ pub trait ProtoWrite {
     #[inline]
     fn write_u16(&mut self, value: u16) -> Result<(), Self::WriteError> {
         let mut bytes = [0; 2];
-        NetworkEndian::write_u16(&mut bytes, value);
+        NativeEndian::write_u16(&mut bytes, value);
         self.write_all(&bytes)
     }
 
     #[inline]
     fn write_i16(&mut self, value: i16) -> Result<(), Self::WriteError> {
         let mut bytes = [0; 2];
-        NetworkEndian::write_i16(&mut bytes, value);
+        NativeEndian::write_i16(&mut bytes, value);
         self.write_all(&bytes)
     }
 
     #[inline]
     fn write_u32(&mut self, value: u32) -> Result<(), Self::WriteError> {
         let mut bytes = [0; 4];
-        NetworkEndian::write_u32(&mut bytes, value);
+        NativeEndian::write_u32(&mut bytes, value);
         self.write_all(&bytes)
     }
 
     #[inline]
     fn write_i32(&mut self, value: i32) -> Result<(), Self::WriteError> {
         let mut bytes = [0; 4];
-        NetworkEndian::write_i32(&mut bytes, value);
+        NativeEndian::write_i32(&mut bytes, value);
         self.write_all(&bytes)
     }
 
     #[inline]
     fn write_u64(&mut self, value: u64) -> Result<(), Self::WriteError> {
         let mut bytes = [0; 8];
-        NetworkEndian::write_u64(&mut bytes, value);
+        NativeEndian::write_u64(&mut bytes, value);
         self.write_all(&bytes)
     }
 
     #[inline]
     fn write_i64(&mut self, value: i64) -> Result<(), Self::WriteError> {
         let mut bytes = [0; 8];
-        NetworkEndian::write_i64(&mut bytes, value);
+        NativeEndian::write_i64(&mut bytes, value);
         self.write_all(&bytes)
     }
 
@@ -136,12 +138,15 @@ pub trait ProtoWrite {
     }
 
     #[inline]
+    #[cfg(feature = "alloc")]
     fn write_string(&mut self, value: &str) -> Result<(), Self::WriteError> {
         self.write_bytes(value.as_bytes())
     }
 }
 
-impl<T> ProtoRead for T where T: Read + ?Sized {
+impl<T> ProtoRead for T
+where T: Read + ?Sized
+{
     type ReadError = IoError;
 
     fn read_exact(&mut self, buf: &mut [u8]) -> Result<(), Self::ReadError> {
@@ -149,7 +154,9 @@ impl<T> ProtoRead for T where T: Read + ?Sized {
     }
 }
 
-impl<T> ProtoWrite for T where T: Write + ?Sized {
+impl<T> ProtoWrite for T
+where T: Write + ?Sized
+{
     type WriteError = IoError;
 
     fn write_all(&mut self, buf: &[u8]) -> Result<(), Self::WriteError> {

src/libksupport/Cargo.toml.tpl

@@ -0,0 +1,40 @@
+[package]
+name = "ksupport"
+description = "Kernel support for Zynq-based platforms"
+version = "0.1.0"
+authors = ["M-Labs"]
+edition = "2018"
+
+[build-dependencies]
+build_zynq = { path = "../libbuild_zynq" }
+
+[dependencies]
+cslice = "0.3"
+log = "0.4"
+nb = "0.1"
+core_io = { version = "0.1", features = ["collections"] }
+byteorder = { version = "1.3", default-features = false }
+void = { version = "1", default-features = false }
+log_buffer = { version = "1.2" }
+libm = { version = "0.2", features = ["unstable"] }
+vcell = "0.1"
+
+libboard_zynq = { path = "@@ZYNQ_RS@@/libboard_zynq", features = ["ipv6"]}
+libsupport_zynq = { path = "@@ZYNQ_RS@@/libsupport_zynq", default-features = false, features = ["alloc_core"] }
+libcortex_a9 = { path = "@@ZYNQ_RS@@/libcortex_a9" }
+libasync = { path = "@@ZYNQ_RS@@/libasync" }
+libregister = { path = "@@ZYNQ_RS@@/libregister" }
+libconfig = { path = "@@ZYNQ_RS@@/libconfig", features = ["fat_lfn", "ipv6"] }
+
+dyld = { path = "../libdyld" }
+dwarf = { path = "../libdwarf" }
+unwind = { path = "../libunwind" }
+libc = { path = "../libc" }
+io = { path = "../libio" }
+libboard_artiq = { path = "../libboard_artiq" }
+
+[dependencies.nalgebra]
+git = "https://git.m-labs.hk/M-Labs/nalgebra.git"
+rev = "dd00f9b"
+default-features = false
+features = ["libm", "alloc"]

src/libksupport/build.rs

@@ -0,0 +1,5 @@
+extern crate build_zynq;
+
+fn main() {
+    build_zynq::cfg();
+}

src/libksupport/src/eh_artiq.rs

@@ -0,0 +1,561 @@
+// From Current artiq firmware ksupport implementation.
+// Modified to suit the case of artiq-zynq port, for ARM EHABI.
+// Portions of the code in this file are derived from code by:
+//
+// Copyright 2015 The Rust Project Developers. See the COPYRIGHT
+// file at http://rust-lang.org/COPYRIGHT.
+//
+// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
+// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
+// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
+// option. This file may not be copied, modified, or distributed
+// except according to those terms.
+#![allow(non_camel_case_types)]
+
+use core::mem;
+
+use core_io::Error as ReadError;
+use cslice::{AsCSlice, CSlice};
+use dwarf::eh::{self, EHAction, EHContext};
+use io::{Cursor, ProtoRead};
+use libc::{c_int, c_void, uintptr_t};
+use log::{error, trace};
+use unwind as uw;
+
+use crate::kernel::KERNEL_IMAGE;
+
+const EXCEPTION_CLASS: uw::_Unwind_Exception_Class = 0x4d_4c_42_53_41_52_54_51; /* 'MLBSARTQ' */
+
+#[cfg(target_arch = "arm")]
+const UNWIND_DATA_REG: (i32, i32) = (0, 1); // R0, R1
+
+// Note: CSlice within an exception may not be actual cslice, they may be strings that exist only
+// in the host. If the length == usize:MAX, the pointer is actually a string key in the host.
+#[repr(C)]
+#[derive(Clone, Copy)]
+pub struct Exception<'a> {
+    pub id: u32,
+    pub file: CSlice<'a, u8>,
+    pub line: u32,
+    pub column: u32,
+    pub function: CSlice<'a, u8>,
+    pub message: CSlice<'a, u8>,
+    pub param: [i64; 3],
+}
+
+fn str_err(_: core::str::Utf8Error) -> core::fmt::Error {
+    core::fmt::Error
+}
+
+fn exception_str<'a>(s: &'a CSlice<'a, u8>) -> Result<&'a str, core::str::Utf8Error> {
+    if s.len() == usize::MAX {
+        Ok("<host string>")
+    } else {
+        core::str::from_utf8(s.as_ref())
+    }
+}
+
+impl<'a> core::fmt::Debug for Exception<'a> {
+    fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result {
+        write!(
+            f,
+            "Exception {} from {} in {}:{}:{}, message: {}",
+            self.id,
+            exception_str(&self.function).map_err(str_err)?,
+            exception_str(&self.file).map_err(str_err)?,
+            self.line,
+            self.column,
+            exception_str(&self.message).map_err(str_err)?
+        )
+    }
+}
+
+const MAX_INFLIGHT_EXCEPTIONS: usize = 10;
+const MAX_BACKTRACE_SIZE: usize = 128;
+
+#[derive(Debug, Default)]
+pub struct StackPointerBacktrace {
+    pub stack_pointer: usize,
+    pub initial_backtrace_size: usize,
+    pub current_backtrace_size: usize,
+}
+
+struct ExceptionBuffer {
+    // we need n _Unwind_Exception, because each will have their own private data
+    uw_exceptions: [uw::_Unwind_Exception; MAX_INFLIGHT_EXCEPTIONS],
+    exceptions: [Option<Exception<'static>>; MAX_INFLIGHT_EXCEPTIONS + 1],
+    exception_stack: [isize; MAX_INFLIGHT_EXCEPTIONS + 1],
+    // nested exceptions will share the backtrace buffer, treated as a tree
+    // backtrace contains a tuple of IP and SP
+    backtrace: [(usize, usize); MAX_BACKTRACE_SIZE],
+    backtrace_size: usize,
+    // stack pointers are stored to reconstruct backtrace for each exception
+    stack_pointers: [StackPointerBacktrace; MAX_INFLIGHT_EXCEPTIONS + 1],
+    // current allocated nested exceptions
+    exception_count: usize,
+}
+
+static mut EXCEPTION_BUFFER: ExceptionBuffer = ExceptionBuffer {
+    uw_exceptions: [uw::_Unwind_Exception {
+        exception_class: EXCEPTION_CLASS,
+        exception_cleanup: cleanup,
+        private: [0; uw::unwinder_private_data_size],
+    }; MAX_INFLIGHT_EXCEPTIONS],
+    exceptions: [None; MAX_INFLIGHT_EXCEPTIONS + 1],
+    exception_stack: [-1; MAX_INFLIGHT_EXCEPTIONS + 1],
+    backtrace: [(0, 0); MAX_BACKTRACE_SIZE],
+    backtrace_size: 0,
+    stack_pointers: [StackPointerBacktrace {
+        stack_pointer: 0,
+        initial_backtrace_size: 0,
+        current_backtrace_size: 0,
+    }; MAX_INFLIGHT_EXCEPTIONS + 1],
+    exception_count: 0,
+};
+
+pub unsafe extern "C" fn reset_exception_buffer() {
+    trace!("reset exception buffer");
+    EXCEPTION_BUFFER.uw_exceptions = [uw::_Unwind_Exception {
+        exception_class: EXCEPTION_CLASS,
+        exception_cleanup: cleanup,
+        private: [0; uw::unwinder_private_data_size],
+    }; MAX_INFLIGHT_EXCEPTIONS];
+    EXCEPTION_BUFFER.exceptions = [None; MAX_INFLIGHT_EXCEPTIONS + 1];
+    EXCEPTION_BUFFER.exception_stack = [-1; MAX_INFLIGHT_EXCEPTIONS + 1];
+    EXCEPTION_BUFFER.backtrace_size = 0;
+    EXCEPTION_BUFFER.exception_count = 0;
+}
+
+type _Unwind_Stop_Fn = extern "C" fn(
+    version: c_int,
+    actions: i32,
+    exception_class: uw::_Unwind_Exception_Class,
+    exception_object: *mut uw::_Unwind_Exception,
+    context: *mut uw::_Unwind_Context,
+    stop_parameter: *mut c_void,
+) -> uw::_Unwind_Reason_Code;
+
+extern "C" {
+    // not defined in EHABI, but LLVM added it and is useful to us
+    fn _Unwind_ForcedUnwind(
+        exception: *mut uw::_Unwind_Exception,
+        stop_fn: _Unwind_Stop_Fn,
+        stop_parameter: *mut c_void,
+    ) -> uw::_Unwind_Reason_Code;
+}
+
+unsafe fn find_eh_action(context: *mut uw::_Unwind_Context, foreign_exception: bool, id: u32) -> Result<EHAction, ()> {
+    let lsda = uw::_Unwind_GetLanguageSpecificData(context) as *const u8;
+    let mut ip_before_instr: c_int = 0;
+    let ip = uw::_Unwind_GetIPInfo(context, &mut ip_before_instr);
+    let eh_context = EHContext {
+        // The return address points 1 byte past the call instruction,
+        // which could be in the next IP range in LSDA range table.
+        ip: if ip_before_instr != 0 { ip } else { ip - 1 },
+        func_start: uw::_Unwind_GetRegionStart(context),
+        get_text_start: &|| uw::_Unwind_GetTextRelBase(context),
+        get_data_start: &|| uw::_Unwind_GetDataRelBase(context),
+    };
+    eh::find_eh_action(lsda, &eh_context, foreign_exception, id)
+}
+
+pub unsafe fn artiq_personality(
+    _state: uw::_Unwind_State,
+    exception_object: *mut uw::_Unwind_Exception,
+    context: *mut uw::_Unwind_Context,
+) -> uw::_Unwind_Reason_Code {
+    // we will only do phase 2 forced unwinding now
+    // The DWARF unwinder assumes that _Unwind_Context holds things like the function
+    // and LSDA pointers, however ARM EHABI places them into the exception object.
+    // To preserve signatures of functions like _Unwind_GetLanguageSpecificData(), which
+    // take only the context pointer, GCC personality routines stash a pointer to
+    // exception_object in the context, using location reserved for ARM's
+    // "scratch register" (r12).
+    uw::_Unwind_SetGR(context, uw::UNWIND_POINTER_REG, exception_object as uw::_Unwind_Ptr);
+    // ...A more principled approach would be to provide the full definition of ARM's
+    // _Unwind_Context in our libunwind bindings and fetch the required data from there
+    // directly, bypassing DWARF compatibility functions.
+
+    let exception_class = (*exception_object).exception_class;
+    let foreign_exception = exception_class != EXCEPTION_CLASS;
+    assert!(!foreign_exception, "we do not expect foreign exceptions");
+    let index = EXCEPTION_BUFFER.exception_stack[EXCEPTION_BUFFER.exception_count - 1];
+    assert!(index != -1);
+    let exception = EXCEPTION_BUFFER.exceptions[index as usize].as_ref().unwrap();
+
+    let id = exception.id;
+    let eh_action = match find_eh_action(context, foreign_exception, id) {
+        Ok(action) => action,
+        Err(_) => return uw::_URC_FAILURE,
+    };
+    match eh_action {
+        EHAction::None => return continue_unwind(exception_object, context),
+        EHAction::Cleanup(lpad) | EHAction::Catch(lpad) => {
+            uw::_Unwind_SetGR(context, UNWIND_DATA_REG.0, exception_object as uintptr_t);
+            uw::_Unwind_SetGR(context, UNWIND_DATA_REG.1, exception as *const _ as uw::_Unwind_Word);
+            uw::_Unwind_SetIP(context, lpad);
+            return uw::_URC_INSTALL_CONTEXT;
+        }
+        EHAction::Terminate => return uw::_URC_FAILURE,
+    }
+
+    // On ARM EHABI the personality routine is responsible for actually
+    // unwinding a single stack frame before returning (ARM EHABI Sec. 6.1).
+    unsafe fn continue_unwind(
+        exception_object: *mut uw::_Unwind_Exception,
+        context: *mut uw::_Unwind_Context,
+    ) -> uw::_Unwind_Reason_Code {
+        let reason = __gnu_unwind_frame(exception_object, context);
+        if reason == uw::_URC_NO_REASON {
+            uw::_URC_CONTINUE_UNWIND
+        } else {
+            reason
+        }
+    }
+    // defined in libgcc
+    extern "C" {
+        fn __gnu_unwind_frame(
+            exception_object: *mut uw::_Unwind_Exception,
+            context: *mut uw::_Unwind_Context,
+        ) -> uw::_Unwind_Reason_Code;
+    }
+}
+
+pub unsafe extern "C" fn raise(exception: *const Exception) -> ! {
+    let count = EXCEPTION_BUFFER.exception_count;
+    let stack = &mut EXCEPTION_BUFFER.exception_stack;
+    let diff = exception as isize - EXCEPTION_BUFFER.exceptions.as_ptr() as isize;
+    if 0 <= diff && diff <= (mem::size_of::<Option<Exception>>() * MAX_INFLIGHT_EXCEPTIONS) as isize {
+        let index = diff / (mem::size_of::<Option<Exception>>() as isize);
+        trace!("reraise at {}", index);
+
+        let mut found = false;
+        for i in 0..=MAX_INFLIGHT_EXCEPTIONS + 1 {
+            if found {
+                if stack[i] == -1 {
+                    stack[i - 1] = index;
+                    assert!(i == count);
+                    break;
+                } else {
+                    stack[i - 1] = stack[i];
+                }
+            } else {
+                if stack[i] == index {
+                    found = true;
+                }
+            }
+        }
+        assert!(found);
+        let _result = _Unwind_ForcedUnwind(
+            &mut EXCEPTION_BUFFER.uw_exceptions[stack[count - 1] as usize],
+            stop_fn,
+            core::ptr::null_mut(),
+        );
+    } else {
+        if count < MAX_INFLIGHT_EXCEPTIONS {
+            trace!("raising exception at level {}", count);
+            let exception = &*exception;
+            for (i, slot) in EXCEPTION_BUFFER.exceptions.iter_mut().enumerate() {
+                // we should always be able to find a slot
+                if slot.is_none() {
+                    *slot = Some(*mem::transmute::<*const Exception, *const Exception<'static>>(
+                        exception,
+                    ));
+                    EXCEPTION_BUFFER.exception_stack[count] = i as isize;
+                    EXCEPTION_BUFFER.uw_exceptions[i].private = [0; uw::unwinder_private_data_size];
+                    EXCEPTION_BUFFER.stack_pointers[i] = StackPointerBacktrace {
+                        stack_pointer: 0,
+                        initial_backtrace_size: EXCEPTION_BUFFER.backtrace_size,
+                        current_backtrace_size: 0,
+                    };
+                    EXCEPTION_BUFFER.exception_count += 1;
+                    let _result =
+                        _Unwind_ForcedUnwind(&mut EXCEPTION_BUFFER.uw_exceptions[i], stop_fn, core::ptr::null_mut());
+                }
+            }
+        } else {
+            error!("too many nested exceptions");
+            // TODO: better reporting?
+            let exception = Exception {
+                id: get_exception_id("RuntimeError"),
+                file: file!().as_c_slice(),
+                line: line!(),
+                column: column!(),
+                // https://github.com/rust-lang/rfcs/pull/1719
+                function: "__artiq_raise".as_c_slice(),
+                message: "too many nested exceptions".as_c_slice(),
+                param: [0, 0, 0],
+            };
+            EXCEPTION_BUFFER.exceptions[MAX_INFLIGHT_EXCEPTIONS] = Some(mem::transmute(exception));
+            EXCEPTION_BUFFER.stack_pointers[MAX_INFLIGHT_EXCEPTIONS] = Default::default();
+            EXCEPTION_BUFFER.exception_count += 1;
+            uncaught_exception()
+        }
+    }
+    unreachable!();
+}
+
+fn read_exception_string<'a>(reader: &mut Cursor<&[u8]>) -> Result<CSlice<'a, u8>, ReadError> {
+    let len = reader.read_u32()? as usize;
+    if len == usize::MAX {
+        let data = reader.read_u32()?;
+        Ok(unsafe { CSlice::new(data as *const u8, len) })
+    } else {
+        let pos = reader.position();
+        let slice = unsafe {
+            let ptr = reader.get_ref().as_ptr().offset(pos as isize);
+            CSlice::new(ptr, len)
+        };
+        reader.set_position(pos + len);
+        Ok(slice)
+    }
+}
+
+fn read_exception(raw_exception: &[u8]) -> Result<Exception, ReadError> {
+    let mut reader = Cursor::new(raw_exception);
+
+    let mut byte = reader.read_u8()?;
+    // to sync
+    while byte != 0x5a {
+        byte = reader.read_u8()?;
+    }
+    // skip sync bytes, 0x09 indicates exception
+    while byte != 0x09 {
+        byte = reader.read_u8()?;
+    }
+    let _len = reader.read_u32()?;
+    // ignore the remaining exceptions, stack traces etc. - unwinding from another device would be unwise anyway
+    Ok(Exception {
+        id: reader.read_u32()?,
+        message: read_exception_string(&mut reader)?,
+        param: [
+            reader.read_u64()? as i64,
+            reader.read_u64()? as i64,
+            reader.read_u64()? as i64,
+        ],
+        file: read_exception_string(&mut reader)?,
+        line: reader.read_u32()?,
+        column: reader.read_u32()?,
+        function: read_exception_string(&mut reader)?,
+    })
+}
+
+pub fn raise_raw(raw_exception: &[u8]) -> ! {
+    use crate::artiq_raise;
+    if let Ok(exception) = read_exception(raw_exception) {
+        unsafe { raise(&exception) };
+    } else {
+        artiq_raise!("SubkernelError", "Error passing exception");
+    }
+}
+
+pub unsafe extern "C" fn resume() -> ! {
+    trace!("resume");
+    assert!(EXCEPTION_BUFFER.exception_count != 0);
+    let i = EXCEPTION_BUFFER.exception_stack[EXCEPTION_BUFFER.exception_count - 1];
+    assert!(i != -1);
+    let _result = _Unwind_ForcedUnwind(
+        &mut EXCEPTION_BUFFER.uw_exceptions[i as usize],
+        stop_fn,
+        core::ptr::null_mut(),
+    );
+    unreachable!()
+}
+
+pub unsafe extern "C" fn end_catch() {
+    let mut count = EXCEPTION_BUFFER.exception_count;
+    assert!(count != 0);
+    // we remove all exceptions with SP <= current exception SP
+    // i.e. the outer exception escapes the finally block
+    let index = EXCEPTION_BUFFER.exception_stack[count - 1] as usize;
+    EXCEPTION_BUFFER.exception_stack[count - 1] = -1;
+    EXCEPTION_BUFFER.exceptions[index] = None;
+    let outer_sp = EXCEPTION_BUFFER.stack_pointers[index].stack_pointer;
+    count -= 1;
+    for i in (0..count).rev() {
+        let index = EXCEPTION_BUFFER.exception_stack[i];
+        assert!(index != -1);
+        let index = index as usize;
+        let sp = EXCEPTION_BUFFER.stack_pointers[index].stack_pointer;
+        if sp >= outer_sp {
+            break;
+        }
+        EXCEPTION_BUFFER.exceptions[index] = None;
+        EXCEPTION_BUFFER.exception_stack[i] = -1;
+        count -= 1;
+    }
+    EXCEPTION_BUFFER.exception_count = count;
+    EXCEPTION_BUFFER.backtrace_size = if count > 0 {
+        let index = EXCEPTION_BUFFER.exception_stack[count - 1];
+        assert!(index != -1);
+        EXCEPTION_BUFFER.stack_pointers[index as usize].current_backtrace_size
+    } else {
+        0
+    };
+}
+
+extern "C" fn cleanup(_unwind_code: uw::_Unwind_Reason_Code, _uw_exception: *mut uw::_Unwind_Exception) {
+    unimplemented!()
+}
+
+fn uncaught_exception() -> ! {
+    unsafe {
+        // dump way to reorder the stack
+        for i in 0..EXCEPTION_BUFFER.exception_count {
+            if EXCEPTION_BUFFER.exception_stack[i] != i as isize {
+                // find the correct index
+                let index = EXCEPTION_BUFFER
+                    .exception_stack
+                    .iter()
+                    .position(|v| *v == i as isize)
+                    .unwrap();
+                let a = EXCEPTION_BUFFER.exception_stack[index];
+                let b = EXCEPTION_BUFFER.exception_stack[i];
+                assert!(a != -1 && b != -1);
+                core::mem::swap(
+                    &mut EXCEPTION_BUFFER.exception_stack[index],
+                    &mut EXCEPTION_BUFFER.exception_stack[i],
+                );
+                core::mem::swap(
+                    &mut EXCEPTION_BUFFER.exceptions[a as usize],
+                    &mut EXCEPTION_BUFFER.exceptions[b as usize],
+                );
+                core::mem::swap(
+                    &mut EXCEPTION_BUFFER.stack_pointers[a as usize],
+                    &mut EXCEPTION_BUFFER.stack_pointers[b as usize],
+                );
+            }
+        }
+    }
+    unsafe {
+        crate::kernel::core1::terminate(
+            EXCEPTION_BUFFER.exceptions[..EXCEPTION_BUFFER.exception_count].as_ref(),
+            EXCEPTION_BUFFER.stack_pointers[..EXCEPTION_BUFFER.exception_count].as_ref(),
+            EXCEPTION_BUFFER.backtrace[..EXCEPTION_BUFFER.backtrace_size].as_mut(),
+        )
+    }
+}
+
+// stop function which would be executed when we unwind each frame
+extern "C" fn stop_fn(
+    _version: c_int,
+    actions: i32,
+    _uw_exception_class: uw::_Unwind_Exception_Class,
+    _uw_exception: *mut uw::_Unwind_Exception,
+    context: *mut uw::_Unwind_Context,
+    _stop_parameter: *mut c_void,
+) -> uw::_Unwind_Reason_Code {
+    unsafe {
+        let load_addr = KERNEL_IMAGE.as_ref().unwrap().get_load_addr();
+        let backtrace_size = EXCEPTION_BUFFER.backtrace_size;
+        // we try to remove unrelated backtrace here to save some buffer size
+        if backtrace_size < MAX_BACKTRACE_SIZE {
+            let ip = uw::_Unwind_GetIP(context);
+            if ip >= load_addr {
+                let ip = ip - load_addr;
+                let sp = uw::_Unwind_GetGR(context, uw::UNWIND_SP_REG);
+                trace!("SP: {:X}, backtrace_size: {}", sp, backtrace_size);
+                EXCEPTION_BUFFER.backtrace[backtrace_size] = (ip, sp);
+                EXCEPTION_BUFFER.backtrace_size += 1;
+                let last_index = EXCEPTION_BUFFER.exception_stack[EXCEPTION_BUFFER.exception_count - 1];
+                assert!(last_index != -1);
+                let sp_info = &mut EXCEPTION_BUFFER.stack_pointers[last_index as usize];
+                sp_info.stack_pointer = sp;
+                sp_info.current_backtrace_size = backtrace_size + 1;
+            }
+        } else {
+            trace!("backtrace size exceeded");
+        }
+
+        if actions as u32 & uw::_US_END_OF_STACK as u32 != 0 {
+            uncaught_exception()
+        } else {
+            uw::_URC_NO_REASON
+        }
+    }
+}
+
+// Must be kept in sync with preallocate_runtime_exception_names() in `artiq.compiler.embedding`
+static EXCEPTION_ID_LOOKUP: [(&str, u32); 22] = [
+    ("RTIOUnderflow", 0),
+    ("RTIOOverflow", 1),
+    ("RTIODestinationUnreachable", 2),
+    ("DMAError", 3),
+    ("I2CError", 4),
+    ("CacheError", 5),
+    ("SPIError", 6),
+    ("SubkernelError", 7),
+    ("AssertionError", 8),
+    ("AttributeError", 9),
+    ("IndexError", 10),
+    ("IOError", 11),
+    ("KeyError", 12),
+    ("NotImplementedError", 13),
+    ("OverflowError", 14),
+    ("RuntimeError", 15),
+    ("TimeoutError", 16),
+    ("TypeError", 17),
+    ("ValueError", 18),
+    ("ZeroDivisionError", 19),
+    ("LinAlgError", 20),
+    ("UnwrapNoneError", 21),
+];
+
+pub fn get_exception_id(name: &str) -> u32 {
+    for (n, id) in EXCEPTION_ID_LOOKUP.iter() {
+        if *n == name {
+            return *id;
+        }
+    }
+    unimplemented!("unallocated internal exception id")
+}
+
+#[macro_export]
+macro_rules! artiq_raise {
+    ($name:expr, $message:expr, $param0:expr, $param1:expr, $param2:expr) => {{
+        use cslice::AsCSlice;
+        let name_id = $crate::eh_artiq::get_exception_id($name);
+        let message_cl = $message.clone();
+        let exn = $crate::eh_artiq::Exception {
+            id: name_id,
+            file: file!().as_c_slice(),
+            line: line!(),
+            column: column!(),
+            // https://github.com/rust-lang/rfcs/pull/1719
+            function: "(Rust function)".as_c_slice(),
+            message: message_cl.as_c_slice(),
+            param: [$param0, $param1, $param2],
+        };
+        #[allow(unused_unsafe)]
+        unsafe {
+            $crate::eh_artiq::raise(&exn)
+        }
+    }};
+    ($name:expr, $message:expr) => {{ artiq_raise!($name, $message, 0, 0, 0) }};
+}
+
+/// Takes as input exception id from host
+/// Generates a new exception with:
+///   * `id` set to `exn_id`
+///   * `message` set to corresponding exception name from `EXCEPTION_ID_LOOKUP`
+///
+/// The message is matched on host to ensure correct exception is being referred
+/// This test checks the synchronization of exception ids for runtime errors
+#[no_mangle]
+pub extern "C" fn test_exception_id_sync(exn_id: u32) {
+    let message = EXCEPTION_ID_LOOKUP
+        .iter()
+        .find_map(|&(name, id)| if id == exn_id { Some(name) } else { None })
+        .unwrap_or("unallocated internal exception id");
+
+    let exn = Exception {
+        id: exn_id,
+        file: file!().as_c_slice(),
+        line: 0,
+        column: 0,
+        function: "test_exception_id_sync".as_c_slice(),
+        message: message.as_c_slice(),
+        param: [0, 0, 0],
+    };
+    unsafe { raise(&exn) };
+}

src/libksupport/src/i2c.rs

@@ -0,0 +1,97 @@
+use libboard_zynq;
+
+use crate::artiq_raise;
+
+pub static mut I2C_BUS: Option<libboard_zynq::i2c::I2c> = None;
+
+pub extern "C" fn start(busno: i32) {
+    if busno > 0 {
+        artiq_raise!("I2CError", "I2C bus could not be accessed");
+    }
+    unsafe {
+        if (&mut I2C_BUS).as_mut().unwrap().start().is_err() {
+            artiq_raise!("I2CError", "I2C start failed");
+        }
+    }
+}
+
+pub extern "C" fn restart(busno: i32) {
+    if busno > 0 {
+        artiq_raise!("I2CError", "I2C bus could not be accessed");
+    }
+    unsafe {
+        if (&mut I2C_BUS).as_mut().unwrap().restart().is_err() {
+            artiq_raise!("I2CError", "I2C restart failed");
+        }
+    }
+}
+
+pub extern "C" fn stop(busno: i32) {
+    if busno > 0 {
+        artiq_raise!("I2CError", "I2C bus could not be accessed");
+    }
+    unsafe {
+        if (&mut I2C_BUS).as_mut().unwrap().stop().is_err() {
+            artiq_raise!("I2CError", "I2C stop failed");
+        }
+    }
+}
+
+pub extern "C" fn write(busno: i32, data: i32) -> bool {
+    if busno > 0 {
+        artiq_raise!("I2CError", "I2C bus could not be accessed");
+    }
+    unsafe {
+        match (&mut I2C_BUS).as_mut().unwrap().write(data as u8) {
+            Ok(r) => r,
+            Err(_) => artiq_raise!("I2CError", "I2C write failed"),
+        }
+    }
+}
+
+pub extern "C" fn read(busno: i32, ack: bool) -> i32 {
+    if busno > 0 {
+        artiq_raise!("I2CError", "I2C bus could not be accessed");
+    }
+    unsafe {
+        match (&mut I2C_BUS).as_mut().unwrap().read(ack) {
+            Ok(r) => r as i32,
+            Err(_) => artiq_raise!("I2CError", "I2C read failed"),
+        }
+    }
+}
+
+pub extern "C" fn switch_select(busno: i32, address: i32, mask: i32) {
+    if busno > 0 {
+        artiq_raise!("I2CError", "I2C bus could not be accessed");
+    }
+    let ch = match mask {
+        //decode from mainline, PCA9548-centric API
+        0x00 => None,
+        0x01 => Some(0),
+        0x02 => Some(1),
+        0x04 => Some(2),
+        0x08 => Some(3),
+        0x10 => Some(4),
+        0x20 => Some(5),
+        0x40 => Some(6),
+        0x80 => Some(7),
+        _ => artiq_raise!("I2CError", "switch select supports only one channel"),
+    };
+    unsafe {
+        if (&mut I2C_BUS)
+            .as_mut()
+            .unwrap()
+            .pca954x_select(address as u8, ch)
+            .is_err()
+        {
+            artiq_raise!("I2CError", "switch select failed");
+        }
+    }
+}
+
+pub fn init() {
+    let mut i2c = libboard_zynq::i2c::I2c::i2c0();
+    i2c.init().expect("I2C bus initialization failed");
+    unsafe { I2C_BUS = Some(i2c) };
+}

src/libksupport/src/irq.rs

@@ -0,0 +1,47 @@
+use core::sync::atomic::{AtomicBool, Ordering};
+
+use libboard_zynq::{gic, mpcore, println, stdio};
+use libcortex_a9::{asm, interrupt_handler, notify_spin_lock, regs::MPIDR, spin_lock_yield};
+use libregister::RegisterR;
+
+extern "C" {
+    static mut __stack1_start: u32;
+    fn main_core1() -> !;
+}
+
+static CORE1_RESTART: AtomicBool = AtomicBool::new(false);
+
+interrupt_handler!(IRQ, irq, __irq_stack0_start, __irq_stack1_start, {
+    if MPIDR.read().cpu_id() == 1 {
+        let mpcore = mpcore::RegisterBlock::mpcore();
+        let mut gic = gic::InterruptController::gic(mpcore);
+        let id = gic.get_interrupt_id();
+        if id.0 == 0 {
+            gic.end_interrupt(id);
+            asm::exit_irq();
+            asm!("b core1_restart");
+        }
+    }
+    stdio::drop_uart();
+    println!("IRQ");
+    loop {}
+});
+
+// This is actually not an interrupt handler, just use the macro for convenience.
+// This function would be called in normal mode (instead of interrupt mode), the outer naked
+// function wrapper is to tell libunwind to stop when it reaches here.
+interrupt_handler!(core1_restart, core1_restart_impl, __stack0_start, __stack1_start, {
+    asm::enable_irq();
+    CORE1_RESTART.store(false, Ordering::Relaxed);
+    notify_spin_lock();
+    main_core1();
+});
+
+pub fn restart_core1() {
+    let mut interrupt_controller = gic::InterruptController::gic(mpcore::RegisterBlock::mpcore());
+    CORE1_RESTART.store(true, Ordering::Relaxed);
+    interrupt_controller.send_sgi(gic::InterruptId(0), gic::CPUCore::Core1.into());
+    while CORE1_RESTART.load(Ordering::Relaxed) {
+        spin_lock_yield();
+    }
+}

src/libksupport/src/kernel/api.rs

@@ -0,0 +1,346 @@
+use alloc::vec;
+use core::{ffi::VaList, ptr, str};
+
+use libc::{c_char, c_int, size_t};
+use libm;
+use log::{info, warn};
+
+#[cfg(has_drtio)]
+use super::subkernel;
+use super::{cache,
+            core1::rtio_get_destination_status,
+            dma, linalg,
+            rpc::{rpc_recv, rpc_send, rpc_send_async}};
+use crate::{eh_artiq, i2c, rtio};
+
+extern "C" {
+    fn vsnprintf_(buffer: *mut c_char, count: size_t, format: *const c_char, va: VaList) -> c_int;
+}
+
+unsafe extern "C" fn core_log(fmt: *const c_char, mut args: ...) {
+    let size = vsnprintf_(ptr::null_mut(), 0, fmt, args.as_va_list()) as usize;
+    let mut buf = vec![0; size + 1];
+    vsnprintf_(buf.as_mut_ptr() as *mut i8, size + 1, fmt, args.as_va_list());
+    let buf: &[u8] = &buf.as_slice()[..size - 1]; // strip \n and NUL
+    match str::from_utf8(buf) {
+        Ok(s) => info!("kernel: {}", s),
+        Err(e) => {
+            info!("kernel: {}", (str::from_utf8(&buf[..e.valid_up_to()]).unwrap()));
+            warn!("kernel: invalid utf-8");
+        }
+    }
+}
+
+unsafe extern "C" fn rtio_log(fmt: *const c_char, mut args: ...) {
+    let size = vsnprintf_(ptr::null_mut(), 0, fmt, args.as_va_list()) as usize;
+    let mut buf = vec![0; size + 1];
+    vsnprintf_(buf.as_mut_ptr(), size + 1, fmt, args.as_va_list());
+    rtio::write_log(buf.as_slice());
+}
+
+macro_rules! api {
+    ($i:ident) => ({
+        extern { static $i: u8; }
+        unsafe { api!($i = &$i as *const _) }
+    });
+    ($i:ident, $d:item) => ({
+        $d
+        api!($i = $i)
+    });
+    ($i:ident = $e:expr) => {
+        (stringify!($i), $e as *const ())
+    }
+}
+
+macro_rules! api_libm_f64f64 {
+    ($i:ident) => {{
+        extern "C" fn $i(x: f64) -> f64 {
+            libm::$i(x)
+        }
+        api!($i = $i)
+    }};
+}
+
+macro_rules! api_libm_f64f64f64 {
+    ($i:ident) => {{
+        extern "C" fn $i(x: f64, y: f64) -> f64 {
+            libm::$i(x, y)
+        }
+        api!($i = $i)
+    }};
+}
+
+pub fn resolve(required: &[u8]) -> Option<u32> {
+    #[rustfmt::skip]
+    let api = &[
+        // timing
+        api!(now_mu = rtio::now_mu),
+        api!(at_mu = rtio::at_mu),
+        api!(delay_mu = rtio::delay_mu),
+
+        // rpc
+        api!(rpc_send = rpc_send),
+        api!(rpc_send_async = rpc_send_async),
+        api!(rpc_recv = rpc_recv),
+
+        // rtio
+        api!(rtio_init = rtio::init),
+        api!(rtio_get_destination_status = rtio_get_destination_status),
+        api!(rtio_get_counter = rtio::get_counter),
+        api!(rtio_output = rtio::output),
+        api!(rtio_output_wide = rtio::output_wide),
+        api!(rtio_input_timestamp = rtio::input_timestamp),
+        api!(rtio_input_data = rtio::input_data),
+        api!(rtio_input_timestamped_data = rtio::input_timestamped_data),
+
+        // log
+        api!(core_log = core_log),
+        api!(rtio_log = rtio_log),
+
+        // rtio dma
+        api!(dma_record_start = dma::dma_record_start),
+        api!(dma_record_stop = dma::dma_record_stop),
+        api!(dma_erase = dma::dma_erase),
+        api!(dma_retrieve = dma::dma_retrieve),
+        api!(dma_playback = dma::dma_playback),
+
+        // cache
+        api!(cache_get = cache::get),
+        api!(cache_put = cache::put),
+
+        // i2c
+        api!(i2c_start = i2c::start),
+        api!(i2c_restart = i2c::restart),
+        api!(i2c_stop = i2c::stop),
+        api!(i2c_write = i2c::write),
+        api!(i2c_read = i2c::read),
+        api!(i2c_switch_select = i2c::switch_select),
+
+        // subkernel
+        #[cfg(has_drtio)]
+        api!(subkernel_load_run = subkernel::load_run),
+        #[cfg(has_drtio)]
+        api!(subkernel_await_finish = subkernel::await_finish),
+        #[cfg(has_drtio)]
+        api!(subkernel_send_message = subkernel::send_message),
+        #[cfg(has_drtio)]
+        api!(subkernel_await_message = subkernel::await_message),
+
+        // Double-precision floating-point arithmetic helper functions
+        // RTABI chapter 4.1.2, Table 2
+        api!(__aeabi_dadd),
+        api!(__aeabi_ddiv),
+        api!(__aeabi_dmul),
+        api!(__aeabi_dsub),
+
+        // Double-precision floating-point comparison helper functions
+        // RTABI chapter 4.1.2, Table 3
+        api!(__aeabi_dcmpeq),
+        api!(__aeabi_dcmpeq),
+        api!(__aeabi_dcmplt),
+        api!(__aeabi_dcmple),
+        api!(__aeabi_dcmpge),
+        api!(__aeabi_dcmpgt),
+        api!(__aeabi_dcmpun),
+
+        // Single-precision floating-point arithmetic helper functions
+        // RTABI chapter 4.1.2, Table 4
+        api!(__aeabi_fadd),
+        api!(__aeabi_fdiv),
+        api!(__aeabi_fmul),
+        api!(__aeabi_fsub),
+
+        // Single-precision floating-point comparison helper functions
+        // RTABI chapter 4.1.2, Table 5
+        api!(__aeabi_fcmpeq),
+        api!(__aeabi_fcmpeq),
+        api!(__aeabi_fcmplt),
+        api!(__aeabi_fcmple),
+        api!(__aeabi_fcmpge),
+        api!(__aeabi_fcmpgt),
+        api!(__aeabi_fcmpun),
+
+        // Floating-point to integer conversions.
+        // RTABI chapter 4.1.2, Table 6
+        api!(__aeabi_d2iz),
+        api!(__aeabi_d2uiz),
+        api!(__aeabi_d2lz),
+        api!(__aeabi_d2ulz),
+        api!(__aeabi_f2iz),
+        api!(__aeabi_f2uiz),
+        api!(__aeabi_f2lz),
+        api!(__aeabi_f2ulz),
+
+        // Conversions between floating types.
+        // RTABI chapter 4.1.2, Table 7
+        api!(__aeabi_f2d),
+
+        // Integer to floating-point conversions.
+        // RTABI chapter 4.1.2, Table 8
+        api!(__aeabi_i2d),
+        api!(__aeabi_ui2d),
+        api!(__aeabi_l2d),
+        api!(__aeabi_ul2d),
+        api!(__aeabi_i2f),
+        api!(__aeabi_ui2f),
+        api!(__aeabi_l2f),
+        api!(__aeabi_ul2f),
+
+        // Long long helper functions
+        // RTABI chapter 4.2, Table 9
+        api!(__aeabi_lmul),
+        api!(__aeabi_llsl),
+        api!(__aeabi_llsr),
+        api!(__aeabi_lasr),
+
+        // Integer division functions
+        // RTABI chapter 4.3.1
+        api!(__aeabi_idiv),
+        api!(__aeabi_ldivmod),
+        api!(__aeabi_idivmod),
+        api!(__aeabi_uidiv),
+        api!(__aeabi_uldivmod),
+
+        // 4.3.4 Memory copying, clearing, and setting
+        api!(__aeabi_memcpy8),
+        api!(__aeabi_memcpy4),
+        api!(__aeabi_memcpy),
+        api!(__aeabi_memmove8),
+        api!(__aeabi_memmove4),
+        api!(__aeabi_memmove),
+        api!(__aeabi_memset8),
+        api!(__aeabi_memset4),
+        api!(__aeabi_memset),
+        api!(__aeabi_memclr8),
+        api!(__aeabi_memclr4),
+        api!(__aeabi_memclr),
+
+        // libc
+        api!(
+            memcpy,
+            extern "C" {
+                fn memcpy(dest: *mut u8, src: *const u8, n: usize) -> *mut u8;
+            }
+        ),
+        api!(
+            memmove,
+            extern "C" {
+                fn memmove(dest: *mut u8, src: *const u8, n: usize) -> *mut u8;
+            }
+        ),
+        api!(
+            memset,
+            extern "C" {
+                fn memset(s: *mut u8, c: i32, n: usize) -> *mut u8;
+            }
+        ),
+        api!(
+            memcmp,
+            extern "C" {
+                fn memcmp(s1: *const u8, s2: *const u8, n: usize) -> i32;
+            }
+        ),
+
+        // exceptions
+        api!(_Unwind_Resume = unwind::_Unwind_Resume),
+        api!(__nac3_personality = eh_artiq::artiq_personality),
+        api!(__nac3_raise = eh_artiq::raise),
+        api!(__nac3_resume = eh_artiq::resume),
+        api!(__nac3_end_catch = eh_artiq::end_catch),
+
+        // legacy exception symbols
+        api!(__artiq_personality = eh_artiq::artiq_personality),
+        api!(__artiq_raise = eh_artiq::raise),
+        api!(__artiq_resume = eh_artiq::resume),
+        api!(__artiq_end_catch = eh_artiq::end_catch),
+
+        // Implementations for LLVM math intrinsics
+        api!(__powidf2),
+
+        // libm
+        api_libm_f64f64!(acos),
+        api_libm_f64f64!(acosh),
+        api_libm_f64f64!(asin),
+        api_libm_f64f64!(asinh),
+        api_libm_f64f64!(atan),
+        api_libm_f64f64f64!(atan2),
+        api_libm_f64f64!(atanh),
+        api_libm_f64f64!(cbrt),
+        api_libm_f64f64!(ceil),
+        api_libm_f64f64f64!(copysign),
+        api_libm_f64f64!(cos),
+        api_libm_f64f64!(cosh),
+        api_libm_f64f64!(erf),
+        api_libm_f64f64!(erfc),
+        api_libm_f64f64!(exp),
+        api_libm_f64f64!(exp2),
+        api_libm_f64f64!(exp10),
+        api_libm_f64f64!(expm1),
+        api_libm_f64f64!(fabs),
+        api_libm_f64f64!(floor),
+        {
+            extern "C" fn fma(x: f64, y: f64, z: f64) -> f64 {
+                libm::fma(x, y, z)
+            }
+            api!(fma = fma)
+        },
+        api_libm_f64f64f64!(fmax),
+        api_libm_f64f64f64!(fmin),
+        api_libm_f64f64f64!(fmod),
+        api_libm_f64f64f64!(hypot),
+        api_libm_f64f64!(j0),
+        api_libm_f64f64!(j1),
+        {
+            extern "C" fn jn(n: i32, x: f64) -> f64 {
+                libm::jn(n, x)
+            }
+            api!(jn = jn)
+        },
+        api_libm_f64f64!(lgamma),
+        api_libm_f64f64!(log),
+        api_libm_f64f64!(log2),
+        api_libm_f64f64!(log10),
+        api_libm_f64f64f64!(nextafter),
+        api_libm_f64f64f64!(pow),
+        api_libm_f64f64!(round),
+        api_libm_f64f64!(rint),
+        api_libm_f64f64!(sin),
+        api_libm_f64f64!(sinh),
+        api_libm_f64f64!(sqrt),
+        api_libm_f64f64!(tan),
+        api_libm_f64f64!(tanh),
+        api_libm_f64f64!(tgamma),
+        api_libm_f64f64!(trunc),
+        api_libm_f64f64!(y0),
+        api_libm_f64f64!(y1),
+        {
+            extern "C" fn yn(n: i32, x: f64) -> f64 {
+                libm::yn(n, x)
+            }
+            api!(yn = yn)
+        },
+
+        // linalg
+        api!(np_linalg_cholesky = linalg::np_linalg_cholesky),
+        api!(np_linalg_qr = linalg::np_linalg_qr),
+        api!(np_linalg_svd = linalg::np_linalg_svd),
+        api!(np_linalg_inv = linalg::np_linalg_inv),
+        api!(np_linalg_pinv = linalg::np_linalg_pinv),
+        api!(np_linalg_matrix_power = linalg::np_linalg_matrix_power),
+        api!(np_linalg_det = linalg::np_linalg_det),
+        api!(sp_linalg_lu = linalg::sp_linalg_lu),
+        api!(sp_linalg_schur = linalg::sp_linalg_schur),
+        api!(sp_linalg_hessenberg = linalg::sp_linalg_hessenberg),
+
+        /*
+         * syscall for unit tests
+         * Used in `artiq.tests.coredevice.test_exceptions.ExceptionTest.test_raise_exceptions_kernel`
+         * This syscall checks that the exception IDs used in the Python `EmbeddingMap` (in `artiq.language.embedding`)
+         * match the `EXCEPTION_ID_LOOKUP` defined in the firmware (`libksupport::src::eh_artiq`)
+         */
+        api!(test_exception_id_sync = eh_artiq::test_exception_id_sync)
+    ];
+    api.iter()
+        .find(|&&(exported, _)| exported.as_bytes() == required)
+        .map(|&(_, ptr)| ptr as u32)
+}

src/libksupport/src/kernel/cache.rs

@@ -0,0 +1,37 @@
+use alloc::{boxed::Box, string::String};
+use core::mem::{forget, transmute};
+
+use cslice::{AsCSlice, CSlice};
+
+use super::{Message, KERNEL_CHANNEL_0TO1, KERNEL_CHANNEL_1TO0};
+
+pub extern "C" fn get(key: CSlice<u8>) -> &CSlice<'static, i32> {
+    let key = String::from_utf8(key.as_ref().to_vec()).unwrap();
+    unsafe {
+        KERNEL_CHANNEL_1TO0
+            .as_mut()
+            .unwrap()
+            .send(Message::CacheGetRequest(key));
+        let msg = KERNEL_CHANNEL_0TO1.as_mut().unwrap().recv();
+        if let Message::CacheGetReply(v) = msg {
+            let leaked = Box::new(v.as_c_slice());
+            let reference = transmute(leaked.as_ref());
+            forget(leaked);
+            forget(v);
+            reference
+        } else {
+            panic!("Expected CacheGetReply for CacheGetRequest");
+        }
+    }
+}
+
+pub extern "C" fn put(key: CSlice<u8>, list: &CSlice<i32>) {
+    let key = String::from_utf8(key.as_ref().to_vec()).unwrap();
+    let value = list.as_ref().to_vec();
+    unsafe {
+        KERNEL_CHANNEL_1TO0
+            .as_mut()
+            .unwrap()
+            .send(Message::CachePutRequest(key, value));
+    }
+}

src/libksupport/src/kernel/control.rs

@@ -0,0 +1,55 @@
+use core::mem::{forget, replace};
+
+use libcortex_a9::sync_channel::{Receiver, Sender};
+use libsupport_zynq::boot::Core1;
+
+use super::{Message, CHANNEL_0TO1, CHANNEL_1TO0, CHANNEL_SEM, INIT_LOCK};
+use crate::irq::restart_core1;
+
+pub struct Control {
+    pub tx: Sender<'static, Message>,
+    pub rx: Receiver<'static, Message>,
+}
+
+fn get_channels() -> (Sender<'static, Message>, Receiver<'static, Message>) {
+    CHANNEL_SEM.wait();
+    let mut core0_tx = None;
+    while core0_tx.is_none() {
+        core0_tx = CHANNEL_0TO1.lock().take();
+    }
+    let core0_tx = core0_tx.unwrap();
+
+    let mut core0_rx = None;
+    while core0_rx.is_none() {
+        core0_rx = CHANNEL_1TO0.lock().take();
+    }
+    let core0_rx = core0_rx.unwrap();
+
+    (core0_tx, core0_rx)
+}
+
+impl Control {
+    pub fn start() -> Self {
+        Core1::start(true);
+        let (core0_tx, core0_rx) = get_channels();
+
+        Control {
+            tx: core0_tx,
+            rx: core0_rx,
+        }
+    }
+
+    pub fn restart(&mut self) {
+        {
+            let _lock = INIT_LOCK.lock();
+            restart_core1();
+            unsafe {
+                self.tx.drop_elements();
+            }
+        }
+        let (core0_tx, core0_rx) = get_channels();
+        // dangling pointer here, so we forget it
+        forget(replace(&mut self.tx, core0_tx));
+        forget(replace(&mut self.rx, core0_rx));
+    }
+}

src/libksupport/src/kernel/core1.rs

@@ -0,0 +1,254 @@
+//! Kernel prologue/epilogue that runs on the 2nd CPU core
+
+use alloc::borrow::ToOwned;
+use core::{cell::UnsafeCell, mem, ptr};
+
+use cslice::CSlice;
+use dyld::{self, elf::EXIDX_Entry, Library};
+use libboard_zynq::{gic, mpcore};
+use libcortex_a9::{asm::{dsb, isb},
+                   cache::{bpiall, dcci_slice, iciallu},
+                   enable_fpu, sync_channel};
+use libsupport_zynq::ram;
+use log::{debug, error, info};
+
+use super::{api::resolve, dma, rpc::rpc_send_async, Message, CHANNEL_0TO1, CHANNEL_1TO0, CHANNEL_SEM, INIT_LOCK,
+            KERNEL_CHANNEL_0TO1, KERNEL_CHANNEL_1TO0, KERNEL_IMAGE};
+use crate::{eh_artiq, get_async_errors, rtio};
+
+// linker symbols
+extern "C" {
+    static __text_start: u32;
+    static __text_end: u32;
+    static __exidx_start: EXIDX_Entry;
+    static __exidx_end: EXIDX_Entry;
+}
+
+unsafe fn attribute_writeback(typeinfo: *const ()) {
+    struct Attr {
+        offset: usize,
+        tag: CSlice<'static, u8>,
+        name: CSlice<'static, u8>,
+    }
+
+    struct Type {
+        attributes: *const *const Attr,
+        objects: *const *const (),
+    }
+
+    let mut tys = typeinfo as *const *const Type;
+    while !(*tys).is_null() {
+        let ty = *tys;
+        tys = tys.offset(1);
+
+        let mut objects = (*ty).objects;
+        while !(*objects).is_null() {
+            let object = *objects;
+            objects = objects.offset(1);
+
+            let mut attributes = (*ty).attributes;
+            while !(*attributes).is_null() {
+                let attribute = *attributes;
+                attributes = attributes.offset(1);
+
+                if (*attribute).tag.len() > 0 {
+                    rpc_send_async(
+                        0,
+                        &(*attribute).tag,
+                        [
+                            &object as *const _ as *const (),
+                            &(*attribute).name as *const _ as *const (),
+                            (object as usize + (*attribute).offset) as *const (),
+                        ]
+                        .as_ptr(),
+                    );
+                }
+            }
+        }
+    }
+}
+
+pub struct KernelImage {
+    library: UnsafeCell<Library>,
+    __modinit__: u32,
+    typeinfo: Option<u32>,
+}
+
+impl KernelImage {
+    pub fn new(library: Library) -> Result<Self, dyld::Error> {
+        let __modinit__ = library
+            .lookup(b"__modinit__")
+            .ok_or(dyld::Error::Lookup("__modinit__".to_owned()))?;
+        let typeinfo = library.lookup(b"typeinfo");
+
+        // clear .bss
+        let bss_start = library.lookup(b"__bss_start");
+        let end = library.lookup(b"_end");
+        if let Some(bss_start) = bss_start {
+            let end = end.ok_or(dyld::Error::Lookup("_end".to_owned()))?;
+            unsafe {
+                ptr::write_bytes(bss_start as *mut u8, 0, (end - bss_start) as usize);
+            }
+        }
+
+        Ok(KernelImage {
+            library: UnsafeCell::new(library),
+            __modinit__,
+            typeinfo,
+        })
+    }
+
+    pub unsafe fn rebind(&self, name: &[u8], addr: *const ()) -> Result<(), dyld::Error> {
+        let library = self.library.get().as_mut().unwrap();
+        library.rebind(name, addr)
+    }
+
+    pub unsafe fn exec(&self) {
+        // Flush data cache entries for the image in DDR, including
+        // Memory/Instruction Synchronization Barriers
+        dcci_slice(self.library.get().as_ref().unwrap().image.data);
+        iciallu();
+        bpiall();
+        dsb();
+        isb();
+
+        (mem::transmute::<u32, extern "C" fn()>(self.__modinit__))();
+
+        if let Some(typeinfo) = self.typeinfo {
+            attribute_writeback(typeinfo as *const ());
+        }
+    }
+
+    pub fn get_load_addr(&self) -> usize {
+        unsafe { self.library.get().as_ref().unwrap().image.as_ptr() as usize }
+    }
+}
+
+#[no_mangle]
+pub extern "C" fn main_core1() {
+    enable_fpu();
+    debug!("Core1 started");
+
+    ram::init_alloc_core1();
+    gic::InterruptController::gic(mpcore::RegisterBlock::mpcore()).enable_interrupts();
+
+    let (mut core0_tx, mut core1_rx) = sync_channel!(Message, 4);
+    let (mut core1_tx, core0_rx) = sync_channel!(Message, 4);
+    unsafe {
+        INIT_LOCK.lock();
+        core0_tx.reset();
+        core1_tx.reset();
+        if !KERNEL_IMAGE.is_null() {
+            // indicates forceful termination of previous kernel
+            KERNEL_IMAGE = core::ptr::null();
+            debug!("rtio init");
+            rtio::init();
+        }
+        dma::init_dma_recorder();
+    }
+    *CHANNEL_0TO1.lock() = Some(core0_tx);
+    *CHANNEL_1TO0.lock() = Some(core0_rx);
+    CHANNEL_SEM.signal();
+
+    // set on load, cleared on start
+    let mut loaded_kernel = None;
+    loop {
+        let message = core1_rx.recv();
+        match message {
+            Message::LoadRequest(data) => {
+                let result = dyld::load(&data, &resolve).and_then(KernelImage::new);
+                match result {
+                    Ok(kernel) => {
+                        loaded_kernel = Some(kernel);
+                        debug!("kernel loaded");
+                        core1_tx.send(Message::LoadCompleted);
+                    }
+                    Err(error) => {
+                        error!("failed to load shared library: {}", error);
+                        core1_tx.send(Message::LoadFailed);
+                    }
+                }
+            }
+            Message::StartRequest => {
+                info!("kernel starting");
+                if let Some(kernel) = loaded_kernel.take() {
+                    unsafe {
+                        eh_artiq::reset_exception_buffer();
+                        KERNEL_CHANNEL_0TO1 = Some(core1_rx);
+                        KERNEL_CHANNEL_1TO0 = Some(core1_tx);
+                        KERNEL_IMAGE = &kernel as *const KernelImage;
+                        kernel.exec();
+                        KERNEL_IMAGE = ptr::null();
+                        core1_rx = KERNEL_CHANNEL_0TO1.take().unwrap();
+                        core1_tx = KERNEL_CHANNEL_1TO0.take().unwrap();
+                    }
+                }
+                info!("kernel finished");
+                let async_errors = unsafe { get_async_errors() };
+                core1_tx.send(Message::KernelFinished(async_errors));
+            }
+            _ => error!("Core1 received unexpected message: {:?}", message),
+        }
+    }
+}
+
+/// Called by eh_artiq
+pub fn terminate(
+    exceptions: &'static [Option<eh_artiq::Exception<'static>>],
+    stack_pointers: &'static [eh_artiq::StackPointerBacktrace],
+    backtrace: &'static mut [(usize, usize)],
+) -> ! {
+    {
+        let core1_tx = unsafe { KERNEL_CHANNEL_1TO0.as_mut().unwrap() };
+        let errors = unsafe { get_async_errors() };
+        core1_tx.send(Message::KernelException(exceptions, stack_pointers, backtrace, errors));
+    }
+    loop {}
+}
+
+/// Called by llvm_libunwind
+#[no_mangle]
+extern "C" fn dl_unwind_find_exidx(pc: *const u32, len_ptr: *mut u32) -> *const u32 {
+    let length;
+    let start: *const EXIDX_Entry;
+    unsafe {
+        if &__text_start as *const u32 <= pc && pc < &__text_end as *const u32 {
+            length = (&__exidx_end as *const EXIDX_Entry).offset_from(&__exidx_start) as u32;
+            start = &__exidx_start;
+        } else if KERNEL_IMAGE != ptr::null() {
+            let exidx = KERNEL_IMAGE
+                .as_ref()
+                .expect("dl_unwind_find_exidx kernel image")
+                .library
+                .get()
+                .as_ref()
+                .unwrap()
+                .exidx();
+            length = exidx.len() as u32;
+            start = exidx.as_ptr();
+        } else {
+            length = 0;
+            start = ptr::null();
+        }
+        *len_ptr = length;
+    }
+    start as *const u32
+}
+
+pub extern "C" fn rtio_get_destination_status(destination: i32) -> bool {
+    #[cfg(has_drtio)]
+    if destination > 0 && destination < 255 {
+        let reply = unsafe {
+            let core1_rx = KERNEL_CHANNEL_0TO1.as_mut().unwrap();
+            let core1_tx = KERNEL_CHANNEL_1TO0.as_mut().unwrap();
+            core1_tx.send(Message::UpDestinationsRequest(destination));
+            core1_rx.recv()
+        };
+        return match reply {
+            Message::UpDestinationsReply(x) => x,
+            _ => panic!("received unexpected reply to UpDestinationsRequest: {:?}", reply),
+        };
+    }
+
+    destination == 0
+}

src/libksupport/src/kernel/dma.rs

@@ -0,0 +1,255 @@
+use alloc::{string::String, vec::Vec};
+use core::mem;
+
+use cslice::CSlice;
+
+use super::{Message, KERNEL_CHANNEL_0TO1, KERNEL_CHANNEL_1TO0, KERNEL_IMAGE};
+use crate::{artiq_raise, pl::csr, rtio};
+
+#[repr(C)]
+pub struct DmaTrace {
+    duration: i64,
+    address: i32,
+    uses_ddma: bool,
+}
+
+#[derive(Clone, Debug)]
+pub struct DmaRecorder {
+    pub name: String,
+    pub buffer: Vec<u8>,
+    pub duration: i64,
+    pub enable_ddma: bool,
+}
+
+static mut RECORDER: Option<DmaRecorder> = None;
+
+pub unsafe fn init_dma_recorder() {
+    // as static would remain after restart, we have to reset it,
+    // without running its destructor.
+    mem::forget(mem::replace(&mut RECORDER, None));
+}
+
+pub extern "C" fn dma_record_start(name: CSlice<u8>) {
+    let name = String::from_utf8(name.as_ref().to_vec()).unwrap();
+    unsafe {
+        KERNEL_CHANNEL_1TO0
+            .as_mut()
+            .unwrap()
+            .send(Message::DmaEraseRequest(name.clone()));
+    }
+    unsafe {
+        if RECORDER.is_some() {
+            artiq_raise!("DMAError", "DMA is already recording")
+        }
+
+        let library = KERNEL_IMAGE.as_ref().unwrap();
+        library.rebind(b"rtio_output", dma_record_output as *const ()).unwrap();
+        library
+            .rebind(b"rtio_output_wide", dma_record_output_wide as *const ())
+            .unwrap();
+
+        RECORDER = Some(DmaRecorder {
+            name,
+            buffer: Vec::new(),
+            duration: 0,
+            enable_ddma: false,
+        });
+    }
+}
+
+pub extern "C" fn dma_record_stop(duration: i64, enable_ddma: bool) {
+    unsafe {
+        if RECORDER.is_none() {
+            artiq_raise!("DMAError", "DMA is not recording")
+        }
+
+        let library = KERNEL_IMAGE.as_ref().unwrap();
+        library.rebind(b"rtio_output", rtio::output as *const ()).unwrap();
+        library
+            .rebind(b"rtio_output_wide", rtio::output_wide as *const ())
+            .unwrap();
+
+        let mut recorder = RECORDER.take().unwrap();
+        recorder.duration = duration;
+        recorder.enable_ddma = enable_ddma;
+        KERNEL_CHANNEL_1TO0
+            .as_mut()
+            .unwrap()
+            .send(Message::DmaPutRequest(recorder));
+    }
+}
+
+#[inline(always)]
+unsafe fn dma_record_output_prepare(timestamp: i64, target: i32, words: usize) {
+    // See gateware/rtio/dma.py.
+    const HEADER_LENGTH: usize = /*length*/1 + /*channel*/3 + /*timestamp*/8 + /*address*/1;
+    let length = HEADER_LENGTH + /*data*/words * 4;
+
+    let buffer = &mut RECORDER.as_mut().unwrap().buffer;
+    buffer.reserve(length);
+    buffer.extend_from_slice(&[
+        (length >> 0) as u8,
+        (target >> 8) as u8,
+        (target >> 16) as u8,
+        (target >> 24) as u8,
+        (timestamp >> 0) as u8,
+        (timestamp >> 8) as u8,
+        (timestamp >> 16) as u8,
+        (timestamp >> 24) as u8,
+        (timestamp >> 32) as u8,
+        (timestamp >> 40) as u8,
+        (timestamp >> 48) as u8,
+        (timestamp >> 56) as u8,
+        (target >> 0) as u8,
+    ]);
+}
+
+pub extern "C" fn dma_record_output(target: i32, word: i32) {
+    unsafe {
+        let timestamp = rtio::now_mu();
+        dma_record_output_prepare(timestamp, target, 1);
+        RECORDER.as_mut().unwrap().buffer.extend_from_slice(&[
+            (word >> 0) as u8,
+            (word >> 8) as u8,
+            (word >> 16) as u8,
+            (word >> 24) as u8,
+        ]);
+    }
+}
+
+pub extern "C" fn dma_record_output_wide(target: i32, words: &CSlice<i32>) {
+    assert!(words.len() <= 16); // enforce the hardware limit
+
+    unsafe {
+        let timestamp = rtio::now_mu();
+        dma_record_output_prepare(timestamp, target, words.len());
+        let buffer = &mut RECORDER.as_mut().unwrap().buffer;
+        for word in words.as_ref().iter() {
+            buffer.extend_from_slice(&[
+                (word >> 0) as u8,
+                (word >> 8) as u8,
+                (word >> 16) as u8,
+                (word >> 24) as u8,
+            ]);
+        }
+    }
+}
+
+pub extern "C" fn dma_erase(name: CSlice<u8>) {
+    let name = String::from_utf8(name.as_ref().to_vec()).unwrap();
+    unsafe {
+        KERNEL_CHANNEL_1TO0
+            .as_mut()
+            .unwrap()
+            .send(Message::DmaEraseRequest(name));
+    }
+}
+
+pub extern "C" fn dma_retrieve(name: CSlice<u8>) -> DmaTrace {
+    let name = String::from_utf8(name.as_ref().to_vec()).unwrap();
+    unsafe {
+        KERNEL_CHANNEL_1TO0.as_mut().unwrap().send(Message::DmaGetRequest(name));
+    }
+    match unsafe { KERNEL_CHANNEL_0TO1.as_mut().unwrap() }.recv() {
+        Message::DmaGetReply(None) => (),
+        Message::DmaGetReply(Some((address, duration, uses_ddma))) => {
+            return DmaTrace {
+                address,
+                duration,
+                uses_ddma,
+            };
+        }
+        _ => panic!("Expected DmaGetReply after DmaGetRequest!"),
+    }
+    // we have to defer raising error as we have to drop the message first...
+    artiq_raise!("DMAError", "DMA trace not found");
+}
+
+pub extern "C" fn dma_playback(timestamp: i64, ptr: i32, _uses_ddma: bool) {
+    unsafe {
+        csr::rtio_dma::base_address_write(ptr as u32);
+        csr::rtio_dma::time_offset_write(timestamp as u64);
+
+        let old_cri_master = csr::cri_con::selected_read();
+        csr::cri_con::selected_write(1);
+        csr::rtio_dma::enable_write(1);
+        #[cfg(has_drtio)]
+        if _uses_ddma {
+            KERNEL_CHANNEL_1TO0
+                .as_mut()
+                .unwrap()
+                .send(Message::DmaStartRemoteRequest {
+                    id: ptr,
+                    timestamp: timestamp,
+                });
+        }
+        while csr::rtio_dma::enable_read() != 0 {}
+        csr::cri_con::selected_write(old_cri_master);
+
+        let error = csr::rtio_dma::error_read();
+        if error != 0 {
+            let timestamp = csr::rtio_dma::error_timestamp_read();
+            let channel = csr::rtio_dma::error_channel_read();
+            csr::rtio_dma::error_write(1);
+            if error & 1 != 0 {
+                artiq_raise!(
+                    "RTIOUnderflow",
+                    "RTIO underflow at {1} mu, channel {rtio_channel_info:0}",
+                    channel as i64,
+                    timestamp as i64,
+                    0
+                );
+            }
+            if error & 2 != 0 {
+                artiq_raise!(
+                    "RTIODestinationUnreachable",
+                    "RTIO destination unreachable, output, at {1} mu, channel {rtio_channel_info:0}",
+                    channel as i64,
+                    timestamp as i64,
+                    0
+                );
+            }
+        }
+        #[cfg(has_drtio)]
+        if _uses_ddma {
+            KERNEL_CHANNEL_1TO0
+                .as_mut()
+                .unwrap()
+                .send(Message::DmaAwaitRemoteRequest(ptr));
+            match KERNEL_CHANNEL_0TO1.as_mut().unwrap().recv() {
+                Message::DmaAwaitRemoteReply {
+                    timeout,
+                    error,
+                    channel,
+                    timestamp,
+                } => {
+                    if timeout {
+                        artiq_raise!(
+                            "DMAError",
+                            "Error running DMA on satellite device, timed out waiting for results"
+                        );
+                    }
+                    if error & 1 != 0 {
+                        artiq_raise!(
+                            "RTIOUnderflow",
+                            "RTIO underflow at {1} mu, channel {rtio_channel_info:0}",
+                            channel as i64,
+                            timestamp as i64,
+                            0
+                        );
+                    }
+                    if error & 2 != 0 {
+                        artiq_raise!(
+                            "RTIODestinationUnreachable",
+                            "RTIO destination unreachable, output, at {1} mu, channel {rtio_channel_info:0}",
+                            channel as i64,
+                            timestamp as i64,
+                            0
+                        );
+                    }
+                }
+                _ => panic!("Expected DmaAwaitRemoteReply after DmaAwaitRemoteRequest!"),
+            }
+        }
+    }
+}

src/libksupport/src/kernel/linalg.rs

@@ -0,0 +1,440 @@
+// Uses `nalgebra` crate to invoke `np_linalg` and `sp_linalg` functions
+// When converting between `nalgebra::Matrix` and `NDArray` following considerations are necessary
+//
+// * Both `nalgebra::Matrix` and `NDArray` require their content to be stored in row-major order
+// * `NDArray` data pointer can be directly read and converted to `nalgebra::Matrix` (row and column number must be known)
+// * `nalgebra::Matrix::as_slice` returns the content of matrix in column-major order and initial data needs to be transposed before storing it in `NDArray` data pointer
+
+use alloc::vec::Vec;
+use core::slice;
+
+use nalgebra::DMatrix;
+
+use crate::artiq_raise;
+
+pub struct InputMatrix {
+    pub ndims: usize,
+    pub dims: *const usize,
+    pub data: *mut f64,
+}
+
+impl InputMatrix {
+    fn get_dims(&mut self) -> Vec<usize> {
+        let dims = unsafe { slice::from_raw_parts(self.dims, self.ndims) };
+        dims.to_vec()
+    }
+}
+
+/// # Safety
+///
+/// `mat1` should point to a valid 2DArray of `f64` floats in row-major order
+#[no_mangle]
+pub unsafe extern "C" fn np_linalg_cholesky(mat1: *mut InputMatrix, out: *mut InputMatrix) {
+    let mat1 = mat1.as_mut().unwrap();
+    let out = out.as_mut().unwrap();
+
+    if mat1.ndims != 2 {
+        artiq_raise!(
+            "ValueError",
+            "expected 2D Vector Input, but received {1}D input)",
+            0,
+            mat1.ndims as i64,
+            0
+        );
+    }
+
+    let dim1 = (*mat1).get_dims();
+    if dim1[0] != dim1[1] {
+        artiq_raise!(
+            "ValueError",
+            "last 2 dimensions of the array must be square: {1} != {2}",
+            0,
+            dim1[0] as i64,
+            dim1[1] as i64
+        );
+    }
+
+    let outdim = out.get_dims();
+    let out_slice = slice::from_raw_parts_mut(out.data, outdim[0] * outdim[1]);
+    let data_slice1 = slice::from_raw_parts_mut(mat1.data, dim1[0] * dim1[1]);
+
+    let matrix1 = DMatrix::from_row_slice(dim1[0], dim1[1], data_slice1);
+    let result = matrix1.cholesky();
+    match result {
+        Some(res) => {
+            out_slice.copy_from_slice(res.unpack().transpose().as_slice());
+        }
+        None => {
+            artiq_raise!("LinAlgError", "Matrix is not positive definite");
+        }
+    };
+}
+
+/// # Safety
+///
+/// `mat1` should point to a valid 2DArray of `f64` floats in row-major order
+#[no_mangle]
+pub unsafe extern "C" fn np_linalg_qr(mat1: *mut InputMatrix, out_q: *mut InputMatrix, out_r: *mut InputMatrix) {
+    let mat1 = mat1.as_mut().unwrap();
+    let out_q = out_q.as_mut().unwrap();
+    let out_r = out_r.as_mut().unwrap();
+
+    if mat1.ndims != 2 {
+        artiq_raise!(
+            "ValueError",
+            "expected 2D Vector Input, but received {1}D input)",
+            0,
+            mat1.ndims as i64,
+            0
+        );
+    }
+
+    let dim1 = (*mat1).get_dims();
+    let outq_dim = (*out_q).get_dims();
+    let outr_dim = (*out_r).get_dims();
+
+    let data_slice1 = slice::from_raw_parts_mut(mat1.data, dim1[0] * dim1[1]);
+    let out_q_slice = slice::from_raw_parts_mut(out_q.data, outq_dim[0] * outq_dim[1]);
+    let out_r_slice = slice::from_raw_parts_mut(out_r.data, outr_dim[0] * outr_dim[1]);
+
+    // Refer to https://github.com/dimforge/nalgebra/issues/735
+    let matrix1 = DMatrix::from_row_slice(dim1[0], dim1[1], data_slice1);
+
+    let res = matrix1.qr();
+    let (q, r) = res.unpack();
+
+    // Uses different algo need to match numpy
+    out_q_slice.copy_from_slice(q.transpose().as_slice());
+    out_r_slice.copy_from_slice(r.transpose().as_slice());
+}
+
+/// # Safety
+///
+/// `mat1` should point to a valid 2DArray of `f64` floats in row-major order
+#[no_mangle]
+pub unsafe extern "C" fn np_linalg_svd(
+    mat1: *mut InputMatrix,
+    outu: *mut InputMatrix,
+    outs: *mut InputMatrix,
+    outvh: *mut InputMatrix,
+) {
+    let mat1 = mat1.as_mut().unwrap();
+    let outu = outu.as_mut().unwrap();
+    let outs = outs.as_mut().unwrap();
+    let outvh = outvh.as_mut().unwrap();
+
+    if mat1.ndims != 2 {
+        artiq_raise!(
+            "ValueError",
+            "expected 2D Vector Input, but received {1}D input)",
+            0,
+            mat1.ndims as i64,
+            0
+        );
+    }
+
+    let dim1 = (*mat1).get_dims();
+    let outu_dim = (*outu).get_dims();
+    let outs_dim = (*outs).get_dims();
+    let outvh_dim = (*outvh).get_dims();
+
+    let data_slice1 = slice::from_raw_parts_mut(mat1.data, dim1[0] * dim1[1]);
+    let out_u_slice = slice::from_raw_parts_mut(outu.data, outu_dim[0] * outu_dim[1]);
+    let out_s_slice = slice::from_raw_parts_mut(outs.data, outs_dim[0]);
+    let out_vh_slice = slice::from_raw_parts_mut(outvh.data, outvh_dim[0] * outvh_dim[1]);
+
+    let matrix = DMatrix::from_row_slice(dim1[0], dim1[1], data_slice1);
+    let result = matrix.svd(true, true);
+    out_u_slice.copy_from_slice(result.u.unwrap().transpose().as_slice());
+    out_s_slice.copy_from_slice(result.singular_values.as_slice());
+    out_vh_slice.copy_from_slice(result.v_t.unwrap().transpose().as_slice());
+}
+
+/// # Safety
+///
+/// `mat1` should point to a valid 2DArray of `f64` floats in row-major order
+#[no_mangle]
+pub unsafe extern "C" fn np_linalg_inv(mat1: *mut InputMatrix, out: *mut InputMatrix) {
+    let mat1 = mat1.as_mut().unwrap();
+    let out = out.as_mut().unwrap();
+
+    if mat1.ndims != 2 {
+        artiq_raise!(
+            "ValueError",
+            "expected 2D Vector Input, but received {1}D input)",
+            0,
+            mat1.ndims as i64,
+            0
+        );
+    }
+    let dim1 = (*mat1).get_dims();
+
+    if dim1[0] != dim1[1] {
+        artiq_raise!(
+            "ValueError",
+            "last 2 dimensions of the array must be square: {1} != {2}",
+            0,
+            dim1[0] as i64,
+            dim1[1] as i64
+        );
+    }
+
+    let outdim = out.get_dims();
+    let out_slice = slice::from_raw_parts_mut(out.data, outdim[0] * outdim[1]);
+    let data_slice1 = slice::from_raw_parts_mut(mat1.data, dim1[0] * dim1[1]);
+
+    let matrix = DMatrix::from_row_slice(dim1[0], dim1[1], data_slice1);
+    if !matrix.is_invertible() {
+        artiq_raise!("LinAlgError", "no inverse for Singular Matrix");
+    }
+    let inv = matrix.try_inverse().unwrap();
+    out_slice.copy_from_slice(inv.transpose().as_slice());
+}
+
+/// # Safety
+///
+/// `mat1` should point to a valid 2DArray of `f64` floats in row-major order
+#[no_mangle]
+pub unsafe extern "C" fn np_linalg_pinv(mat1: *mut InputMatrix, out: *mut InputMatrix) {
+    let mat1 = mat1.as_mut().unwrap();
+    let out = out.as_mut().unwrap();
+
+    if mat1.ndims != 2 {
+        artiq_raise!(
+            "ValueError",
+            "expected 2D Vector Input, but received {1}D input)",
+            0,
+            mat1.ndims as i64,
+            0
+        );
+    }
+    let dim1 = (*mat1).get_dims();
+    let outdim = out.get_dims();
+    let out_slice = slice::from_raw_parts_mut(out.data, outdim[0] * outdim[1]);
+    let data_slice1 = slice::from_raw_parts_mut(mat1.data, dim1[0] * dim1[1]);
+
+    let matrix = DMatrix::from_row_slice(dim1[0], dim1[1], data_slice1);
+    let svd = matrix.svd(true, true);
+    let inv = svd.pseudo_inverse(1e-15);
+
+    match inv {
+        Ok(m) => {
+            out_slice.copy_from_slice(m.transpose().as_slice());
+        }
+        Err(_) => {
+            artiq_raise!("LinAlgError", "SVD computation does not converge");
+        }
+    }
+}
+
+/// # Safety
+///
+/// `mat1` should point to a valid 2DArray of `f64` floats in row-major order
+#[no_mangle]
+pub unsafe extern "C" fn np_linalg_matrix_power(mat1: *mut InputMatrix, mat2: *mut InputMatrix, out: *mut InputMatrix) {
+    let mat1 = mat1.as_mut().unwrap();
+    let mat2 = mat2.as_mut().unwrap();
+    let out = out.as_mut().unwrap();
+
+    if mat1.ndims != 2 {
+        artiq_raise!(
+            "ValueError",
+            "expected 2D Vector Input, but received {1}D input)",
+            0,
+            mat1.ndims as i64,
+            0
+        );
+    }
+
+    let dim1 = (*mat1).get_dims();
+    let power = slice::from_raw_parts_mut(mat2.data, 1);
+    let power = power[0];
+    let outdim = out.get_dims();
+    let out_slice = slice::from_raw_parts_mut(out.data, outdim[0] * outdim[1]);
+    let data_slice1 = slice::from_raw_parts_mut(mat1.data, dim1[0] * dim1[1]);
+    let mut abs_power = power;
+    if abs_power < 0.0 {
+        abs_power = abs_power * -1.0;
+    }
+    let matrix1 = DMatrix::from_row_slice(dim1[0], dim1[1], data_slice1);
+    if !matrix1.is_square() {
+        artiq_raise!(
+            "ValueError",
+            "last 2 dimensions of the array must be square: {1} != {2}",
+            0,
+            dim1[0] as i64,
+            dim1[1] as i64
+        );
+    }
+    let mut result = matrix1.pow(abs_power as u32);
+
+    if power < 0.0 {
+        if !matrix1.is_invertible() {
+            artiq_raise!("LinAlgError", "no inverse for Singular Matrix");
+        }
+        result = result.try_inverse().unwrap();
+    }
+    out_slice.copy_from_slice(result.transpose().as_slice());
+}
+
+/// # Safety
+///
+/// `mat1` should point to a valid 2DArray of `f64` floats in row-major order
+#[no_mangle]
+pub unsafe extern "C" fn np_linalg_det(mat1: *mut InputMatrix, out: *mut InputMatrix) {
+    let mat1 = mat1.as_mut().unwrap();
+    let out = out.as_mut().unwrap();
+
+    if mat1.ndims != 2 {
+        artiq_raise!(
+            "ValueError",
+            "expected 2D Vector Input, but received {1}D input)",
+            0,
+            mat1.ndims as i64,
+            0
+        );
+    }
+    let dim1 = (*mat1).get_dims();
+    let out_slice = slice::from_raw_parts_mut(out.data, 1);
+    let data_slice1 = slice::from_raw_parts_mut(mat1.data, dim1[0] * dim1[1]);
+
+    let matrix = DMatrix::from_row_slice(dim1[0], dim1[1], data_slice1);
+    if !matrix.is_square() {
+        artiq_raise!(
+            "ValueError",
+            "last 2 dimensions of the array must be square: {1} != {2}",
+            0,
+            dim1[0] as i64,
+            dim1[1] as i64
+        );
+    }
+    out_slice[0] = matrix.determinant();
+}
+
+/// # Safety
+///
+/// `mat1` should point to a valid 2DArray of `f64` floats in row-major order
+#[no_mangle]
+pub unsafe extern "C" fn sp_linalg_lu(mat1: *mut InputMatrix, out_l: *mut InputMatrix, out_u: *mut InputMatrix) {
+    let mat1 = mat1.as_mut().unwrap();
+    let out_l = out_l.as_mut().unwrap();
+    let out_u = out_u.as_mut().unwrap();
+
+    if mat1.ndims != 2 {
+        artiq_raise!(
+            "ValueError",
+            "expected 2D Vector Input, but received {1}D input)",
+            0,
+            mat1.ndims as i64,
+            0
+        );
+    }
+
+    let dim1 = (*mat1).get_dims();
+    let outl_dim = (*out_l).get_dims();
+    let outu_dim = (*out_u).get_dims();
+
+    let data_slice1 = slice::from_raw_parts_mut(mat1.data, dim1[0] * dim1[1]);
+    let out_l_slice = slice::from_raw_parts_mut(out_l.data, outl_dim[0] * outl_dim[1]);
+    let out_u_slice = slice::from_raw_parts_mut(out_u.data, outu_dim[0] * outu_dim[1]);
+
+    let matrix = DMatrix::from_row_slice(dim1[0], dim1[1], data_slice1);
+    let (_, l, u) = matrix.lu().unpack();
+
+    out_l_slice.copy_from_slice(l.transpose().as_slice());
+    out_u_slice.copy_from_slice(u.transpose().as_slice());
+}
+
+/// # Safety
+///
+/// `mat1` should point to a valid 2DArray of `f64` floats in row-major order
+#[no_mangle]
+pub unsafe extern "C" fn sp_linalg_schur(mat1: *mut InputMatrix, out_t: *mut InputMatrix, out_z: *mut InputMatrix) {
+    let mat1 = mat1.as_mut().unwrap();
+    let out_t = out_t.as_mut().unwrap();
+    let out_z = out_z.as_mut().unwrap();
+
+    if mat1.ndims != 2 {
+        artiq_raise!(
+            "ValueError",
+            "expected 2D Vector Input, but received {1}D input)",
+            0,
+            mat1.ndims as i64,
+            0
+        );
+    }
+
+    let dim1 = (*mat1).get_dims();
+
+    if dim1[0] != dim1[1] {
+        artiq_raise!(
+            "ValueError",
+            "last 2 dimensions of the array must be square: {1} != {2}",
+            0,
+            dim1[0] as i64,
+            dim1[1] as i64
+        );
+    }
+
+    let out_t_dim = (*out_t).get_dims();
+    let out_z_dim = (*out_z).get_dims();
+
+    let data_slice1 = slice::from_raw_parts_mut(mat1.data, dim1[0] * dim1[1]);
+    let out_t_slice = slice::from_raw_parts_mut(out_t.data, out_t_dim[0] * out_t_dim[1]);
+    let out_z_slice = slice::from_raw_parts_mut(out_z.data, out_z_dim[0] * out_z_dim[1]);
+
+    let matrix = DMatrix::from_row_slice(dim1[0], dim1[1], data_slice1);
+    let (z, t) = matrix.schur().unpack();
+
+    out_t_slice.copy_from_slice(t.transpose().as_slice());
+    out_z_slice.copy_from_slice(z.transpose().as_slice());
+}
+
+/// # Safety
+///
+/// `mat1` should point to a valid 2DArray of `f64` floats in row-major order
+#[no_mangle]
+pub unsafe extern "C" fn sp_linalg_hessenberg(
+    mat1: *mut InputMatrix,
+    out_h: *mut InputMatrix,
+    out_q: *mut InputMatrix,
+) {
+    let mat1 = mat1.as_mut().unwrap();
+    let out_h = out_h.as_mut().unwrap();
+    let out_q = out_q.as_mut().unwrap();
+
+    if mat1.ndims != 2 {
+        artiq_raise!(
+            "ValueError",
+            "expected 2D Vector Input, but received {1}D input)",
+            0,
+            mat1.ndims as i64,
+            0
+        );
+    }
+
+    let dim1 = (*mat1).get_dims();
+
+    if dim1[0] != dim1[1] {
+        artiq_raise!(
+            "ValueError",
+            "last 2 dimensions of the array must be square: {1} != {2}",
+            0,
+            dim1[0] as i64,
+            dim1[1] as i64
+        );
+    }
+
+    let out_h_dim = (*out_h).get_dims();
+    let out_q_dim = (*out_q).get_dims();
+
+    let data_slice1 = slice::from_raw_parts_mut(mat1.data, dim1[0] * dim1[1]);
+    let out_h_slice = slice::from_raw_parts_mut(out_h.data, out_h_dim[0] * out_h_dim[1]);
+    let out_q_slice = slice::from_raw_parts_mut(out_q.data, out_q_dim[0] * out_q_dim[1]);
+
+    let matrix = DMatrix::from_row_slice(dim1[0], dim1[1], data_slice1);
+    let (q, h) = matrix.hessenberg().unpack();
+
+    out_h_slice.copy_from_slice(h.transpose().as_slice());
+    out_q_slice.copy_from_slice(q.transpose().as_slice());
+}

src/libksupport/src/kernel/mod.rs

@@ -0,0 +1,128 @@
+use alloc::{string::String, vec::Vec};
+use core::ptr;
+
+use libcortex_a9::{mutex::Mutex, semaphore::Semaphore, sync_channel};
+
+use crate::{eh_artiq, RPCException};
+
+mod control;
+pub use control::Control;
+mod api;
+pub mod core1;
+mod dma;
+mod rpc;
+pub use dma::DmaRecorder;
+mod cache;
+mod linalg;
+#[cfg(has_drtio)]
+mod subkernel;
+
+#[cfg(has_drtio)]
+#[derive(Debug, Clone)]
+pub enum SubkernelStatus {
+    NoError,
+    Timeout,
+    IncorrectState,
+    CommLost,
+    Exception(Vec<u8>),
+    OtherError,
+}
+
+#[derive(Debug, Clone)]
+pub enum Message {
+    LoadRequest(Vec<u8>),
+    LoadCompleted,
+    LoadFailed,
+    StartRequest,
+    KernelFinished(u8),
+    KernelException(
+        &'static [Option<eh_artiq::Exception<'static>>],
+        &'static [eh_artiq::StackPointerBacktrace],
+        &'static [(usize, usize)],
+        u8,
+    ),
+    RpcSend {
+        is_async: bool,
+        data: Vec<u8>,
+    },
+    RpcRecvRequest(*mut ()),
+    RpcRecvReply(Result<usize, RPCException>),
+
+    CacheGetRequest(String),
+    CacheGetReply(Vec<i32>),
+    CachePutRequest(String, Vec<i32>),
+
+    DmaPutRequest(DmaRecorder),
+    DmaEraseRequest(String),
+    DmaGetRequest(String),
+    DmaGetReply(Option<(i32, i64, bool)>),
+    #[cfg(has_drtio)]
+    DmaStartRemoteRequest {
+        id: i32,
+        timestamp: i64,
+    },
+    #[cfg(has_drtio)]
+    DmaAwaitRemoteRequest(i32),
+    #[cfg(has_drtio)]
+    DmaAwaitRemoteReply {
+        timeout: bool,
+        error: u8,
+        channel: u32,
+        timestamp: u64,
+    },
+
+    #[cfg(has_drtio)]
+    UpDestinationsRequest(i32),
+    #[cfg(has_drtio)]
+    UpDestinationsReply(bool),
+
+    #[cfg(has_drtio)]
+    SubkernelLoadRunRequest {
+        id: u32,
+        destination: u8,
+        run: bool,
+        timestamp: u64,
+    },
+    #[cfg(has_drtio)]
+    SubkernelLoadRunReply {
+        succeeded: bool,
+    },
+    #[cfg(has_drtio)]
+    SubkernelAwaitFinishRequest {
+        id: u32,
+        timeout: i64,
+    },
+    #[cfg(has_drtio)]
+    SubkernelAwaitFinishReply,
+    #[cfg(has_drtio)]
+    SubkernelMsgSend {
+        id: u32,
+        destination: Option<u8>,
+        data: Vec<u8>,
+    },
+    #[cfg(has_drtio)]
+    SubkernelMsgSent,
+    #[cfg(has_drtio)]
+    SubkernelMsgRecvRequest {
+        id: i32,
+        timeout: i64,
+        tags: Vec<u8>,
+    },
+    #[cfg(has_drtio)]
+    SubkernelMsgRecvReply {
+        count: u8,
+    },
+    #[cfg(has_drtio)]
+    SubkernelError(SubkernelStatus),
+}
+
+static CHANNEL_0TO1: Mutex<Option<sync_channel::Sender<'static, Message>>> = Mutex::new(None);
+static CHANNEL_1TO0: Mutex<Option<sync_channel::Receiver<'static, Message>>> = Mutex::new(None);
+static CHANNEL_SEM: Semaphore = Semaphore::new(0, 1);
+
+static mut KERNEL_CHANNEL_0TO1: Option<sync_channel::Receiver<'static, Message>> = None;
+static mut KERNEL_CHANNEL_1TO0: Option<sync_channel::Sender<'static, Message>> = None;
+
+pub static mut KERNEL_IMAGE: *const core1::KernelImage = ptr::null();
+
+static INIT_LOCK: Mutex<()> = Mutex::new(());

src/libksupport/src/kernel/rpc.rs

@@ -0,0 +1,47 @@
+//! Kernel-side RPC API
+
+use alloc::vec::Vec;
+
+use cslice::CSlice;
+
+use super::{Message, KERNEL_CHANNEL_0TO1, KERNEL_CHANNEL_1TO0};
+use crate::{eh_artiq, rpc::send_args};
+
+fn rpc_send_common(is_async: bool, service: u32, tag: &CSlice<u8>, data: *const *const ()) {
+    let core1_tx = unsafe { KERNEL_CHANNEL_1TO0.as_mut().unwrap() };
+    let mut buffer = Vec::<u8>::new();
+    send_args(&mut buffer, service, tag.as_ref(), data, true).expect("RPC encoding failed");
+    core1_tx.send(Message::RpcSend { is_async, data: buffer });
+}
+
+pub extern "C" fn rpc_send(service: u32, tag: &CSlice<u8>, data: *const *const ()) {
+    rpc_send_common(false, service, tag, data);
+}
+
+pub extern "C" fn rpc_send_async(service: u32, tag: &CSlice<u8>, data: *const *const ()) {
+    rpc_send_common(true, service, tag, data);
+}
+
+pub extern "C" fn rpc_recv(slot: *mut ()) -> usize {
+    let reply = unsafe {
+        let core1_rx = KERNEL_CHANNEL_0TO1.as_mut().unwrap();
+        let core1_tx = KERNEL_CHANNEL_1TO0.as_mut().unwrap();
+        core1_tx.send(Message::RpcRecvRequest(slot));
+        core1_rx.recv()
+    };
+    match reply {
+        Message::RpcRecvReply(Ok(alloc_size)) => alloc_size,
+        Message::RpcRecvReply(Err(exception)) => unsafe {
+            eh_artiq::raise(&eh_artiq::Exception {
+                id: exception.id,
+                file: CSlice::new(exception.file as *const u8, usize::MAX),
+                line: exception.line as u32,
+                column: exception.column as u32,
+                function: CSlice::new(exception.function as *const u8, usize::MAX),
+                message: CSlice::new(exception.message as *const u8, usize::MAX),
+                param: exception.param,
+            })
+        },
+        _ => panic!("received unexpected reply to RpcRecvRequest: {:?}", reply),
+    }
+}

src/libksupport/src/kernel/subkernel.rs

@@ -0,0 +1,112 @@
+use alloc::vec::Vec;
+
+use cslice::CSlice;
+
+use super::{Message, SubkernelStatus, KERNEL_CHANNEL_0TO1, KERNEL_CHANNEL_1TO0};
+use crate::{artiq_raise, eh_artiq, rpc::send_args, rtio::now_mu};
+
+pub extern "C" fn load_run(id: u32, destination: u8, run: bool) {
+    unsafe {
+        KERNEL_CHANNEL_1TO0
+            .as_mut()
+            .unwrap()
+            .send(Message::SubkernelLoadRunRequest {
+                id: id,
+                destination: destination,
+                run: run,
+                timestamp: now_mu() as u64,
+            });
+    }
+    match unsafe { KERNEL_CHANNEL_0TO1.as_mut().unwrap() }.recv() {
+        Message::SubkernelLoadRunReply { succeeded: true } => (),
+        Message::SubkernelLoadRunReply { succeeded: false } => {
+            artiq_raise!("SubkernelError", "Error loading or running the subkernel")
+        }
+        _ => panic!("Expected SubkernelLoadRunReply after SubkernelLoadRunRequest!"),
+    }
+}
+
+pub extern "C" fn await_finish(id: u32, timeout: i64) {
+    unsafe {
+        KERNEL_CHANNEL_1TO0
+            .as_mut()
+            .unwrap()
+            .send(Message::SubkernelAwaitFinishRequest {
+                id: id,
+                timeout: timeout,
+            });
+    }
+    match unsafe { KERNEL_CHANNEL_0TO1.as_mut().unwrap() }.recv() {
+        Message::SubkernelAwaitFinishReply => (),
+        Message::SubkernelError(SubkernelStatus::IncorrectState) => {
+            artiq_raise!("SubkernelError", "Subkernel not running")
+        }
+        Message::SubkernelError(SubkernelStatus::Timeout) => artiq_raise!("SubkernelError", "Subkernel timed out"),
+        Message::SubkernelError(SubkernelStatus::CommLost) => {
+            artiq_raise!("SubkernelError", "Lost communication with satellite")
+        }
+        Message::SubkernelError(SubkernelStatus::OtherError) => {
+            artiq_raise!("SubkernelError", "An error occurred during subkernel operation")
+        }
+        Message::SubkernelError(SubkernelStatus::Exception(raw_exception)) => eh_artiq::raise_raw(&raw_exception),
+        _ => panic!("expected SubkernelAwaitFinishReply after SubkernelAwaitFinishRequest"),
+    }
+}
+
+pub extern "C" fn send_message(
+    id: u32,
+    is_return: bool,
+    destination: u8,
+    count: u8,
+    tag: &CSlice<u8>,
+    data: *const *const (),
+) {
+    let mut buffer = Vec::<u8>::new();
+    send_args(&mut buffer, 0, tag.as_ref(), data, false).expect("RPC encoding failed");
+    // overwrite service tag, include how many tags are in the message
+    buffer[3] = count;
+    unsafe {
+        KERNEL_CHANNEL_1TO0.as_mut().unwrap().send(Message::SubkernelMsgSend {
+            id: id,
+            destination: if is_return { None } else { Some(destination) },
+            data: buffer[3..].to_vec(),
+        });
+    }
+    match unsafe { KERNEL_CHANNEL_0TO1.as_mut().unwrap() }.recv() {
+        Message::SubkernelMsgSent => (),
+        _ => panic!("expected SubkernelMsgSent after SubkernelMsgSend"),
+    }
+}
+
+pub extern "C" fn await_message(id: i32, timeout: i64, tags: &CSlice<u8>, min: u8, max: u8) {
+    unsafe {
+        KERNEL_CHANNEL_1TO0
+            .as_mut()
+            .unwrap()
+            .send(Message::SubkernelMsgRecvRequest {
+                id: id,
+                timeout: timeout,
+                tags: tags.as_ref().to_vec(),
+            });
+    }
+    match unsafe { KERNEL_CHANNEL_0TO1.as_mut().unwrap() }.recv() {
+        Message::SubkernelMsgRecvReply { count } => {
+            if min > count || count > max {
+                artiq_raise!("SubkernelError", "Received more or less arguments than required")
+            }
+        }
+        Message::SubkernelError(SubkernelStatus::IncorrectState) => {
+            artiq_raise!("SubkernelError", "Subkernel not running")
+        }
+        Message::SubkernelError(SubkernelStatus::Timeout) => artiq_raise!("SubkernelError", "Subkernel timed out"),
+        Message::SubkernelError(SubkernelStatus::CommLost) => {
+            artiq_raise!("SubkernelError", "Lost communication with satellite")
+        }
+        Message::SubkernelError(SubkernelStatus::OtherError) => {
+            artiq_raise!("SubkernelError", "An error occurred during subkernel operation")
+        }
+        Message::SubkernelError(SubkernelStatus::Exception(raw_exception)) => eh_artiq::raise_raw(&raw_exception),
+        _ => panic!("expected SubkernelMsgRecvReply after SubkernelMsgRecvRequest"),
+    }
+    // RpcRecvRequest should be called after this to receive message data
+}

src/libksupport/src/lib.rs

@@ -0,0 +1,163 @@
+#![no_std]
+#![feature(c_variadic)]
+#![feature(const_btree_new)]
+#![feature(const_in_array_repeat_expressions)]
+#![feature(naked_functions)]
+#![feature(asm)]
+
+#[macro_use]
+extern crate alloc;
+
+use alloc::{collections::BTreeMap, string::String};
+
+use io::{Cursor, ProtoRead};
+use libasync::block_async;
+use libconfig::Config;
+use log::{error, warn};
+#[cfg(has_drtiosat)]
+pub use pl::csr::drtiosat as rtio_core;
+#[cfg(has_rtio_core)]
+pub use pl::csr::rtio_core;
+use void::Void;
+
+pub mod eh_artiq;
+pub mod i2c;
+pub mod irq;
+pub mod kernel;
+pub mod rpc;
+#[cfg(ki_impl = "csr")]
+#[path = "rtio_csr.rs"]
+pub mod rtio;
+#[cfg(ki_impl = "acp")]
+#[path = "rtio_acp.rs"]
+pub mod rtio;
+#[rustfmt::skip]
+#[path = "../../../build/pl.rs"]
+pub mod pl;
+
+
+#[derive(Debug, Clone)]
+pub struct RPCException {
+    pub id: u32,
+    pub message: u32,
+    pub param: [i64; 3],
+    pub file: u32,
+    pub line: i32,
+    pub column: i32,
+    pub function: u32,
+}
+
+pub static mut SEEN_ASYNC_ERRORS: u8 = 0;
+
+pub const ASYNC_ERROR_COLLISION: u8 = 1 << 0;
+pub const ASYNC_ERROR_BUSY: u8 = 1 << 1;
+pub const ASYNC_ERROR_SEQUENCE_ERROR: u8 = 1 << 2;
+
+pub unsafe fn get_async_errors() -> u8 {
+    let errors = SEEN_ASYNC_ERRORS;
+    SEEN_ASYNC_ERRORS = 0;
+    errors
+}
+
+fn wait_for_async_rtio_error() -> nb::Result<(), Void> {
+    unsafe {
+        #[cfg(has_rtio_core)]
+        let errors = rtio_core::async_error_read();
+        #[cfg(has_drtiosat)]
+        let errors = rtio_core::protocol_error_read();
+        if errors != 0 {
+            Ok(())
+        } else {
+            Err(nb::Error::WouldBlock)
+        }
+    }
+}
+
+pub async fn report_async_rtio_errors() {
+    loop {
+        let _ = block_async!(wait_for_async_rtio_error()).await;
+        unsafe {
+            #[cfg(has_rtio_core)]
+            let errors = rtio_core::async_error_read();
+            #[cfg(has_drtiosat)]
+            let errors = rtio_core::protocol_error_read();
+            if errors & ASYNC_ERROR_COLLISION != 0 {
+                let channel = rtio_core::collision_channel_read();
+                error!(
+                    "RTIO collision involving channel 0x{:04x}:{}",
+                    channel,
+                    resolve_channel_name(channel as u32)
+                );
+            }
+            if errors & ASYNC_ERROR_BUSY != 0 {
+                let channel = rtio_core::busy_channel_read();
+                error!(
+                    "RTIO busy error involving channel 0x{:04x}:{}",
+                    channel,
+                    resolve_channel_name(channel as u32)
+                );
+            }
+            if errors & ASYNC_ERROR_SEQUENCE_ERROR != 0 {
+                let channel = rtio_core::sequence_error_channel_read();
+                error!(
+                    "RTIO sequence error involving channel 0x{:04x}:{}",
+                    channel,
+                    resolve_channel_name(channel as u32)
+                );
+            }
+            SEEN_ASYNC_ERRORS = errors;
+            #[cfg(has_rtio_core)]
+            rtio_core::async_error_write(errors);
+            #[cfg(has_drtiosat)]
+            rtio_core::protocol_error_write(errors);
+        }
+    }
+}
+
+static mut RTIO_DEVICE_MAP: BTreeMap<u32, String> = BTreeMap::new();
+
+fn read_device_map(cfg: &Config) -> BTreeMap<u32, String> {
+    let mut device_map: BTreeMap<u32, String> = BTreeMap::new();
+    let _ = cfg
+        .read("device_map")
+        .and_then(|raw_bytes| {
+            let mut bytes_cr = Cursor::new(raw_bytes);
+            let size = bytes_cr.read_u32().unwrap();
+            for _ in 0..size {
+                let channel = bytes_cr.read_u32().unwrap();
+                let device_name = bytes_cr.read_string().unwrap();
+                if let Some(old_entry) = device_map.insert(channel, device_name.clone()) {
+                    warn!(
+                        "conflicting device map entries for RTIO channel {}: '{}' and '{}'",
+                        channel, old_entry, device_name
+                    );
+                }
+            }
+            Ok(())
+        })
+        .or_else(|err| {
+            warn!(
+                "error reading device map ({}), device names will not be available in RTIO error messages",
+                err
+            );
+            Err(err)
+        });
+    device_map
+}
+
+fn _resolve_channel_name(channel: u32, device_map: &BTreeMap<u32, String>) -> String {
+    match device_map.get(&channel) {
+        Some(val) => val.clone(),
+        None => String::from("unknown"),
+    }
+}
+
+pub fn resolve_channel_name(channel: u32) -> String {
+    _resolve_channel_name(channel, unsafe { &RTIO_DEVICE_MAP })
+}
+
+pub fn setup_device_map(cfg: &Config) {
+    unsafe {
+        RTIO_DEVICE_MAP = read_device_map(cfg);
+    }
+}

src/libksupport/src/rpc.rs

@@ -0,0 +1,591 @@
+use core::str;
+
+use byteorder::{ByteOrder, NativeEndian};
+use core_io::{Error, Read, Write};
+use cslice::{CMutSlice, CSlice};
+use io::{ProtoRead, ProtoWrite};
+use log::trace;
+
+use self::tag::{split_tag, Tag, TagIterator};
+
+#[inline]
+pub fn round_up(val: usize, power_of_two: usize) -> usize {
+    assert!(power_of_two.is_power_of_two());
+    let max_rem = power_of_two - 1;
+    (val + max_rem) & (!max_rem)
+}
+
+#[inline]
+pub unsafe fn round_up_mut<T>(ptr: *mut T, power_of_two: usize) -> *mut T {
+    round_up(ptr as usize, power_of_two) as *mut T
+}
+
+#[inline]
+pub unsafe fn round_up_const<T>(ptr: *const T, power_of_two: usize) -> *const T {
+    round_up(ptr as usize, power_of_two) as *const T
+}
+
+#[inline]
+pub unsafe fn align_ptr<T>(ptr: *const ()) -> *const T {
+    round_up_const(ptr, core::mem::align_of::<T>()) as *const T
+}
+
+#[inline]
+pub unsafe fn align_ptr_mut<T>(ptr: *mut ()) -> *mut T {
+    round_up_mut(ptr, core::mem::align_of::<T>()) as *mut T
+}
+
+// versions for reader rather than TcpStream
+// they will be made into sync for satellite subkernels later
+unsafe fn recv_elements<F, R>(
+    reader: &mut R,
+    elt_tag: Tag,
+    length: usize,
+    storage: *mut (),
+    alloc: &mut F,
+) -> Result<(), Error>
+where
+    F: FnMut(usize) -> *mut (),
+    R: Read + ?Sized,
+{
+    match elt_tag {
+        Tag::Bool => {
+            let dest = core::slice::from_raw_parts_mut(storage as *mut u8, length);
+            reader.read_exact(dest)?;
+        }
+        Tag::Int32 => {
+            let ptr = storage as *mut u32;
+            let dest = core::slice::from_raw_parts_mut(ptr as *mut u8, length * 4);
+            reader.read_exact(dest)?;
+            drop(dest);
+            let dest = core::slice::from_raw_parts_mut(ptr, length);
+            NativeEndian::from_slice_u32(dest);
+        }
+        Tag::Int64 | Tag::Float64 => {
+            let ptr = storage as *mut u64;
+            let dest = core::slice::from_raw_parts_mut(ptr as *mut u8, length * 8);
+            reader.read_exact(dest)?;
+            drop(dest);
+            let dest = core::slice::from_raw_parts_mut(ptr, length);
+            NativeEndian::from_slice_u64(dest);
+        }
+        _ => {
+            let mut data = storage;
+            for _ in 0..length {
+                recv_value(reader, elt_tag, &mut data, alloc)?
+            }
+        }
+    }
+    Ok(())
+}
+
+unsafe fn recv_value<F, R>(reader: &mut R, tag: Tag, data: &mut *mut (), alloc: &mut F) -> Result<(), Error>
+where
+    F: FnMut(usize) -> *mut (),
+    R: Read + ?Sized,
+{
+    macro_rules! consume_value {
+        ($ty:ty, | $ptr:ident | $map:expr) => {{
+            let $ptr = align_ptr_mut::<$ty>(*data);
+            *data = $ptr.offset(1) as *mut ();
+            $map
+        }};
+    }
+
+    match tag {
+        Tag::None => Ok(()),
+        Tag::Bool => consume_value!(i8, |ptr| {
+            *ptr = reader.read_u8()? as i8;
+            Ok(())
+        }),
+        Tag::Int32 => consume_value!(i32, |ptr| {
+            *ptr = reader.read_u32()? as i32;
+            Ok(())
+        }),
+        Tag::Int64 | Tag::Float64 => consume_value!(i64, |ptr| {
+            *ptr = reader.read_u64()? as i64;
+            Ok(())
+        }),
+        Tag::String | Tag::Bytes | Tag::ByteArray => {
+            consume_value!(CMutSlice<u8>, |ptr| {
+                let length = reader.read_u32()? as usize;
+                *ptr = CMutSlice::new(alloc(length) as *mut u8, length);
+                reader.read_exact((*ptr).as_mut())?;
+                Ok(())
+            })
+        }
+        Tag::Tuple(it, arity) => {
+            let alignment = tag.alignment();
+            *data = round_up_mut(*data, alignment);
+            let mut it = it.clone();
+            for _ in 0..arity {
+                let tag = it.next().expect("truncated tag");
+                recv_value(reader, tag, data, alloc)?
+            }
+            *data = round_up_mut(*data, alignment);
+            Ok(())
+        }
+        Tag::List(it) => {
+            #[repr(C)]
+            struct List {
+                elements: *mut (),
+                length: usize,
+            }
+            consume_value!(*mut List, |ptr_to_list| {
+                let tag = it.clone().next().expect("truncated tag");
+                let length = reader.read_u32()? as usize;
+
+                let list_size = 4 + 4;
+                let storage_offset = round_up(list_size, tag.alignment());
+                let storage_size = tag.size() * length;
+
+                let allocation = alloc(storage_offset + storage_size) as *mut u8;
+                *ptr_to_list = allocation as *mut List;
+                let storage = allocation.offset(storage_offset as isize) as *mut ();
+
+                (**ptr_to_list).length = length;
+                (**ptr_to_list).elements = storage;
+                recv_elements(reader, tag, length, storage, alloc)
+            })
+        }
+        Tag::Array(it, num_dims) => {
+            consume_value!(*mut (), |buffer| {
+                let mut total_len: usize = 1;
+                for _ in 0..num_dims {
+                    let len = reader.read_u32()? as usize;
+                    total_len *= len;
+                    consume_value!(usize, |ptr| *ptr = len)
+                }
+
+                let elt_tag = it.clone().next().expect("truncated tag");
+                *buffer = alloc(elt_tag.size() * total_len);
+                recv_elements(reader, elt_tag, total_len, *buffer, alloc)
+            })
+        }
+        Tag::Range(it) => {
+            *data = round_up_mut(*data, tag.alignment());
+            let tag = it.clone().next().expect("truncated tag");
+            recv_value(reader, tag, data, alloc)?;
+            recv_value(reader, tag, data, alloc)?;
+            recv_value(reader, tag, data, alloc)?;
+            Ok(())
+        }
+        Tag::Keyword(_) => unreachable!(),
+        Tag::Object => unreachable!(),
+    }
+}
+
+pub fn recv_return<'a, F, R>(
+    reader: &mut R,
+    tag_bytes: &'a [u8],
+    data: *mut (),
+    alloc: &mut F,
+) -> Result<&'a [u8], Error>
+where
+    F: FnMut(usize) -> *mut (),
+    R: Read + ?Sized,
+{
+    let mut it = TagIterator::new(tag_bytes);
+    trace!("recv ...->{}", it);
+
+    let tag = it.next().expect("truncated tag");
+    let mut data = data;
+    unsafe { recv_value(reader, tag, &mut data, alloc)? };
+
+    Ok(it.data)
+}
+
+unsafe fn send_elements<W>(
+    writer: &mut W,
+    elt_tag: Tag,
+    length: usize,
+    data: *const (),
+    write_tags: bool,
+) -> Result<(), Error>
+where
+    W: Write + ?Sized,
+{
+    if write_tags {
+        writer.write_u8(elt_tag.as_u8())?;
+    }
+    match elt_tag {
+        // we cannot use NativeEndian::from_slice_i32 as the data is not mutable,
+        // and that is not needed as the data is already in native endian
+        Tag::Bool => {
+            let slice = core::slice::from_raw_parts(data as *const u8, length);
+            writer.write_all(slice)?;
+        }
+        Tag::Int32 => {
+            let slice = core::slice::from_raw_parts(data as *const u8, length * 4);
+            writer.write_all(slice)?;
+        }
+        Tag::Int64 | Tag::Float64 => {
+            let slice = core::slice::from_raw_parts(data as *const u8, length * 8);
+            writer.write_all(slice)?;
+        }
+        _ => {
+            let mut data = data;
+            for _ in 0..length {
+                send_value(writer, elt_tag, &mut data, write_tags)?;
+            }
+        }
+    }
+    Ok(())
+}
+
+unsafe fn send_value<W>(writer: &mut W, tag: Tag, data: &mut *const (), write_tags: bool) -> Result<(), Error>
+where W: Write + ?Sized {
+    macro_rules! consume_value {
+        ($ty:ty, | $ptr:ident | $map:expr) => {{
+            let $ptr = align_ptr::<$ty>(*data);
+            *data = $ptr.offset(1) as *const ();
+            $map
+        }};
+    }
+
+    if write_tags {
+        writer.write_u8(tag.as_u8())?;
+    }
+    match tag {
+        Tag::None => Ok(()),
+        Tag::Bool => consume_value!(u8, |ptr| writer.write_u8(*ptr)),
+        Tag::Int32 => consume_value!(u32, |ptr| writer.write_u32(*ptr)),
+        Tag::Int64 | Tag::Float64 => consume_value!(u64, |ptr| writer.write_u64(*ptr)),
+        Tag::String => consume_value!(CSlice<u8>, |ptr| {
+            writer.write_string(str::from_utf8((*ptr).as_ref()).unwrap())
+        }),
+        Tag::Bytes | Tag::ByteArray => consume_value!(CSlice<u8>, |ptr| writer.write_bytes((*ptr).as_ref())),
+        Tag::Tuple(it, arity) => {
+            let mut it = it.clone();
+            if write_tags {
+                writer.write_u8(arity)?;
+            }
+            let mut max_alignment = 0;
+            for _ in 0..arity {
+                let tag = it.next().expect("truncated tag");
+                max_alignment = core::cmp::max(max_alignment, tag.alignment());
+                send_value(writer, tag, data, write_tags)?
+            }
+            *data = round_up_const(*data, max_alignment);
+            Ok(())
+        }
+        Tag::List(it) => {
+            #[repr(C)]
+            struct List {
+                elements: *const (),
+                length: u32,
+            }
+            consume_value!(&List, |ptr| {
+                let length = (**ptr).length as usize;
+                writer.write_u32((*ptr).length)?;
+                let tag = it.clone().next().expect("truncated tag");
+                send_elements(writer, tag, length, (**ptr).elements, write_tags)
+            })
+        }
+        Tag::Array(it, num_dims) => {
+            if write_tags {
+                writer.write_u8(num_dims)?;
+            }
+            consume_value!(*const (), |buffer| {
+                let elt_tag = it.clone().next().expect("truncated tag");
+
+                let mut total_len = 1;
+                for _ in 0..num_dims {
+                    consume_value!(u32, |len| {
+                        writer.write_u32(*len)?;
+                        total_len *= *len;
+                    })
+                }
+                let length = total_len as usize;
+                send_elements(writer, elt_tag, length, *buffer, write_tags)
+            })
+        }
+        Tag::Range(it) => {
+            let tag = it.clone().next().expect("truncated tag");
+            send_value(writer, tag, data, write_tags)?;
+            send_value(writer, tag, data, write_tags)?;
+            send_value(writer, tag, data, write_tags)?;
+            Ok(())
+        }
+        Tag::Keyword(it) => {
+            #[repr(C)]
+            struct Keyword<'a> {
+                name: CSlice<'a, u8>,
+            }
+            consume_value!(Keyword, |ptr| {
+                writer.write_string(str::from_utf8((*ptr).name.as_ref()).unwrap())?;
+                let tag = it.clone().next().expect("truncated tag");
+                let mut data = ptr.offset(1) as *const ();
+                send_value(writer, tag, &mut data, write_tags)
+            })
+            // Tag::Keyword never appears in composite types, so we don't have
+            // to accurately advance data.
+        }
+        Tag::Object => {
+            #[repr(C)]
+            struct Object {
+                id: u32,
+            }
+            consume_value!(*const Object, |ptr| writer.write_u32((**ptr).id))
+        }
+    }
+}
+
+pub fn send_args<W>(
+    writer: &mut W,
+    service: u32,
+    tag_bytes: &[u8],
+    data: *const *const (),
+    write_tags: bool,
+) -> Result<(), Error>
+where
+    W: Write + ?Sized,
+{
+    let (arg_tags_bytes, return_tag_bytes) = split_tag(tag_bytes);
+
+    let mut args_it = TagIterator::new(arg_tags_bytes);
+    let return_it = TagIterator::new(return_tag_bytes);
+    trace!("send<{}>({})->{}", service, args_it, return_it);
+
+    writer.write_u32(service)?;
+    for index in 0.. {
+        if let Some(arg_tag) = args_it.next() {
+            let mut data = unsafe { *data.offset(index) };
+            unsafe { send_value(writer, arg_tag, &mut data, write_tags)? };
+        } else {
+            break;
+        }
+    }
+    writer.write_u8(0)?;
+    writer.write_bytes(return_tag_bytes)?;
+
+    Ok(())
+}
+
+pub mod tag {
+    use core::fmt;
+
+    pub fn split_tag(tag_bytes: &[u8]) -> (&[u8], &[u8]) {
+        let tag_separator = tag_bytes
+            .iter()
+            .position(|&b| b == b':')
+            .expect("tag without a return separator");
+        let (arg_tags_bytes, rest) = tag_bytes.split_at(tag_separator);
+        let return_tag_bytes = &rest[1..];
+        (arg_tags_bytes, return_tag_bytes)
+    }
+
+    #[derive(Debug, Clone, Copy)]
+    pub enum Tag<'a> {
+        None,
+        Bool,
+        Int32,
+        Int64,
+        Float64,
+        String,
+        Bytes,
+        ByteArray,
+        Tuple(TagIterator<'a>, u8),
+        List(TagIterator<'a>),
+        Array(TagIterator<'a>, u8),
+        Range(TagIterator<'a>),
+        Keyword(TagIterator<'a>),
+        Object,
+    }
+
+    impl<'a> Tag<'a> {
+        pub fn as_u8(self) -> u8 {
+            match self {
+                Tag::None => b'n',
+                Tag::Bool => b'b',
+                Tag::Int32 => b'i',
+                Tag::Int64 => b'I',
+                Tag::Float64 => b'f',
+                Tag::String => b's',
+                Tag::Bytes => b'B',
+                Tag::ByteArray => b'A',
+                Tag::Tuple(_, _) => b't',
+                Tag::List(_) => b'l',
+                Tag::Array(_, _) => b'a',
+                Tag::Range(_) => b'r',
+                Tag::Keyword(_) => b'k',
+                Tag::Object => b'O',
+            }
+        }
+
+        pub fn alignment(self) -> usize {
+            use cslice::CSlice;
+            match self {
+                Tag::None => 1,
+                Tag::Bool => core::mem::align_of::<u8>(),
+                Tag::Int32 => core::mem::align_of::<i32>(),
+                Tag::Int64 => core::mem::align_of::<i64>(),
+                Tag::Float64 => core::mem::align_of::<f64>(),
+                // struct type: align to largest element
+                Tag::Tuple(it, arity) => {
+                    let it = it.clone();
+                    it.take(arity.into()).map(|t| t.alignment()).max().unwrap()
+                }
+                Tag::Range(it) => {
+                    let it = it.clone();
+                    it.take(3).map(|t| t.alignment()).max().unwrap()
+                }
+                // the ptr/length(s) pair is basically CSlice
+                Tag::Bytes | Tag::String | Tag::ByteArray | Tag::List(_) | Tag::Array(_, _) => {
+                    core::mem::align_of::<CSlice<()>>()
+                }
+                Tag::Keyword(_) => unreachable!("Tag::Keyword should not appear in composite types"),
+                Tag::Object => core::mem::align_of::<u32>(),
+            }
+        }
+
+        pub fn size(self) -> usize {
+            match self {
+                Tag::None => 0,
+                Tag::Bool => 1,
+                Tag::Int32 => 4,
+                Tag::Int64 => 8,
+                Tag::Float64 => 8,
+                Tag::String => 8,
+                Tag::Bytes => 8,
+                Tag::ByteArray => 8,
+                Tag::Tuple(it, arity) => {
+                    let mut size = 0;
+                    let mut max_alignment = 0;
+                    let mut it = it.clone();
+                    for _ in 0..arity {
+                        let tag = it.next().expect("truncated tag");
+                        let alignment = tag.alignment();
+                        max_alignment = core::cmp::max(max_alignment, alignment);
+                        size = super::round_up(size, alignment);
+                        size += tag.size();
+                    }
+                    // Take into account any tail padding (if element(s) with largest
+                    // alignment are not at the end).
+                    size = super::round_up(size, max_alignment);
+                    size
+                }
+                Tag::List(_) => 4,
+                Tag::Array(_, num_dims) => 4 * (1 + num_dims as usize),
+                Tag::Range(it) => {
+                    let tag = it.clone().next().expect("truncated tag");
+                    tag.size() * 3
+                }
+                Tag::Keyword(_) => unreachable!(),
+                Tag::Object => unreachable!(),
+            }
+        }
+    }
+
+    #[derive(Debug, Clone, Copy)]
+    pub struct TagIterator<'a> {
+        pub data: &'a [u8],
+    }
+
+    impl<'a> TagIterator<'a> {
+        pub fn new(data: &'a [u8]) -> TagIterator<'a> {
+            TagIterator { data }
+        }
+
+        fn sub(&mut self, count: u8) -> TagIterator<'a> {
+            let data = self.data;
+            for _ in 0..count {
+                self.next().expect("truncated tag");
+            }
+            TagIterator {
+                data: &data[..(data.len() - self.data.len())],
+            }
+        }
+    }
+
+    impl<'a> core::iter::Iterator for TagIterator<'a> {
+        type Item = Tag<'a>;
+
+        fn next(&mut self) -> Option<Tag<'a>> {
+            if self.data.len() == 0 {
+                return None;
+            }
+
+            let tag_byte = self.data[0];
+            self.data = &self.data[1..];
+            Some(match tag_byte {
+                b'n' => Tag::None,
+                b'b' => Tag::Bool,
+                b'i' => Tag::Int32,
+                b'I' => Tag::Int64,
+                b'f' => Tag::Float64,
+                b's' => Tag::String,
+                b'B' => Tag::Bytes,
+                b'A' => Tag::ByteArray,
+                b't' => {
+                    let count = self.data[0];
+                    self.data = &self.data[1..];
+                    Tag::Tuple(self.sub(count), count)
+                }
+                b'l' => Tag::List(self.sub(1)),
+                b'a' => {
+                    let count = self.data[0];
+                    self.data = &self.data[1..];
+                    Tag::Array(self.sub(1), count)
+                }
+                b'r' => Tag::Range(self.sub(1)),
+                b'k' => Tag::Keyword(self.sub(1)),
+                b'O' => Tag::Object,
+                _ => unreachable!(),
+            })
+        }
+    }
+
+    impl<'a> fmt::Display for TagIterator<'a> {
+        fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+            let mut it = self.clone();
+            let mut first = true;
+            while let Some(tag) = it.next() {
+                if first {
+                    first = false
+                } else {
+                    write!(f, ", ")?
+                }
+
+                match tag {
+                    Tag::None => write!(f, "None")?,
+                    Tag::Bool => write!(f, "Bool")?,
+                    Tag::Int32 => write!(f, "Int32")?,
+                    Tag::Int64 => write!(f, "Int64")?,
+                    Tag::Float64 => write!(f, "Float64")?,
+                    Tag::String => write!(f, "String")?,
+                    Tag::Bytes => write!(f, "Bytes")?,
+                    Tag::ByteArray => write!(f, "ByteArray")?,
+                    Tag::Tuple(it, _) => {
+                        write!(f, "Tuple(")?;
+                        it.fmt(f)?;
+                        write!(f, ")")?;
+                    }
+                    Tag::List(it) => {
+                        write!(f, "List(")?;
+                        it.fmt(f)?;
+                        write!(f, ")")?;
+                    }
+                    Tag::Array(it, num_dims) => {
+                        write!(f, "Array(")?;
+                        it.fmt(f)?;
+                        write!(f, ", {})", num_dims)?;
+                    }
+                    Tag::Range(it) => {
+                        write!(f, "Range(")?;
+                        it.fmt(f)?;
+                        write!(f, ")")?;
+                    }
+                    Tag::Keyword(it) => {
+                        write!(f, "Keyword(")?;
+                        it.fmt(f)?;
+                        write!(f, ")")?;
+                    }
+                    Tag::Object => write!(f, "Object")?,
+                }
+            }
+
+            Ok(())
+        }
+    }
+}

src/libksupport/src/rtio_acp.rs

@@ -0,0 +1,347 @@
+use core::sync::atomic::{fence, Ordering};
+
+use cslice::CSlice;
+use libcortex_a9::asm;
+use vcell::VolatileCell;
+
+use crate::{artiq_raise, pl::csr, resolve_channel_name, rtio_core};
+
+pub const RTIO_O_STATUS_WAIT: i32 = 1;
+pub const RTIO_O_STATUS_UNDERFLOW: i32 = 2;
+pub const RTIO_O_STATUS_DESTINATION_UNREACHABLE: i32 = 4;
+pub const RTIO_I_STATUS_WAIT_EVENT: i32 = 1;
+pub const RTIO_I_STATUS_OVERFLOW: i32 = 2;
+#[allow(unused)]
+pub const RTIO_I_STATUS_WAIT_STATUS: i32 = 4; // TODO
+pub const RTIO_I_STATUS_DESTINATION_UNREACHABLE: i32 = 8;
+
+#[repr(C)]
+pub struct TimestampedData {
+    timestamp: i64,
+    data: i32,
+}
+
+#[repr(C, align(64))]
+struct Transaction {
+    request_cmd: i8,
+    data_width: i8,
+    padding0: [i8; 2],
+    request_target: i32,
+    request_timestamp: i64,
+    request_data: [i32; 16],
+    padding1: [i64; 2],
+    reply_status: VolatileCell<i32>,
+    reply_data: VolatileCell<i32>,
+    reply_timestamp: VolatileCell<i64>,
+    padding2: [i64; 2],
+}
+
+static mut TRANSACTION_BUFFER: Transaction = Transaction {
+    request_cmd: 0,
+    data_width: 0,
+    request_target: 0,
+    request_timestamp: 0,
+    request_data: [0; 16],
+    reply_status: VolatileCell::new(0),
+    reply_data: VolatileCell::new(0),
+    reply_timestamp: VolatileCell::new(0),
+    padding0: [0; 2],
+    padding1: [0; 2],
+    padding2: [0; 2],
+};
+
+pub extern "C" fn init() {
+    unsafe {
+        rtio_core::reset_write(1);
+        csr::rtio::engine_addr_base_write(&TRANSACTION_BUFFER as *const Transaction as u32);
+        csr::rtio::enable_write(1);
+    }
+}
+
+pub extern "C" fn get_counter() -> i64 {
+    unsafe {
+        csr::rtio::counter_update_write(1);
+        csr::rtio::counter_read() as i64
+    }
+}
+
+static mut NOW: i64 = 0;
+
+pub extern "C" fn now_mu() -> i64 {
+    unsafe { NOW }
+}
+
+pub extern "C" fn at_mu(t: i64) {
+    unsafe { NOW = t }
+}
+
+pub extern "C" fn delay_mu(dt: i64) {
+    unsafe { NOW += dt }
+}
+
+#[inline(never)]
+unsafe fn process_exceptional_status(channel: i32, status: i32) {
+    let timestamp = now_mu();
+    if status & RTIO_O_STATUS_WAIT != 0 {
+        // FIXME: this is a kludge and probably buggy (kernel interrupted?)
+        while csr::rtio::o_status_read() as i32 & RTIO_O_STATUS_WAIT != 0 {}
+    }
+    if status & RTIO_O_STATUS_UNDERFLOW != 0 {
+        artiq_raise!(
+            "RTIOUnderflow",
+            format!(
+                "RTIO underflow at {{1}} mu, channel 0x{:04x}:{}, slack {{2}} mu",
+                channel,
+                resolve_channel_name(channel as u32)
+            ),
+            channel as i64,
+            timestamp,
+            timestamp - get_counter()
+        );
+    }
+    if status & RTIO_O_STATUS_DESTINATION_UNREACHABLE != 0 {
+        artiq_raise!(
+            "RTIODestinationUnreachable",
+            format!(
+                "RTIO destination unreachable, output, at {{0}} mu, channel 0x{:04x}:{}",
+                channel,
+                resolve_channel_name(channel as u32)
+            ),
+            timestamp,
+            channel as i64,
+            0
+        );
+    }
+}
+
+pub extern "C" fn output(target: i32, data: i32) {
+    unsafe {
+        // Clear status so we can observe response
+        TRANSACTION_BUFFER.reply_status.set(0);
+
+        TRANSACTION_BUFFER.request_cmd = 0;
+        TRANSACTION_BUFFER.data_width = 1;
+        TRANSACTION_BUFFER.request_target = target;
+        TRANSACTION_BUFFER.request_timestamp = NOW;
+        TRANSACTION_BUFFER.request_data[0] = data;
+
+        fence(Ordering::SeqCst);
+        asm::sev();
+        let mut status;
+        loop {
+            status = TRANSACTION_BUFFER.reply_status.get();
+            if status != 0 {
+                break;
+            }
+        }
+
+        let status = status & !0x10000;
+        if status != 0 {
+            process_exceptional_status(target >> 8, status);
+        }
+    }
+}
+
+pub extern "C" fn output_wide(target: i32, data: CSlice<i32>) {
+    unsafe {
+        // Clear status so we can observe response
+        TRANSACTION_BUFFER.reply_status.set(0);
+
+        TRANSACTION_BUFFER.request_cmd = 0;
+        TRANSACTION_BUFFER.data_width = data.len() as i8;
+        TRANSACTION_BUFFER.request_target = target;
+        TRANSACTION_BUFFER.request_timestamp = NOW;
+        TRANSACTION_BUFFER.request_data[..data.len()].copy_from_slice(data.as_ref());
+
+        fence(Ordering::SeqCst);
+        asm::sev();
+        let mut status;
+        loop {
+            status = TRANSACTION_BUFFER.reply_status.get();
+            if status != 0 {
+                break;
+            }
+        }
+
+        let status = status & !0x10000;
+        if status != 0 {
+            process_exceptional_status(target >> 8, status);
+        }
+    }
+}
+
+pub extern "C" fn input_timestamp(timeout: i64, channel: i32) -> i64 {
+    unsafe {
+        // Clear status so we can observe response
+        TRANSACTION_BUFFER.reply_status.set(0);
+
+        TRANSACTION_BUFFER.request_cmd = 1;
+        TRANSACTION_BUFFER.request_timestamp = timeout;
+        TRANSACTION_BUFFER.request_target = channel << 8;
+        TRANSACTION_BUFFER.data_width = 0;
+
+        fence(Ordering::SeqCst);
+        asm::sev();
+
+        let mut status;
+        loop {
+            status = TRANSACTION_BUFFER.reply_status.get();
+            if status != 0 {
+                break;
+            }
+        }
+
+        if status & RTIO_I_STATUS_OVERFLOW != 0 {
+            artiq_raise!(
+                "RTIOOverflow",
+                format!(
+                    "RTIO input overflow on channel 0x{:04x}:{}",
+                    channel,
+                    resolve_channel_name(channel as u32)
+                ),
+                channel as i64,
+                0,
+                0
+            );
+        }
+        if status & RTIO_I_STATUS_WAIT_EVENT != 0 {
+            return -1;
+        }
+        if status & RTIO_I_STATUS_DESTINATION_UNREACHABLE != 0 {
+            artiq_raise!(
+                "RTIODestinationUnreachable",
+                format!(
+                    "RTIO destination unreachable, input, on channel 0x{:04x}:{}",
+                    channel,
+                    resolve_channel_name(channel as u32)
+                ),
+                channel as i64,
+                0,
+                0
+            );
+        }
+
+        TRANSACTION_BUFFER.reply_timestamp.get()
+    }
+}
+
+pub extern "C" fn input_data(channel: i32) -> i32 {
+    unsafe {
+        TRANSACTION_BUFFER.reply_status.set(0);
+
+        TRANSACTION_BUFFER.request_cmd = 1;
+        TRANSACTION_BUFFER.request_timestamp = -1;
+        TRANSACTION_BUFFER.request_target = channel << 8;
+        TRANSACTION_BUFFER.data_width = 0;
+
+        fence(Ordering::SeqCst);
+        asm::sev();
+
+        let mut status;
+        loop {
+            status = TRANSACTION_BUFFER.reply_status.get();
+            if status != 0 {
+                break;
+            }
+        }
+
+        if status & RTIO_I_STATUS_OVERFLOW != 0 {
+            artiq_raise!(
+                "RTIOOverflow",
+                format!(
+                    "RTIO input overflow on channel 0x{:04x}:{}",
+                    channel,
+                    resolve_channel_name(channel as u32)
+                ),
+                channel as i64,
+                0,
+                0
+            );
+        }
+        if status & RTIO_I_STATUS_DESTINATION_UNREACHABLE != 0 {
+            artiq_raise!(
+                "RTIODestinationUnreachable",
+                format!(
+                    "RTIO destination unreachable, input, on channel 0x{:04x}:{}",
+                    channel,
+                    resolve_channel_name(channel as u32)
+                ),
+                channel as i64,
+                0,
+                0
+            );
+        }
+
+        TRANSACTION_BUFFER.reply_data.get()
+    }
+}
+
+pub extern "C" fn input_timestamped_data(timeout: i64, channel: i32) -> TimestampedData {
+    unsafe {
+        TRANSACTION_BUFFER.reply_status.set(0);
+
+        TRANSACTION_BUFFER.request_cmd = 1;
+        TRANSACTION_BUFFER.request_timestamp = timeout;
+        TRANSACTION_BUFFER.request_target = channel << 8;
+        TRANSACTION_BUFFER.data_width = 0;
+
+        fence(Ordering::SeqCst);
+        asm::sev();
+
+        let mut status;
+        loop {
+            status = TRANSACTION_BUFFER.reply_status.get();
+            if status != 0 {
+                break;
+            }
+        }
+
+        if status & RTIO_I_STATUS_OVERFLOW != 0 {
+            artiq_raise!(
+                "RTIOOverflow",
+                format!(
+                    "RTIO input overflow on channel 0x{:04x}:{}",
+                    channel,
+                    resolve_channel_name(channel as u32)
+                ),
+                channel as i64,
+                0,
+                0
+            );
+        }
+        if status & RTIO_I_STATUS_DESTINATION_UNREACHABLE != 0 {
+            artiq_raise!(
+                "RTIODestinationUnreachable",
+                format!(
+                    "RTIO destination unreachable, input, on channel 0x{:04x}:{}",
+                    channel,
+                    resolve_channel_name(channel as u32)
+                ),
+                channel as i64,
+                0,
+                0
+            );
+        }
+
+        TimestampedData {
+            timestamp: TRANSACTION_BUFFER.reply_timestamp.get(),
+            data: TRANSACTION_BUFFER.reply_data.get(),
+        }
+    }
+}
+
+pub fn write_log(data: &[i8]) {
+    let mut word: u32 = 0;
+    for i in 0..data.len() {
+        word <<= 8;
+        word |= data[i] as u32;
+        if i % 4 == 3 {
+            output((csr::CONFIG_RTIO_LOG_CHANNEL << 8) as i32, word as i32);
+            word = 0;
+        }
+    }
+
+    if word != 0 {
+        output((csr::CONFIG_RTIO_LOG_CHANNEL << 8) as i32, word as i32);
+    }
+}

src/libksupport/src/rtio_csr.rs

@@ -0,0 +1,274 @@
+use core::ptr::{read_volatile, write_volatile};
+
+use cslice::CSlice;
+
+use crate::{artiq_raise, pl::csr, resolve_channel_name, rtio_core};
+
+pub const RTIO_O_STATUS_WAIT: u8 = 1;
+pub const RTIO_O_STATUS_UNDERFLOW: u8 = 2;
+pub const RTIO_O_STATUS_DESTINATION_UNREACHABLE: u8 = 4;
+pub const RTIO_I_STATUS_WAIT_EVENT: u8 = 1;
+pub const RTIO_I_STATUS_OVERFLOW: u8 = 2;
+pub const RTIO_I_STATUS_WAIT_STATUS: u8 = 4;
+pub const RTIO_I_STATUS_DESTINATION_UNREACHABLE: u8 = 8;
+
+#[repr(C)]
+pub struct TimestampedData {
+    timestamp: i64,
+    data: i32,
+}
+
+pub extern "C" fn init() {
+    unsafe {
+        rtio_core::reset_write(1);
+    }
+}
+
+pub extern "C" fn get_counter() -> i64 {
+    unsafe {
+        csr::rtio::counter_update_write(1);
+        csr::rtio::counter_read() as i64
+    }
+}
+
+pub extern "C" fn now_mu() -> i64 {
+    unsafe { csr::rtio::now_read() as i64 }
+}
+
+pub extern "C" fn at_mu(t: i64) {
+    unsafe {
+        csr::rtio::now_write(t as u64);
+    }
+}
+
+pub extern "C" fn delay_mu(dt: i64) {
+    unsafe {
+        csr::rtio::now_write(csr::rtio::now_read() + dt as u64);
+    }
+}
+
+// writing the LSB of o_data (offset=0) triggers the RTIO write
+#[inline(always)]
+pub unsafe fn rtio_o_data_write(offset: usize, data: u32) {
+    write_volatile(
+        csr::rtio::O_DATA_ADDR.offset((csr::rtio::O_DATA_SIZE - 1 - offset) as isize),
+        data,
+    );
+}
+
+#[inline(always)]
+pub unsafe fn rtio_i_data_read(offset: usize) -> u32 {
+    read_volatile(csr::rtio::I_DATA_ADDR.offset((csr::rtio::I_DATA_SIZE - 1 - offset) as isize))
+}
+
+#[inline(never)]
+unsafe fn process_exceptional_status(channel: i32, status: u8) {
+    let timestamp = csr::rtio::now_read() as i64;
+    if status & RTIO_O_STATUS_WAIT != 0 {
+        while csr::rtio::o_status_read() & RTIO_O_STATUS_WAIT != 0 {}
+    }
+    if status & RTIO_O_STATUS_UNDERFLOW != 0 {
+        artiq_raise!(
+            "RTIOUnderflow",
+            format!(
+                "RTIO underflow at {{1}} mu, channel 0x{:04x}:{}, slack {{2}} mu",
+                channel,
+                resolve_channel_name(channel as u32)
+            ),
+            channel as i64,
+            timestamp,
+            timestamp - get_counter()
+        );
+    }
+    if status & RTIO_O_STATUS_DESTINATION_UNREACHABLE != 0 {
+        artiq_raise!(
+            "RTIODestinationUnreachable",
+            format!(
+                "RTIO destination unreachable, output, at {{0}} mu, channel 0x{:04x}:{}",
+                channel,
+                resolve_channel_name(channel as u32)
+            ),
+            timestamp,
+            channel as i64,
+            0
+        );
+    }
+}
+
+pub extern "C" fn output(target: i32, data: i32) {
+    unsafe {
+        csr::rtio::target_write(target as u32);
+        // writing target clears o_data
+        rtio_o_data_write(0, data as _);
+        let status = csr::rtio::o_status_read();
+        if status != 0 {
+            process_exceptional_status(target >> 8, status);
+        }
+    }
+}
+
+pub extern "C" fn output_wide(target: i32, data: &CSlice<i32>) {
+    unsafe {
+        csr::rtio::target_write(target as u32);
+        // writing target clears o_data
+        for i in (0..data.len()).rev() {
+            rtio_o_data_write(i, data[i] as _)
+        }
+        let status = csr::rtio::o_status_read();
+        if status != 0 {
+            process_exceptional_status(target >> 8, status);
+        }
+    }
+}
+
+pub extern "C" fn input_timestamp(timeout: i64, channel: i32) -> i64 {
+    unsafe {
+        csr::rtio::target_write((channel as u32) << 8);
+        csr::rtio::i_timeout_write(timeout as u64);
+
+        let mut status = RTIO_I_STATUS_WAIT_STATUS;
+        while status & RTIO_I_STATUS_WAIT_STATUS != 0 {
+            status = csr::rtio::i_status_read();
+        }
+
+        if status & RTIO_I_STATUS_OVERFLOW != 0 {
+            artiq_raise!(
+                "RTIOOverflow",
+                format!(
+                    "RTIO input overflow on channel 0x{:04x}:{}",
+                    channel,
+                    resolve_channel_name(channel as u32)
+                ),
+                channel as i64,
+                0,
+                0
+            );
+        }
+        if status & RTIO_I_STATUS_WAIT_EVENT != 0 {
+            return -1;
+        }
+        if status & RTIO_I_STATUS_DESTINATION_UNREACHABLE != 0 {
+            artiq_raise!(
+                "RTIODestinationUnreachable",
+                format!(
+                    "RTIO destination unreachable, input, on channel 0x{:04x}:{}",
+                    channel,
+                    resolve_channel_name(channel as u32)
+                ),
+                channel as i64,
+                0,
+                0
+            );
+        }
+
+        csr::rtio::i_timestamp_read() as i64
+    }
+}
+
+pub extern "C" fn input_data(channel: i32) -> i32 {
+    unsafe {
+        csr::rtio::target_write((channel as u32) << 8);
+        csr::rtio::i_timeout_write(0xffffffff_ffffffff);
+
+        let mut status = RTIO_I_STATUS_WAIT_STATUS;
+        while status & RTIO_I_STATUS_WAIT_STATUS != 0 {
+            status = csr::rtio::i_status_read();
+        }
+
+        if status & RTIO_I_STATUS_OVERFLOW != 0 {
+            artiq_raise!(
+                "RTIOOverflow",
+                format!(
+                    "RTIO input overflow on channel 0x{:04x}:{}",
+                    channel,
+                    resolve_channel_name(channel as u32)
+                ),
+                channel as i64,
+                0,
+                0
+            );
+        }
+        if status & RTIO_I_STATUS_DESTINATION_UNREACHABLE != 0 {
+            artiq_raise!(
+                "RTIODestinationUnreachable",
+                format!(
+                    "RTIO destination unreachable, input, on channel 0x{:04x}:{}",
+                    channel,
+                    resolve_channel_name(channel as u32)
+                ),
+                channel as i64,
+                0,
+                0
+            );
+        }
+
+        rtio_i_data_read(0) as i32
+    }
+}
+
+pub extern "C" fn input_timestamped_data(timeout: i64, channel: i32) -> TimestampedData {
+    unsafe {
+        csr::rtio::target_write((channel as u32) << 8);
+        csr::rtio::i_timeout_write(timeout as u64);
+
+        let mut status = RTIO_I_STATUS_WAIT_STATUS;
+        while status & RTIO_I_STATUS_WAIT_STATUS != 0 {
+            status = csr::rtio::i_status_read();
+        }
+
+        if status & RTIO_I_STATUS_OVERFLOW != 0 {
+            artiq_raise!(
+                "RTIOOverflow",
+                format!(
+                    "RTIO input overflow on channel 0x{:04x}:{}",
+                    channel,
+                    resolve_channel_name(channel as u32)
+                ),
+                channel as i64,
+                0,
+                0
+            );
+        }
+        if status & RTIO_I_STATUS_WAIT_EVENT != 0 {
+            return TimestampedData { timestamp: -1, data: 0 };
+        }
+        if status & RTIO_I_STATUS_DESTINATION_UNREACHABLE != 0 {
+            artiq_raise!(
+                "RTIODestinationUnreachable",
+                format!(
+                    "RTIO destination unreachable, input, on channel 0x{:04x}:{}",
+                    channel,
+                    resolve_channel_name(channel as u32)
+                ),
+                channel as i64,
+                0,
+                0
+            );
+        }
+
+        TimestampedData {
+            timestamp: csr::rtio::i_timestamp_read() as i64,
+            data: rtio_i_data_read(0) as i32,
+        }
+    }
+}
+
+pub fn write_log(data: &[i8]) {
+    unsafe {
+        csr::rtio::target_write(csr::CONFIG_RTIO_LOG_CHANNEL << 8);
+
+        let mut word: u32 = 0;
+        for i in 0..data.len() {
+            word <<= 8;
+            word |= data[i] as u32;
+            if i % 4 == 3 {
+                rtio_o_data_write(0, word);
+                word = 0;
+            }
+        }
+
+        if word != 0 {
+            rtio_o_data_write(0, word);
+        }
+    }
+}