szl: reduced binary size.

* Compiled unlzma with -Oz and enable LTO.
* Changed one unwrap to manual handling to remove fmt code.
* Implemented custom panic handler with minimal code.

LICENSE

@@ -0,0 +1,165 @@
+                   GNU LESSER GENERAL PUBLIC LICENSE
+                       Version 3, 29 June 2007
+
+ Copyright (C) 2007 Free Software Foundation, Inc. <http://fsf.org/>
+ Everyone is permitted to copy and distribute verbatim copies
+ of this license document, but changing it is not allowed.
+
+
+  This version of the GNU Lesser General Public License incorporates
+the terms and conditions of version 3 of the GNU General Public
+License, supplemented by the additional permissions listed below.
+
+  0. Additional Definitions.
+
+  As used herein, "this License" refers to version 3 of the GNU Lesser
+General Public License, and the "GNU GPL" refers to version 3 of the GNU
+General Public License.
+
+  "The Library" refers to a covered work governed by this License,
+other than an Application or a Combined Work as defined below.
+
+  An "Application" is any work that makes use of an interface provided
+by the Library, but which is not otherwise based on the Library.
+Defining a subclass of a class defined by the Library is deemed a mode
+of using an interface provided by the Library.
+
+  A "Combined Work" is a work produced by combining or linking an
+Application with the Library.  The particular version of the Library
+with which the Combined Work was made is also called the "Linked
+Version".
+
+  The "Minimal Corresponding Source" for a Combined Work means the
+Corresponding Source for the Combined Work, excluding any source code
+for portions of the Combined Work that, considered in isolation, are
+based on the Application, and not on the Linked Version.
+
+  The "Corresponding Application Code" for a Combined Work means the
+object code and/or source code for the Application, including any data
+and utility programs needed for reproducing the Combined Work from the
+Application, but excluding the System Libraries of the Combined Work.
+
+  1. Exception to Section 3 of the GNU GPL.
+
+  You may convey a covered work under sections 3 and 4 of this License
+without being bound by section 3 of the GNU GPL.
+
+  2. Conveying Modified Versions.
+
+  If you modify a copy of the Library, and, in your modifications, a
+facility refers to a function or data to be supplied by an Application
+that uses the facility (other than as an argument passed when the
+facility is invoked), then you may convey a copy of the modified
+version:
+
+   a) under this License, provided that you make a good faith effort to
+   ensure that, in the event an Application does not supply the
+   function or data, the facility still operates, and performs
+   whatever part of its purpose remains meaningful, or
+
+   b) under the GNU GPL, with none of the additional permissions of
+   this License applicable to that copy.
+
+  3. Object Code Incorporating Material from Library Header Files.
+
+  The object code form of an Application may incorporate material from
+a header file that is part of the Library.  You may convey such object
+code under terms of your choice, provided that, if the incorporated
+material is not limited to numerical parameters, data structure
+layouts and accessors, or small macros, inline functions and templates
+(ten or fewer lines in length), you do both of the following:
+
+   a) Give prominent notice with each copy of the object code that the
+   Library is used in it and that the Library and its use are
+   covered by this License.
+
+   b) Accompany the object code with a copy of the GNU GPL and this license
+   document.
+
+  4. Combined Works.
+
+  You may convey a Combined Work under terms of your choice that,
+taken together, effectively do not restrict modification of the
+portions of the Library contained in the Combined Work and reverse
+engineering for debugging such modifications, if you also do each of
+the following:
+
+   a) Give prominent notice with each copy of the Combined Work that
+   the Library is used in it and that the Library and its use are
+   covered by this License.
+
+   b) Accompany the Combined Work with a copy of the GNU GPL and this license
+   document.
+
+   c) For a Combined Work that displays copyright notices during
+   execution, include the copyright notice for the Library among
+   these notices, as well as a reference directing the user to the
+   copies of the GNU GPL and this license document.
+
+   d) Do one of the following:
+
+       0) Convey the Minimal Corresponding Source under the terms of this
+       License, and the Corresponding Application Code in a form
+       suitable for, and under terms that permit, the user to
+       recombine or relink the Application with a modified version of
+       the Linked Version to produce a modified Combined Work, in the
+       manner specified by section 6 of the GNU GPL for conveying
+       Corresponding Source.
+
+       1) Use a suitable shared library mechanism for linking with the
+       Library.  A suitable mechanism is one that (a) uses at run time
+       a copy of the Library already present on the user's computer
+       system, and (b) will operate properly with a modified version
+       of the Library that is interface-compatible with the Linked
+       Version.
+
+   e) Provide Installation Information, but only if you would otherwise
+   be required to provide such information under section 6 of the
+   GNU GPL, and only to the extent that such information is
+   necessary to install and execute a modified version of the
+   Combined Work produced by recombining or relinking the
+   Application with a modified version of the Linked Version. (If
+   you use option 4d0, the Installation Information must accompany
+   the Minimal Corresponding Source and Corresponding Application
+   Code. If you use option 4d1, you must provide the Installation
+   Information in the manner specified by section 6 of the GNU GPL
+   for conveying Corresponding Source.)
+
+  5. Combined Libraries.
+
+  You may place library facilities that are a work based on the
+Library side by side in a single library together with other library
+facilities that are not Applications and are not covered by this
+License, and convey such a combined library under terms of your
+choice, if you do both of the following:
+
+   a) Accompany the combined library with a copy of the same work based
+   on the Library, uncombined with any other library facilities,
+   conveyed under the terms of this License.
+
+   b) Give prominent notice with the combined library that part of it
+   is a work based on the Library, and explaining where to find the
+   accompanying uncombined form of the same work.
+
+  6. Revised Versions of the GNU Lesser General Public License.
+
+  The Free Software Foundation may publish revised and/or new versions
+of the GNU Lesser General Public License from time to time. Such new
+versions will be similar in spirit to the present version, but may
+differ in detail to address new problems or concerns.
+
+  Each version is given a distinguishing version number. If the
+Library as you received it specifies that a certain numbered version
+of the GNU Lesser General Public License "or any later version"
+applies to it, you have the option of following the terms and
+conditions either of that published version or of any later version
+published by the Free Software Foundation. If the Library as you
+received it does not specify a version number of the GNU Lesser
+General Public License, you may choose any version of the GNU Lesser
+General Public License ever published by the Free Software Foundation.
+
+  If the Library as you received it specifies that a proxy can decide
+whether future versions of the GNU Lesser General Public License shall
+apply, that proxy's public statement of acceptance of any version is
+permanent authorization for you to choose that version for the
+Library.

README.md

@@ -1,3 +1,34 @@
+ARTIQ on Zynq
+=============
+
+How to use
+----------
+
+1. Install ARTIQ-6 or newer.
+2. Select the latest successful build on Hydra: https://nixbld.m-labs.hk/jobset/artiq/zynq
+3. Search for the job named ``<board>-<variant>-sd`` (for example: ``zc706-nist_clock-sd`` or ``zc706-nist_qc2-sd``).
+4. Download the ``boot.bin`` "binary distribution" and place it at the root of a FAT-formatted SD card.
+5. Optionally, create a ``config.txt`` configuration file at the root of the SD card containing ``key=value`` pairs on each line. Use the ``ip``, ``ip6`` and ``mac`` keys to respectively set the IPv4, IPv6 and MAC address of the board. Configuring an IPv6 address is entirely optional. If these keys are not found, the firmware will use default values that may or may not be compatible with your network.
+6. Insert the SD card into the board and set up the board to boot from the SD card. For the ZC706, this is achieved by placing the large DIP switch SW11 in the 00110 position.
+7. Power up the board. After the firmware starts successfully, it should respond to ping at its IP addresses, and boot messages can be observed from its UART at 115200bps.
+8. Create and use an ARTIQ device database as usual, but set ``"target": "cortexa9"`` in the arguments of the core device.
+
+Configuration
+-------------
+
+Configuring the device is done using the ``config.txt`` text file at the root of the SD card, plus the contents of the ``config`` folder. When searching for a configuration key, the firmware first looks for a file named ``/config/[key].bin`` and, if it exists, returns the contents of that file. If not, it looks into ``/config.txt``, which contains a list of ``key=value`` pairs, one per line. The ``config`` folder allows configuration values that consist in binary data, such as the startup kernel.
+
+The following configuration keys are available:
+
+- ``mac``: Ethernet MAC address.
+- ``ip``: IPv4 address.
+- ``ip6``: IPv6 address.
+- ``startup``: startup kernel in ELF format (as produced by ``artiq_compile``).
+- ``rtioclk``: source of RTIO clock; valid values are ``external`` and ``internal``.
+
+Development instructions
+------------------------
+
 Configure Nix channels:
 
 ```shell
@@ -5,22 +36,46 @@ nix-channel --add https://nixbld.m-labs.hk/channel/custom/artiq/fast-beta/artiq-
 nix-channel --update
 ```
 
-Pure build with Nix:
+Note: if you are using Nix channels the first time, you need to be aware of this bug: https://github.com/NixOS/nix/issues/3831
+
+Pure build with Nix and execution on a remote JTAG server:
 
 ```shell
 nix-build -A zc706-simple-jtag  # or zc706-nist_qc2-jtag or zc706-nist_clock-jtag
 ./remote_run.sh
 ```
 
-Impure incremental build:
+Impure incremental build and execution on a remote JTAG server:
 
 ```shell
 nix-shell
 cd src
-./zc706.py -g ../build/gateware  # build gateware
+gateware/zc706.py -g ../build/gateware  # build gateware
 make                                    # build firmware
 cd ..
 ./remote_run.sh -i
 ```
 
-The impure build process can also be used on non-Nix systems.
+Notes:
+
+- This is known to work with Nixpkgs 20.03 and the ``nixbld.m-labs.hk`` binary substituter can also be used here (see the ARTIQ manual for the public key and instructions).
+- The impure build process is also compatible with non-Nix systems.
+- If the board is connected to the local machine, use the ``local_run.sh`` script.
+
+License
+-------
+
+Copyright (C) 2019-2020 M-Labs Limited.
+
+ARTIQ is free software: you can redistribute it and/or modify
+it under the terms of the GNU Lesser General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+ARTIQ is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU Lesser General Public License for more details.
+
+You should have received a copy of the GNU Lesser General Public License
+along with ARTIQ.  If not, see <http://www.gnu.org/licenses/>.

default.nix

@@ -14,7 +14,7 @@ let
       version = "0.1.0";
 
       src = ./src;
-      cargoSha256 = "0xminds5fyp7c9vsx651zv3yzyhxnl9a02rhjl2wfxf8m679r45l";
+      cargoSha256 = "0w1d9z6k1ag5ylaz961xr3q957nj1ask07rmkmarb1kw933hr3lp";
 
       nativeBuildInputs = [
         pkgs.gnumake
@@ -48,7 +48,7 @@ let
         ];
       }
       ''
-        python ${./src/zc706.py} -g build -V ${variant}
+        python ${./src/gateware}/zc706.py -g build -V ${variant}
         mkdir -p $out $out/nix-support
         cp build/top.bit $out
         echo file binary-dist $out/top.bit >> $out/nix-support/hydra-build-products
@@ -92,5 +92,8 @@ in
   (
     (build-zc706 { variant = "simple"; }) //
     (build-zc706 { variant = "nist_clock"; }) //
-    (build-zc706 { variant = "nist_qc2"; })
+    (build-zc706 { variant = "nist_qc2"; }) //
+    (build-zc706 { variant = "acpki_simple"; }) //
+    (build-zc706 { variant = "acpki_nist_clock"; }) //
+    (build-zc706 { variant = "acpki_nist_qc2"; })
   )

examples/device_db.py

@@ -1,3 +1,5 @@
+# For NIST_QC2
+
 device_db = {
     "core": {
         "type": "local",
@@ -10,7 +12,17 @@ device_db = {
             "target": "cortexa9"
         }
     },
-
+    "core_cache": {
+        "type": "local",
+        "module": "artiq.coredevice.cache",
+        "class": "CoreCache"
+    },
+    "core_dma": {
+        "type": "local",
+        "module": "artiq.coredevice.dma",
+        "class": "CoreDMA"
+    },
+    # led? are common to all variants
     "led0": {
         "type": "local",
         "module": "artiq.coredevice.ttl",
@@ -36,3 +48,20 @@ device_db = {
         "arguments": {"channel": 3}
     },
 }
+
+# TTLs on QC2 backplane
+for i in range(40):
+    device_db["ttl" + str(i)] = {
+        "type": "local",
+        "module": "artiq.coredevice.ttl",
+        "class": "TTLInOut",
+        "arguments": {"channel": 4+i}
+    }
+
+# for ARTIQ test suite
+device_db.update(
+    loop_out="ttl0",
+    loop_in="ttl1",
+    ttl_out="ttl2",
+    ttl_out_serdes="ttl2",
+)

examples/dma.py

@@ -0,0 +1,26 @@
+from artiq.experiment import *
+
+class DMAPulses(EnvExperiment):
+    def build(self):
+        self.setattr_device("core")
+        self.setattr_device("core_dma")
+        self.setattr_device("led0")
+
+    @kernel
+    def record(self):
+        with self.core_dma.record("pulse"):
+            delay(200*ms)
+            # all RTIO operations now go to the "pulse"
+            # DMA buffer, instead of being executed immediately.
+            self.led0.pulse(500*ms)
+
+
+    @kernel
+    def run(self):
+        self.core.reset()
+        self.record()
+        # prefetch the address of the DMA buffer
+        # for faster playback trigger
+        pulse_handle = self.core_dma.get_handle("pulse")
+        self.core.break_realtime()
+        self.core_dma.playback_handle(pulse_handle)

rustPlatform.nix

@@ -3,9 +3,9 @@
 let
   rustcSrc = pkgs.fetchgit {
     url = "https://github.com/rust-lang/rust.git";
-    # master of 2020-04-10
+    # sync with git_commit_hash from pkg.rust in channel-rust-nightly.toml
-    rev = "94d346360da50f159e0dc777dc9bc3c5b6b51a00";
+    rev = "5ef299eb9805b4c86b227b718b39084e8bf24454";
-    sha256 = "1hcqdz4w2vqb12rrqqcjbfs5s0w4qwjn7z45d1zh0fzncdcf6f7d";
+    sha256 = "0gc9hmb1sfkaf3ba8fsynl1n6bs8nk65hbhhx7ss89dfkrsxrn0x";
     fetchSubmodules = true;
   };
   rustManifest = ./channel-rust-nightly.toml;

src/Cargo.lock

@@ -37,9 +37,9 @@ checksum = "08c48aae112d48ed9f069b33538ea9e3e90aa263cfa3d1c24309612b1f7472de"
 
 [[package]]
 name = "cc"
-version = "1.0.57"
+version = "1.0.58"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0fde55d2a2bfaa4c9668bbc63f531fbdeee3ffe188f4662511ce2c22b3eedebe"
+checksum = "f9a06fb2e53271d7c279ec1efea6ab691c35a2ae67ec0d91d7acec0caf13b518"
 
 [[package]]
 name = "cfg-if"
@@ -96,6 +96,7 @@ dependencies = [
 name = "dyld"
 version = "0.1.0"
 dependencies = [
+ "libcortex_a9",
  "log",
 ]
 
@@ -105,15 +106,15 @@ version = "0.2.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "fa998ce59ec9765d15216393af37a58961ddcefb14c753b4816ba2191d865fcb"
 dependencies = [
- "nb",
+ "nb 0.1.3",
  "void",
 ]
 
 [[package]]
 name = "fatfs"
-version = "0.3.3"
+version = "0.3.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f6d1df9e4503954f60504a5ee4fc435cd65cc42e98b2081f7f421be5f2e68e7d"
+checksum = "93079df23039e52059e1f03b4c29fb0c72da2c792aad91bb2236c9fb81d3592e"
 dependencies = [
  "bitflags",
  "byteorder",
@@ -200,11 +201,11 @@ dependencies = [
 [[package]]
 name = "libasync"
 version = "0.0.0"
-source = "git+https://git.m-labs.hk/M-Labs/zc706.git#371e59cef57746e3dd4cae915be7fd3286972822"
+source = "git+https://git.m-labs.hk/M-Labs/zynq-rs.git#e8ba73a8c74f79f543db6a495ac65aae68e18bab"
 dependencies = [
  "embedded-hal",
  "libcortex_a9",
- "nb",
+ "nb 0.1.3",
  "pin-utils",
  "smoltcp",
 ]
@@ -212,14 +213,14 @@ dependencies = [
 [[package]]
 name = "libboard_zynq"
 version = "0.0.0"
-source = "git+https://git.m-labs.hk/M-Labs/zc706.git#371e59cef57746e3dd4cae915be7fd3286972822"
+source = "git+https://git.m-labs.hk/M-Labs/zynq-rs.git#e8ba73a8c74f79f543db6a495ac65aae68e18bab"
 dependencies = [
  "bit_field",
  "embedded-hal",
  "libcortex_a9",
  "libregister",
  "log",
- "nb",
+ "nb 0.1.3",
  "smoltcp",
  "void",
  "volatile-register",
@@ -236,16 +237,22 @@ dependencies = [
 [[package]]
 name = "libcortex_a9"
 version = "0.0.0"
-source = "git+https://git.m-labs.hk/M-Labs/zc706.git#371e59cef57746e3dd4cae915be7fd3286972822"
+source = "git+https://git.m-labs.hk/M-Labs/zynq-rs.git#e8ba73a8c74f79f543db6a495ac65aae68e18bab"
 dependencies = [
  "bit_field",
  "libregister",
 ]
 
+[[package]]
+name = "libm"
+version = "0.2.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c7d73b3f436185384286bd8098d17ec07c9a7d2388a6599f824d8502b529702a"
+
 [[package]]
 name = "libregister"
 version = "0.0.0"
-source = "git+https://git.m-labs.hk/M-Labs/zc706.git#371e59cef57746e3dd4cae915be7fd3286972822"
+source = "git+https://git.m-labs.hk/M-Labs/zynq-rs.git#e8ba73a8c74f79f543db6a495ac65aae68e18bab"
 dependencies = [
  "bit_field",
  "vcell",
@@ -255,7 +262,7 @@ dependencies = [
 [[package]]
 name = "libsupport_zynq"
 version = "0.0.0"
-source = "git+https://git.m-labs.hk/M-Labs/zc706.git#371e59cef57746e3dd4cae915be7fd3286972822"
+source = "git+https://git.m-labs.hk/M-Labs/zynq-rs.git#e8ba73a8c74f79f543db6a495ac65aae68e18bab"
 dependencies = [
  "compiler_builtins",
  "libboard_zynq",
@@ -273,13 +280,19 @@ checksum = "e70e46c13c0e8374c26cec5752e3347ca1087d9711de8f45aa513a7700efd73d"
 
 [[package]]
 name = "log"
-version = "0.4.8"
+version = "0.4.11"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "14b6052be84e6b71ab17edffc2eeabf5c2c3ae1fdb464aae35ac50c67a44e1f7"
+checksum = "4fabed175da42fed1fa0746b0ea71f412aa9d35e76e95e59b192c64b9dc2bf8b"
 dependencies = [
  "cfg-if",
 ]
 
+[[package]]
+name = "log_buffer"
+version = "1.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f033173c9486b7fe97a79c895c0a3483ae395ab6744c985d10078950e2492419"
+
 [[package]]
 name = "managed"
 version = "0.7.2"
@@ -294,15 +307,24 @@ checksum = "3728d817d99e5ac407411fa471ff9800a778d88a24685968b36824eaf4bee400"
 
 [[package]]
 name = "nb"
-version = "0.1.2"
+version = "0.1.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b1411551beb3c11dedfb0a90a0fa256b47d28b9ec2cdff34c25a2fa59e45dbdc"
+checksum = "801d31da0513b6ec5214e9bf433a77966320625a37860f910be265be6e18d06f"
+dependencies = [
+ "nb 1.0.0",
+]
+
+[[package]]
+name = "nb"
+version = "1.0.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "546c37ac5d9e56f55e73b677106873d9d9f5190605e41a856503623648488cae"
 
 [[package]]
 name = "num-derive"
-version = "0.3.0"
+version = "0.3.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0c8b15b261814f992e33760b1fca9fe8b693d8a65299f20c9901688636cfb746"
+checksum = "e0396233fb2d5b0ae3f05ff6aba9a09185f7f6e70f87fb01147d545f85364665"
 dependencies = [
  "proc-macro2",
  "quote",
@@ -320,18 +342,18 @@ dependencies = [
 
 [[package]]
 name = "pin-project"
-version = "0.4.22"
+version = "0.4.23"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "12e3a6cdbfe94a5e4572812a0201f8c0ed98c1c452c7b8563ce2276988ef9c17"
+checksum = "ca4433fff2ae79342e497d9f8ee990d174071408f28f726d6d83af93e58e48aa"
 dependencies = [
  "pin-project-internal",
 ]
 
 [[package]]
 name = "pin-project-internal"
-version = "0.4.22"
+version = "0.4.23"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6a0ffd45cf79d88737d7cc85bfd5d2894bee1139b356e616fe85dc389c61aaf7"
+checksum = "2c0e815c3ee9a031fdf5af21c10aa17c573c9c6a566328d99e3936c34e36461f"
 dependencies = [
  "proc-macro2",
  "quote",
@@ -346,9 +368,9 @@ checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184"
 
 [[package]]
 name = "proc-macro-hack"
-version = "0.5.16"
+version = "0.5.18"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7e0456befd48169b9f13ef0f0ad46d492cf9d2dbb918bcf38e01eed4ce3ec5e4"
+checksum = "99c605b9a0adc77b7211c6b1f722dcb613d68d66859a44f3d485a6da332b0598"
 
 [[package]]
 name = "proc-macro-nested"
@@ -358,9 +380,9 @@ checksum = "eba180dafb9038b050a4c280019bbedf9f2467b61e5d892dcad585bb57aadc5a"
 
 [[package]]
 name = "proc-macro2"
-version = "1.0.18"
+version = "1.0.19"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "beae6331a816b1f65d04c45b078fd8e6c93e8071771f41b8163255bbd8d7c8fa"
+checksum = "04f5f085b5d71e2188cb8271e5da0161ad52c3f227a661a3c135fdf28e258b12"
 dependencies = [
  "unicode-xid",
 ]
@@ -397,13 +419,16 @@ dependencies = [
  "libboard_zynq",
  "libc",
  "libcortex_a9",
+ "libm",
  "libregister",
  "libsupport_zynq",
  "log",
- "nb",
+ "log_buffer",
+ "nb 0.1.3",
  "num-derive",
  "num-traits",
  "unwind",
+ "vcell",
  "void",
 ]
 
@@ -420,9 +445,9 @@ dependencies = [
 
 [[package]]
 name = "syn"
-version = "1.0.33"
+version = "1.0.38"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e8d5d96e8cbb005d6959f119f773bfaebb5684296108fb32600c00cde305b2cd"
+checksum = "e69abc24912995b3038597a7a593be5053eb0fb44f3cc5beec0deb421790c1f4"
 dependencies = [
  "proc-macro2",
  "quote",

src/Cargo.toml

@@ -9,17 +9,12 @@ members = [
   "szl"
 ]
 
-[profile.dev]
-panic = "abort"
-lto = false
-
 [profile.release]
 panic = "abort"
 debug = true
-# Link-Time Optimization:
+codegen-units = 1
-# turn off if you get unusable debug symbols.
+opt-level = 'z'
 lto = true
-opt-level = 'z'  # Optimize for size.
 
 [patch.crates-io]
 core_io = { path = "./libcoreio" }

src/Makefile

@@ -5,11 +5,11 @@ all: ../build/firmware/armv7-none-eabihf/release/szl
 .PHONY: all
 
 
-../build/pl.rs: zc706.py
+../build/pl.rs ../build/rustc-cfg: gateware/*
 	mkdir -p ../build
-	python zc706.py -r ../build/pl.rs -V $(VARIANT)
+	python gateware/zc706.py -r ../build/pl.rs -c ../build/rustc-cfg -V $(VARIANT)
 
-../build/firmware/armv7-none-eabihf/release/runtime: ../build/pl.rs $(shell find . -path ./szl -prune -o -print)
+../build/firmware/armv7-none-eabihf/release/runtime: ../build/pl.rs ../build/rustc-cfg $(shell find . -path ./szl -prune -o -print)
 	XBUILD_SYSROOT_PATH=`pwd`/../build/sysroot cargo xbuild --release -p runtime --target-dir ../build/firmware
 
 ../build/szl-payload.bin.lzma: ../build/firmware/armv7-none-eabihf/release/runtime

src/gateware/acpki.py

@@ -0,0 +1,260 @@
+from operator import attrgetter
+
+from migen import *
+from migen.genlib.cdc import MultiReg
+from migen_axi.interconnect import axi
+from misoc.interconnect.csr import *
+
+from artiq.gateware import rtio
+
+
+OUT_BURST_LEN = 4
+IN_BURST_LEN = 4
+
+
+class Engine(Module, AutoCSR):
+    def __init__(self, bus, user):
+        self.addr_base = CSRStorage(32)
+        self.trig_count = CSRStatus(32)
+        self.write_count = CSRStatus(32)
+
+        self.trigger_stb = Signal()
+
+        # Dout : Data received from CPU, output by DMA module
+        # Din : Data driven into DMA module, written into CPU
+        # When stb assert, index shows word being read/written, dout/din holds
+        # data
+        #
+        # Cycle:
+        # trigger_stb pulsed at start
+        # Then out_burst_len words are strobed out of dout
+        # Then, when din_ready is high, in_burst_len words are strobed in to din
+        self.dout_stb = Signal()
+        self.din_stb = Signal()
+        self.dout_index = Signal(max=16)
+        self.din_index = Signal(max=16)
+        self.din_ready = Signal()
+        self.dout = Signal(64)
+        self.din = Signal(64)
+
+        ###
+
+        self.sync += If(self.trigger_stb, self.trig_count.status.eq(self.trig_count.status+1))
+
+        self.comb += [
+            user.aruser.eq(0x1f),
+            user.awuser.eq(0x1f)
+        ]
+
+        ar, aw, w, r, b = attrgetter("ar", "aw", "w", "r", "b")(bus)
+
+        ### Read
+        self.comb += [
+            ar.addr.eq(self.addr_base.storage),
+            self.dout.eq(r.data),
+            r.ready.eq(1),
+            ar.burst.eq(axi.Burst.incr.value),
+            ar.len.eq(OUT_BURST_LEN-1), # Number of transfers in burst (0->1 transfer, 1->2 transfers...)
+            ar.size.eq(3), # Width of burst: 3 = 8 bytes = 64 bits
+            ar.cache.eq(0xf),
+        ]
+
+        # read control
+        self.submodules.read_fsm = read_fsm = FSM(reset_state="IDLE")
+        read_fsm.act("IDLE",
+            If(self.trigger_stb,
+                ar.valid.eq(1),
+                If(ar.ready,
+                    NextState("READ")
+                ).Else(
+                    NextState("READ_START")
+                )
+            )
+        )
+        read_fsm.act("READ_START",
+            ar.valid.eq(1),
+            If(ar.ready,
+                NextState("READ"),
+            )
+        )
+        read_fsm.act("READ",
+            ar.valid.eq(0),
+            If(r.last & r.valid,
+                NextState("IDLE")
+            )
+        )
+
+        self.sync += [
+            If(read_fsm.ongoing("IDLE"),
+                self.dout_index.eq(0)
+            ).Else(If(r.valid & read_fsm.ongoing("READ"),
+                    self.dout_index.eq(self.dout_index+1)
+                )
+            )
+        ]
+
+        self.comb += self.dout_stb.eq(r.valid & r.ready)
+
+        ### Write
+        self.comb += [
+            w.data.eq(self.din),
+            aw.addr.eq(self.addr_base.storage+32), # Write to next cache line
+            w.strb.eq(0xff),
+            aw.burst.eq(axi.Burst.incr.value),
+            aw.len.eq(IN_BURST_LEN-1), # Number of transfers in burst minus 1
+            aw.size.eq(3), # Width of burst: 3 = 8 bytes = 64 bits
+            aw.cache.eq(0xf),
+            b.ready.eq(1),
+        ]
+
+        # write control
+        self.submodules.write_fsm = write_fsm = FSM(reset_state="IDLE")
+        write_fsm.act("IDLE",
+            w.valid.eq(0),
+            aw.valid.eq(0),
+            If(self.trigger_stb,
+                aw.valid.eq(1),
+                If(aw.ready, # assumes aw.ready is not randomly deasserted
+                    NextState("DATA_WAIT")
+                ).Else(
+                    NextState("AW_READY_WAIT")
+                )
+            )
+        )
+        write_fsm.act("AW_READY_WAIT",
+            aw.valid.eq(1),
+            If(aw.ready,
+                NextState("DATA_WAIT"),
+            )
+        )
+        write_fsm.act("DATA_WAIT",
+            aw.valid.eq(0),
+            If(self.din_ready,
+                w.valid.eq(1),
+                NextState("WRITE")
+            )
+        )
+        write_fsm.act("WRITE",
+            w.valid.eq(1),
+            If(w.ready & w.last,
+                NextState("IDLE")
+            )
+        )
+
+        self.sync += If(w.ready & w.valid, self.write_count.status.eq(self.write_count.status+1))
+
+        self.sync += [
+            If(write_fsm.ongoing("IDLE"),
+                self.din_index.eq(0)
+            ),
+            If(w.ready & w.valid, self.din_index.eq(self.din_index+1))
+        ]
+
+        self.comb += [
+            w.last.eq(0),
+            If(self.din_index==aw.len, w.last.eq(1))
+        ]
+
+        self.comb += self.din_stb.eq(w.valid & w.ready)
+
+
+
+class KernelInitiator(Module, AutoCSR):
+    def __init__(self, tsc, bus, user, evento):
+        # Core is disabled upon reset to avoid spurious triggering if evento toggles from e.g. boot code.
+        self.enable = CSRStorage()
+
+        self.counter = CSRStatus(64)
+        self.counter_update = CSR()
+        self.o_status = CSRStatus(3)
+        self.i_status = CSRStatus(4)
+
+        self.submodules.engine = Engine(bus, user)
+        self.cri = rtio.cri.Interface()
+
+        ###
+
+        evento_stb = Signal()
+        evento_latched = Signal()
+        evento_latched_d = Signal()
+        self.specials += MultiReg(evento, evento_latched)
+        self.sync += evento_latched_d.eq(evento_latched)
+        self.comb += self.engine.trigger_stb.eq(self.enable.storage & (evento_latched != evento_latched_d))
+
+        cri = self.cri
+
+        cmd = Signal(8)
+        cmd_write = Signal()
+        cmd_read = Signal()
+        self.comb += [
+            cmd_write.eq(cmd == 0),
+            cmd_read.eq(cmd == 1)
+        ]
+
+        dout_cases = {}
+        dout_cases[0] = [
+            cmd.eq(self.engine.dout[:8]),
+            cri.chan_sel.eq(self.engine.dout[40:]),
+            cri.o_address.eq(self.engine.dout[32:40])
+        ]
+        dout_cases[1] = [
+            cri.o_timestamp.eq(self.engine.dout)
+        ]
+        dout_cases[2] = [cri.o_data.eq(self.engine.dout)] # only lowest 64 bits
+
+        self.sync += [
+            cri.cmd.eq(rtio.cri.commands["nop"]),
+            If(self.engine.dout_stb,
+                Case(self.engine.dout_index, dout_cases),
+                If(self.engine.dout_index == 2,
+                    If(cmd_write, cri.cmd.eq(rtio.cri.commands["write"])),
+                    If(cmd_read, cri.cmd.eq(rtio.cri.commands["read"]))
+                )
+            )
+        ]
+
+        # If input event, wait for response before allow input data to be
+        # sampled
+        # TODO: If output, wait for wait flag clear
+        RTIO_I_STATUS_WAIT_STATUS = 4
+        RTIO_O_STATUS_WAIT = 1
+
+        self.submodules.fsm = fsm = FSM(reset_state="IDLE")
+
+        fsm.act("IDLE",
+            If(self.engine.trigger_stb, NextState("WAIT_OUT_CYCLE"))
+        )
+        fsm.act("WAIT_OUT_CYCLE",
+            self.engine.din_ready.eq(0),
+            If(self.engine.dout_stb & (self.engine.dout_index == 3),
+                NextState("WAIT_READY")
+            )
+        )
+        fsm.act("WAIT_READY",
+            If(cmd_read & (cri.i_status & RTIO_I_STATUS_WAIT_STATUS == 0) \
+                | cmd_write & ~(cri.o_status & RTIO_O_STATUS_WAIT),
+                self.engine.din_ready.eq(1),
+                NextState("IDLE")
+            )
+        )
+
+        din_cases_cmdwrite = {
+            0: [self.engine.din.eq((1<<16) | cri.o_status)],
+            1: [self.engine.din.eq(0)],
+        }
+        din_cases_cmdread = {
+            0: [self.engine.din[:32].eq((1<<16) | cri.i_status), self.engine.din[32:].eq(cri.i_data)],
+            1: [self.engine.din.eq(cri.i_timestamp)]
+        }
+
+        self.comb += [
+            If(cmd_read, Case(self.engine.din_index, din_cases_cmdread)),
+            If(cmd_write, Case(self.engine.din_index, din_cases_cmdwrite)),
+        ]
+
+        # CRI CSRs
+        self.sync += If(self.counter_update.re, self.counter.status.eq(tsc.full_ts_cri))
+        self.comb += [
+            self.o_status.status.eq(self.cri.o_status),
+            self.i_status.status.eq(self.cri.i_status),
+        ]

src/gateware/analyzer.py

@@ -0,0 +1,121 @@
+from migen import *
+
+from misoc.interconnect.csr import *
+from misoc.interconnect import stream
+from migen_axi.interconnect import axi
+
+from artiq.gateware.rtio.analyzer import message_len, MessageEncoder
+
+import endianness
+
+
+class AXIDMAWriter(Module, AutoCSR):
+    def __init__(self, membus, max_outstanding_requests):
+        aw = len(membus.aw.addr)
+        dw = len(membus.w.data)
+        assert message_len % dw == 0
+        burst_length = message_len//dw
+        alignment_bits = log2_int(message_len//8)
+
+        self.reset = CSR()  # only apply when shut down
+        # All numbers in bytes
+        self.base_address = CSRStorage(aw, alignment_bits=alignment_bits)
+        self.last_address = CSRStorage(aw, alignment_bits=alignment_bits)
+        self.byte_count = CSRStatus(32)  # only read when shut down
+        self.bus_error = CSRStatus()
+
+        self.make_request = Signal()
+        self.sink = stream.Endpoint([("data", dw)])
+
+        # # #
+
+        outstanding_requests = Signal(max=max_outstanding_requests+1)
+        current_address = Signal(aw - alignment_bits)
+        self.comb += [
+            membus.aw.addr.eq(Cat(C(0, alignment_bits), current_address)),
+            membus.aw.id.eq(0),                   # Same ID for all transactions to forbid reordering.
+            membus.aw.burst.eq(axi.Burst.incr.value),
+            membus.aw.len.eq(burst_length-1),     # Number of transfers in burst (0->1 transfer, 1->2 transfers...).
+            membus.aw.size.eq(log2_int(dw//8)),   # Width of burst: 3 = 8 bytes = 64 bits.
+            membus.aw.cache.eq(0xf),
+            membus.aw.valid.eq(outstanding_requests != 0),
+        ]
+        self.sync += [
+            outstanding_requests.eq(outstanding_requests + self.make_request - (membus.aw.valid & membus.aw.ready)),
+
+            If(self.reset.re,
+                current_address.eq(self.base_address.storage)),
+            If(membus.aw.valid & membus.aw.ready,
+                If(current_address == self.last_address.storage,
+                    current_address.eq(self.base_address.storage)
+                ).Else(
+                    current_address.eq(current_address + 1)
+                )
+            )
+        ]
+
+        self.comb += [
+            membus.w.id.eq(0),
+            membus.w.valid.eq(self.sink.stb),
+            self.sink.ack.eq(membus.w.ready),
+            membus.w.data.eq(endianness.convert_signal(self.sink.data)),
+            membus.w.strb.eq(2**(dw//8)-1),
+        ]
+        beat_count = Signal(max=burst_length)
+        self.sync += [
+            If(membus.w.valid & membus.w.ready,
+                membus.w.last.eq(0),
+                If(membus.w.last,
+                    beat_count.eq(0)
+                ).Else(
+                    If(beat_count == burst_length-2, membus.w.last.eq(1)),
+                    beat_count.eq(beat_count + 1)
+                )
+            )
+        ]
+
+        message_count = Signal(32 - log2_int(message_len//8))
+        self.comb += self.byte_count.status.eq(
+            message_count << log2_int(message_len//8))
+        self.sync += [
+            If(self.reset.re, message_count.eq(0)),
+            If(membus.w.valid & membus.w.ready & membus.w.last, message_count.eq(message_count + 1))
+        ]
+
+        self.comb += membus.b.ready.eq(1)
+        self.sync += [
+            If(self.reset.re, self.bus_error.status.eq(0)),
+            If(membus.b.valid & membus.b.ready & (membus.b.resp != axi.Response.okay),
+                self.bus_error.status.eq(1))
+        ]
+
+
+class Analyzer(Module, AutoCSR):
+    def __init__(self, tsc, cri, membus, fifo_depth=128):
+        # shutdown procedure: set enable to 0, wait until busy=0
+        self.enable = CSRStorage()
+        self.busy = CSRStatus()
+
+        self.submodules.message_encoder = MessageEncoder(
+            tsc, cri, self.enable.storage)
+        self.submodules.fifo = stream.SyncFIFO(
+            [("data", message_len)], fifo_depth, True)
+        self.submodules.converter = stream.Converter(
+            message_len, len(membus.w.data), reverse=True)
+        self.submodules.dma = AXIDMAWriter(membus, max_outstanding_requests=fifo_depth)
+
+        enable_r = Signal()
+        self.sync += [
+            enable_r.eq(self.enable.storage),
+            If(self.enable.storage & ~enable_r,
+                self.busy.status.eq(1)),
+            If(self.dma.sink.stb & self.dma.sink.ack & self.dma.sink.eop,
+                self.busy.status.eq(0))
+        ]
+
+        self.comb += [
+            self.message_encoder.source.connect(self.fifo.sink),
+            self.fifo.source.connect(self.converter.sink),
+            self.converter.source.connect(self.dma.sink),
+            self.dma.make_request.eq(self.fifo.sink.stb & self.fifo.sink.ack)
+        ]

src/gateware/dma.py

@@ -0,0 +1,157 @@
+from migen import *
+from migen.genlib.fsm import FSM
+from misoc.interconnect.csr import *
+from misoc.interconnect import stream
+from migen_axi.interconnect import axi
+
+from artiq.gateware.rtio.dma import RawSlicer, RecordConverter, RecordSlicer, TimeOffset, CRIMaster
+
+import endianness
+
+
+AXI_BURST_LEN = 16
+
+
+class AXIReader(Module, AutoCSR):
+    def __init__(self, membus):
+        aw = len(membus.ar.addr)
+        dw = len(membus.r.data)
+        alignment_bits = log2_int(AXI_BURST_LEN*dw//8)
+        self.sink = stream.Endpoint([("address", aw - alignment_bits)])
+        self.source = stream.Endpoint([("data", dw)])
+
+        self.bus_error = CSRStatus()
+
+        # # #
+
+        eop_pending = Signal()
+        self.sync += [
+            If(self.sink.stb & self.sink.ack & self.sink.eop, eop_pending.eq(1)),
+            If(self.source.stb & self.source.ack & self.source.eop, eop_pending.eq(0)),
+        ]
+
+        self.comb += [
+            membus.ar.addr.eq(Cat(C(0, alignment_bits), self.sink.address)),
+            membus.ar.id.eq(0),                   # Same ID for all transactions to forbid reordering.
+            membus.ar.burst.eq(axi.Burst.incr.value),
+            membus.ar.len.eq(AXI_BURST_LEN-1),    # Number of transfers in burst (0->1 transfer, 1->2 transfers...).
+            membus.ar.size.eq(log2_int(dw//8)),   # Width of burst: 3 = 8 bytes = 64 bits.
+            membus.ar.cache.eq(0xf),
+            membus.ar.valid.eq(self.sink.stb & ~eop_pending),
+            self.sink.ack.eq(membus.ar.ready & ~eop_pending)
+        ]
+
+        # UG585: "Large slave interface read acceptance capability in the range of 14 to 70 commands"
+        inflight_cnt = Signal(max=128)
+        request_done = Signal()
+        reply_done = Signal()
+        self.comb += [
+            request_done.eq(membus.ar.valid & membus.ar.ready),
+            reply_done.eq(membus.r.valid & membus.r.ready & membus.r.last)
+        ]
+        self.sync += inflight_cnt.eq(inflight_cnt + request_done - reply_done)
+
+        self.comb += [
+            self.source.stb.eq(membus.r.valid),
+            membus.r.ready.eq(self.source.ack),
+            self.source.data.eq(endianness.convert_signal(membus.r.data)),
+            # Note that when eop_pending=1, no new transactions are made and inflight_cnt is no longer incremented
+            self.source.eop.eq(eop_pending & membus.r.last & (inflight_cnt == 1))
+        ]
+
+        stopped = Signal(reset=1)
+        self.sync += [
+            If(self.source.stb & self.source.ack & self.source.eop, stopped.eq(1)),
+            If(self.sink.stb & self.sink.ack, stopped.eq(0)),
+            If(stopped & (self.sink.stb & self.sink.ack),
+                # reset bus error status on new run
+                self.bus_error.status.eq(0)),
+            If(membus.r.valid & membus.r.valid & (membus.r.resp != axi.Response.okay),
+                self.bus_error.status.eq(1))
+        ]
+
+
+class DMAReader(Module, AutoCSR):
+    def __init__(self, membus, enable):
+        aw = len(membus.ar.addr)
+        alignment_bits = log2_int(AXI_BURST_LEN*len(membus.r.data)//8)
+
+        self.submodules.wb_reader = AXIReader(membus)
+        self.source = self.wb_reader.source
+
+        # All numbers in bytes
+        self.base_address = CSRStorage(aw, alignment_bits=alignment_bits)
+
+        # # #
+
+        enable_r = Signal()
+        address = self.wb_reader.sink
+        assert len(address.address) == len(self.base_address.storage)
+        self.sync += [
+            enable_r.eq(enable),
+            If(enable & ~enable_r,
+                address.address.eq(self.base_address.storage),
+                address.eop.eq(0),
+                address.stb.eq(1),
+            ),
+            If(address.stb & address.ack,
+                If(address.eop,
+                    address.stb.eq(0)
+                ).Else(
+                    address.address.eq(address.address + 1),
+                    If(~enable, address.eop.eq(1))
+                )
+            )
+        ]
+
+
+class DMA(Module):
+    def __init__(self, membus):
+        self.enable = CSR()
+
+        flow_enable = Signal()
+        self.submodules.dma = DMAReader(membus, flow_enable)
+        self.submodules.slicer = RecordSlicer(len(membus.r.data))
+        self.submodules.time_offset = TimeOffset()
+        self.submodules.cri_master = CRIMaster()
+        self.cri = self.cri_master.cri
+
+        self.comb += [
+            self.dma.source.connect(self.slicer.sink),
+            self.slicer.source.connect(self.time_offset.sink),
+            self.time_offset.source.connect(self.cri_master.sink)
+        ]
+
+        fsm = FSM(reset_state="IDLE")
+        self.submodules += fsm
+
+        fsm.act("IDLE",
+            If(self.enable.re, NextState("FLOWING"))
+        )
+        fsm.act("FLOWING",
+            self.enable.w.eq(1),
+            flow_enable.eq(1),
+            If(self.slicer.end_marker_found,
+                NextState("FLUSH")
+            )
+        )
+        fsm.act("FLUSH",
+            self.enable.w.eq(1),
+            self.slicer.flush.eq(1),
+            NextState("WAIT_EOP")
+        )
+        fsm.act("WAIT_EOP",
+            self.enable.w.eq(1),
+            If(self.cri_master.sink.stb & self.cri_master.sink.ack & self.cri_master.sink.eop,
+                NextState("WAIT_CRI_MASTER")
+            )
+        )
+        fsm.act("WAIT_CRI_MASTER",
+            self.enable.w.eq(1),
+            If(~self.cri_master.busy, NextState("IDLE"))
+        )
+
+    def get_csrs(self):
+        return ([self.enable] +
+                self.dma.get_csrs() + self.time_offset.get_csrs() +
+                self.cri_master.get_csrs())

src/gateware/endianness.py

@@ -0,0 +1,21 @@
+from migen import *
+
+
+def convert_signal(signal):
+    assert len(signal) % 8 == 0
+    nbytes = len(signal)//8
+    signal_bytes = []
+    for i in range(nbytes):
+        signal_bytes.append(signal[8*i:8*(i+1)])
+    return Cat(*reversed(signal_bytes))
+
+
+def convert_value(value, size):
+    assert size % 8 == 0
+    nbytes = size//8
+    result = 0
+    for i in range(nbytes):
+        result <<= 8
+        result |= value & 0xff
+        value >>= 8
+    return result

src/gateware/test_dma.py

@@ -0,0 +1,236 @@
+import unittest
+import random
+import itertools
+
+from migen import *
+from migen_axi.interconnect import axi
+
+from artiq.coredevice.exceptions import RTIOUnderflow, RTIODestinationUnreachable
+from artiq.gateware import rtio
+from artiq.gateware.rtio import cri
+from artiq.gateware.rtio.phy import ttl_simple
+
+import endianness
+import dma
+
+
+class AXIMemorySim:
+    def __init__(self, bus, data, max_queue=12):
+        self.bus = bus
+        self.data = data
+        self.max_queue = max_queue
+        self.align = len(bus.r.data)//8
+        self.queue = []
+
+    @passive
+    def ar(self):
+        while True:
+            if len(self.queue) < self.max_queue:
+                request = yield from self.bus.read_ar()
+                self.queue.append(request)
+            else:
+                yield
+
+    @passive
+    def r(self):
+        while True:
+            if self.queue:
+                request = self.queue.pop()
+                if request.burst:
+                    request_len = request.len + 1
+                else:
+                    request_len = 1
+                for i in range(request_len):
+                    if request.addr % self.align:
+                        raise ValueError
+                    addr = request.addr//self.align + i
+                    if addr < len(self.data):
+                        data = self.data[addr]
+                    else:
+                        data = 0
+                    data = endianness.convert_value(data, len(self.bus.r.data))
+                    yield from self.bus.write_r(request.id, data, last=i == request_len-1)
+            else:
+                yield
+
+
+def encode_n(n, min_length, max_length):
+    r = []
+    while n:
+        r.append(n & 0xff)
+        n >>= 8
+    r += [0]*(min_length - len(r))
+    if len(r) > max_length:
+        raise ValueError
+    return r
+
+
+def encode_record(channel, timestamp, address, data):
+    r = []
+    r += encode_n(channel, 3, 3)
+    r += encode_n(timestamp, 8, 8)
+    r += encode_n(address, 1, 1)
+    r += encode_n(data, 1, 64)
+    return encode_n(len(r)+1, 1, 1) + r
+
+
+def pack(x, size):
+    r = []
+    for i in range((len(x)+size-1)//size):
+        n = 0
+        for j in range(i*size, (i+1)*size):
+            n <<= 8
+            try:
+                n |= x[j]
+            except IndexError:
+                pass
+        r.append(n)
+    return r
+
+
+def encode_sequence(writes, ws):
+    sequence = [b for write in writes for b in encode_record(*write)]
+    sequence.append(0)
+    return pack(sequence, ws)
+
+
+def do_dma(dut, address):
+    yield from dut.dma.base_address.write(address)
+    yield from dut.enable.write(1)
+    yield
+    while ((yield from dut.enable.read())):
+        yield
+    error = yield from dut.cri_master.error.read()
+    if error & 1:
+        raise RTIOUnderflow
+    if error & 2:
+        raise RTIODestinationUnreachable
+
+
+test_writes1 = [
+    (0x01, 0x23, 0x12, 0x33),
+    (0x901, 0x902, 0x11, 0xeeeeeeeeeeeeeefffffffffffffffffffffffffffffff28888177772736646717738388488),
+    (0x81, 0x288, 0x88, 0x8888)
+]
+
+
+test_writes2 = [
+    (0x10, 0x10000, 0x20, 0x77),
+    (0x11, 0x10001, 0x22, 0x7777),
+    (0x12, 0x10002, 0x30, 0x777777),
+    (0x13, 0x10003, 0x40, 0x77777788),
+    (0x14, 0x10004, 0x50, 0x7777778899),
+]
+
+
+prng = random.Random(0)
+
+
+class TB(Module):
+    def __init__(self, ws):
+        sequence1 = encode_sequence(test_writes1, ws)
+        sequence2 = encode_sequence(test_writes2, ws)
+        offset = 512//ws
+        assert len(sequence1) < offset
+        sequence = (
+            sequence1 +
+            [prng.randrange(2**(ws*8)) for _ in range(offset-len(sequence1))] +
+            sequence2)
+
+        bus = axi.Interface(ws*8)
+        self.memory = AXIMemorySim(bus, sequence)
+        self.submodules.dut = dma.DMA(bus)
+
+
+test_writes_full_stack = [
+    (0, 32, 0, 1),
+    (1, 40, 0, 1),
+    (0, 48, 0, 0),
+    (1, 50, 0, 0),
+]
+
+
+class FullStackTB(Module):
+    def __init__(self, ws):
+        self.ttl0 = Signal()
+        self.ttl1 = Signal()
+
+        self.submodules.phy0 = ttl_simple.Output(self.ttl0)
+        self.submodules.phy1 = ttl_simple.Output(self.ttl1)
+
+        rtio_channels = [
+            rtio.Channel.from_phy(self.phy0),
+            rtio.Channel.from_phy(self.phy1)
+        ]
+
+        sequence = encode_sequence(test_writes_full_stack, ws)
+
+        bus = axi.Interface(ws*8)
+        self.memory = AXIMemorySim(bus, sequence)
+        self.submodules.dut = dma.DMA(bus)
+        self.submodules.tsc = rtio.TSC("async")
+        self.submodules.rtio = rtio.Core(self.tsc, rtio_channels)
+        self.comb += self.dut.cri.connect(self.rtio.cri)
+
+
+class TestDMA(unittest.TestCase):
+    def test_dma_noerror(self):
+        tb = TB(8)
+
+        def do_writes():
+            yield from do_dma(tb.dut, 0)
+            yield from do_dma(tb.dut, 512)
+
+        received = []
+        @passive
+        def rtio_sim():
+            dut_cri = tb.dut.cri
+            while True:
+                cmd = yield dut_cri.cmd
+                if cmd == cri.commands["nop"]:
+                    pass
+                elif cmd == cri.commands["write"]:
+                    channel = yield dut_cri.chan_sel
+                    timestamp = yield dut_cri.o_timestamp
+                    address = yield dut_cri.o_address
+                    data = yield dut_cri.o_data
+                    received.append((channel, timestamp, address, data))
+
+                    yield dut_cri.o_status.eq(1)
+                    for i in range(prng.randrange(10)):
+                        yield
+                    yield dut_cri.o_status.eq(0)
+                else:
+                    self.fail("unexpected RTIO command")
+                yield
+
+        run_simulation(tb, [do_writes(), rtio_sim(), tb.memory.ar(), tb.memory.r()])
+        self.assertEqual(received, test_writes1 + test_writes2)
+
+    def test_full_stack(self):
+        tb = FullStackTB(8)
+
+        ttl_changes = []
+        @passive
+        def monitor():
+            old_ttl_states = [0, 0]
+            for time in itertools.count():
+                ttl_states = [
+                    (yield tb.ttl0),
+                    (yield tb.ttl1)
+                ]
+                for i, (old, new) in enumerate(zip(old_ttl_states, ttl_states)):
+                    if new != old:
+                        ttl_changes.append((time, i))
+                old_ttl_states = ttl_states
+                yield
+
+        run_simulation(tb, {"sys": [
+            do_dma(tb.dut, 0), monitor(),
+            (None for _ in range(70)),
+            tb.memory.ar(), tb.memory.r()
+        ]}, {"sys": 8, "rsys": 8, "rtio": 8, "rio": 8, "rio_phy": 8})
+
+        correct_changes = [(timestamp + 11, channel)
+                           for channel, timestamp, _, _ in test_writes_full_stack]
+        self.assertEqual(ttl_changes, correct_changes)

src/gateware/zc706.py

@@ -1,8 +1,10 @@
 #!/usr/bin/env python
 
 import argparse
+from operator import itemgetter
 
 from migen import *
+from migen.build.generic_platform import *
 from migen.genlib.resetsync import AsyncResetSynchronizer
 from migen.genlib.cdc import MultiReg
 from migen_axi.integration.soc_core import SoCCore
@@ -13,6 +15,10 @@ from misoc.integration import cpu_interface
 from artiq.gateware import rtio, nist_clock, nist_qc2
 from artiq.gateware.rtio.phy import ttl_simple, ttl_serdes_7series, dds, spi2
 
+import dma
+import analyzer
+import acpki
+
 
 class RTIOCRG(Module, AutoCSR):
     def __init__(self, platform, rtio_internal_clk):
@@ -59,12 +65,18 @@ class RTIOCRG(Module, AutoCSR):
 
 
 class ZC706(SoCCore):
-    def __init__(self):
+    def __init__(self, acpki=False):
+        self.acpki = acpki
+        self.rustc_cfg = dict()
+
         platform = zc706.Platform()
         platform.toolchain.bitstream_commands.extend([
             "set_property BITSTREAM.GENERAL.COMPRESS True [current_design]",
         ])
-        SoCCore.__init__(self, platform=platform, csr_data_width=32, ident=self.__class__.__name__)
+        ident = self.__class__.__name__
+        if self.acpki:
+            ident = "acpki_" + ident
+        SoCCore.__init__(self, platform=platform, csr_data_width=32, ident=ident)
 
         platform.add_platform_command("create_clock -name clk_fpga_0 -period 8 [get_pins \"PS7/FCLKCLK[0]\"]")
         platform.add_platform_command("set_input_jitter clk_fpga_0 0.24")
@@ -80,41 +92,81 @@ class ZC706(SoCCore):
         self.submodules.rtio_tsc = rtio.TSC("async", glbl_fine_ts_width=3)
         self.submodules.rtio_core = rtio.Core(self.rtio_tsc, rtio_channels)
         self.csr_devices.append("rtio_core")
+
+        if self.acpki:
+            self.rustc_cfg["ki_impl"] = "acp"
+            self.submodules.rtio = acpki.KernelInitiator(self.rtio_tsc,
+                                                         bus=self.ps7.s_axi_acp,
+                                                         user=self.ps7.s_axi_acp_user,
+                                                         evento=self.ps7.event.o)
+            self.csr_devices.append("rtio")
+        else:
+            self.rustc_cfg["ki_impl"] = "csr"
             self.submodules.rtio = rtio.KernelInitiator(self.rtio_tsc, now64=True)
             self.csr_devices.append("rtio")
 
-        self.comb += self.rtio.cri.connect(self.rtio_core.cri)
+        self.submodules.rtio_dma = dma.DMA(self.ps7.s_axi_hp0)
+        self.csr_devices.append("rtio_dma")
+
+        self.submodules.cri_con = rtio.CRIInterconnectShared(
+            [self.rtio.cri, self.rtio_dma.cri],
+            [self.rtio_core.cri])
+        self.csr_devices.append("cri_con")
 
         self.submodules.rtio_moninj = rtio.MonInj(rtio_channels)
         self.csr_devices.append("rtio_moninj")
 
+        self.submodules.rtio_analyzer = analyzer.Analyzer(self.rtio_tsc, self.rtio_core.cri,
+                                                          self.ps7.s_axi_hp1)
+        self.csr_devices.append("rtio_analyzer")
+
 
 class Simple(ZC706):
-    def __init__(self):
+    def __init__(self, **kwargs):
-        ZC706.__init__(self)
+        ZC706.__init__(self, **kwargs)
 
         platform = self.platform
 
         rtio_channels = []
         for i in range(4):
-            pad = platform.request("user_led", i)
+            phy = ttl_simple.Output(platform.request("user_led", i))
-            phy = ttl_simple.Output(pad)
             self.submodules += phy
             rtio_channels.append(rtio.Channel.from_phy(phy))
+
+        self.config["RTIO_LOG_CHANNEL"] = len(rtio_channels)
+        rtio_channels.append(rtio.LogChannel())
+
         self.add_rtio(rtio_channels)
 
 
+# The NIST backplanes require setting VADJ to 3.3V by reprogramming the power supply.
+# This also changes the I/O standard for some on-board LEDs.
+leds_fmc33 = [
+    ("user_led_33", 0, Pins("Y21"), IOStandard("LVCMOS33")),
+    ("user_led_33", 1, Pins("G2"), IOStandard("LVCMOS15")),
+    ("user_led_33", 2, Pins("W21"), IOStandard("LVCMOS33")),
+    ("user_led_33", 3, Pins("A17"), IOStandard("LVCMOS15")),
+]
+
+
 class NIST_CLOCK(ZC706):
     """
     NIST clock hardware, with old backplane and 11 DDS channels
     """
-    def __init__(self):
+    def __init__(self, **kwargs):
-        ZC706.__init__(self)
+        ZC706.__init__(self, **kwargs)
 
         platform = self.platform
         platform.add_extension(nist_clock.fmc_adapter_io)
+        platform.add_extension(leds_fmc33)
 
         rtio_channels = []
+
+        for i in range(4):
+            phy = ttl_simple.Output(platform.request("user_led_33", i))
+            self.submodules += phy
+            rtio_channels.append(rtio.Channel.from_phy(phy))
+
         for i in range(16):
             if i % 4 == 3:
                 phy = ttl_serdes_7series.InOut_8X(platform.request("ttl", i))
@@ -130,10 +182,6 @@ class NIST_CLOCK(ZC706):
             self.submodules += phy
             rtio_channels.append(rtio.Channel.from_phy(phy, ififo_depth=512))
 
-        phy = ttl_simple.Output(platform.request("user_led", 1))
-        self.submodules += phy
-        rtio_channels.append(rtio.Channel.from_phy(phy))
-
         phy = ttl_simple.ClockGen(platform.request("la32_p"))
         self.submodules += phy
         rtio_channels.append(rtio.Channel.from_phy(phy))
@@ -148,6 +196,9 @@ class NIST_CLOCK(ZC706):
         self.submodules += phy
         rtio_channels.append(rtio.Channel.from_phy(phy, ififo_depth=4))
 
+        self.config["RTIO_LOG_CHANNEL"] = len(rtio_channels)
+        rtio_channels.append(rtio.LogChannel())
+
         self.add_rtio(rtio_channels)
 
 
@@ -156,14 +207,19 @@ class NIST_QC2(ZC706):
     NIST QC2 hardware, as used in Quantum I and Quantum II, with new backplane
     and 24 DDS channels.  Two backplanes are used.
     """
-    def __init__(self):
+    def __init__(self, **kwargs):
-        ZC706.__init__(self)
+        ZC706.__init__(self, **kwargs)
 
         platform = self.platform
         platform.add_extension(nist_qc2.fmc_adapter_io)
+        platform.add_extension(leds_fmc33)
 
         rtio_channels = []
-        clock_generators = []
+
+        for i in range(4):
+            phy = ttl_simple.Output(platform.request("user_led_33", i))
+            self.submodules += phy
+            rtio_channels.append(rtio.Channel.from_phy(phy))
 
         # All TTL channels are In+Out capable
         for i in range(40):
@@ -176,15 +232,8 @@ class NIST_QC2(ZC706):
             phy = ttl_simple.ClockGen(
                 platform.request("clkout", i))
             self.submodules += phy
-            clock_generators.append(rtio.Channel.from_phy(phy))
-
-        phy = ttl_simple.Output(platform.request("user_led", 1))
-        self.submodules += phy
             rtio_channels.append(rtio.Channel.from_phy(phy))
 
-        # add clock generators after TTLs
-        rtio_channels += clock_generators
-
         for i in range(4):
             phy = spi2.SPIMaster(self.platform.request("spi", i))
             self.submodules += phy
@@ -197,6 +246,9 @@ class NIST_QC2(ZC706):
             self.submodules += phy
             rtio_channels.append(rtio.Channel.from_phy(phy, ififo_depth=4))
 
+        self.config["RTIO_LOG_CHANNEL"] = len(rtio_channels)
+        rtio_channels.append(rtio.LogChannel())
+
         self.add_rtio(rtio_channels)
 
 
@@ -209,31 +261,48 @@ def write_csr_file(soc, filename):
             soc.get_csr_regions(), soc.get_csr_groups(), soc.get_constants()))
 
 
+def write_rustc_cfg_file(soc, filename):
+    with open(filename, "w") as f:
+        for k, v in sorted(soc.rustc_cfg.items(), key=itemgetter(0)):
+            if v is None:
+                f.write("{}\n".format(k))
+            else:
+                f.write("{}=\"{}\"\n".format(k, v))
+
+
 def main():
     parser = argparse.ArgumentParser(
         description="ARTIQ port to the ZC706 Zynq development kit")
     parser.add_argument("-r", default=None,
         help="build Rust interface into the specified file")
+    parser.add_argument("-c", default=None,
+        help="build Rust compiler configuration into the specified file")
     parser.add_argument("-g", default=None,
         help="build gateware into the specified directory")
     parser.add_argument("-V", "--variant", default="simple",
                         help="variant: "
-                             "simple/nist_clock/nist_qc2 "
+                             "[acpki_]simple/nist_clock/nist_qc2 "
                              "(default: %(default)s)")
     args = parser.parse_args()
 
+    variant = args.variant.lower()
+    acpki = variant.startswith("acpki_")
+    if acpki:
+        variant = variant[6:]
     try:
-        cls = VARIANTS[args.variant.lower()]
+        cls = VARIANTS[variant]
     except KeyError:
         raise SystemExit("Invalid variant (-V/--variant)")
 
-    soc = cls()
+    soc = cls(acpki=acpki)
     soc.finalize()
 
-    if args.g is not None:
-        soc.build(build_dir=args.g)
     if args.r is not None:
         write_csr_file(soc, args.r)
+    if args.c is not None:
+        write_rustc_cfg_file(soc, args.c)
+    if args.g is not None:
+        soc.build(build_dir=args.g)
 
 
 if __name__ == "__main__":

src/libc/Cargo.toml

@@ -1,14 +1,12 @@
 [package]
 name = "libc"
 version = "0.1.0"
-authors = ["pca006132 <john.lck40@gmail.com>"]
+authors = ["M-Labs"]
 edition = "2018"
 build = "build.rs"
 
-# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
-
 [dependencies]
-libboard_zynq = { git = "https://git.m-labs.hk/M-Labs/zc706.git" }
+libboard_zynq = { git = "https://git.m-labs.hk/M-Labs/zynq-rs.git" }
 
 [build-dependencies]
 cc = { version = "1.0.1" }

src/libc/src/lib.rs

@@ -4,7 +4,9 @@
 
 use libboard_zynq::stdio;
 
+pub type c_char = i8;
 pub type c_int = i32;
+pub type size_t = usize;
 pub type uintptr_t = usize;
 pub type c_void = core::ffi::c_void;
 

src/libdyld/Cargo.toml

@@ -8,3 +8,4 @@ name = "dyld"
 
 [dependencies]
 log = "0.4"
+libcortex_a9 = { git = "https://git.m-labs.hk/M-Labs/zynq-rs.git" }

src/libdyld/src/file.rs

@@ -56,6 +56,14 @@ impl<'a> File<'a> {
         })
     }
 
+    pub fn section_headers<'b>(&'b self) -> impl Iterator<Item = Option<Elf32_Shdr>> + 'b
+    {
+        (0..self.ehdr.e_shnum).map(move |i| {
+            let shdr_off = self.ehdr.e_shoff as usize + mem::size_of::<Elf32_Shdr>() * i as usize;
+            self.read_unaligned::<Elf32_Shdr>(shdr_off)
+        })
+    }
+
     pub fn dyn_header_vaddr(&self) -> Option<Range<usize>> {
         self.program_headers()
             .filter_map(|phdr| phdr)

src/libdyld/src/lib.rs

@@ -2,8 +2,9 @@
 
 extern crate alloc;
 extern crate log;
+extern crate libcortex_a9;
 
-use core::{convert, fmt, str};
+use core::{convert, fmt, ops::Range, str};
 use alloc::string::String;
 use log::{debug, trace};
 use elf::*;
@@ -57,43 +58,11 @@ fn elf_hash(name: &[u8]) -> u32 {
     h
 }
 
-// linker symbols
-extern "C" {
-    #[no_mangle]
-    static __text_start: u32;
-    #[no_mangle]
-    static __text_end: u32;
-    #[no_mangle]
-    static __exidx_start: u32;
-    #[no_mangle]
-    static __exidx_end: u32;
-}
-
-static mut KERNEL_EXIDX_START: u32 = 0;
-static mut KERNEL_EXIDX_END: u32 = 0;
-
-#[no_mangle]
-extern fn dl_unwind_find_exidx(pc: u32, len_ptr: *mut u32) -> u32 {
-    let length: u32;
-    let start: u32;
-    unsafe {
-        if (&__text_start as *const u32 as u32) <= pc && pc < (&__text_end as *const u32 as u32) {
-            length = (&__exidx_end - &__exidx_start) as u32;
-            start = &__exidx_start as *const u32 as u32;
-        } else {
-            // make sure that the kernel is loaded
-            assert_ne!(KERNEL_EXIDX_START, 0);
-            length = (KERNEL_EXIDX_END - KERNEL_EXIDX_START) / core::mem::size_of::<u32>() as u32;
-            start = KERNEL_EXIDX_START;
-        }
-        *len_ptr = length;
-    }
-    start
-}
-
 pub struct Library {
     pub image: Image,
+    pub arch: Arch,
     dyn_section: DynamicSection,
+    exidx: Range<usize>,
 }
 
 impl Library {
@@ -163,6 +132,15 @@ impl Library {
         Ok(self.strtab().get(offset..offset + size)
            .ok_or("cannot read symbol name")?)
     }
+
+    /// Rebind Rela by `name` to a new `addr`
+    pub fn rebind(&self, name: &[u8], addr: *const ()) -> Result<(), Error> {
+        reloc::rebind(self.arch, self, name, addr as Elf32_Word)
+    }
+
+    pub fn exidx(&self) -> &[u32] {
+        self.image.get_ref_slice_unchecked(&self.exidx)
+    }
 }
 
 pub fn load(
@@ -215,15 +193,22 @@ pub fn load(
                     .ok_or("program header requests an out of bounds load (in target)")?;
                 dst.copy_from_slice(src);
             }
-            PT_ARM_EXIDX => {
+            _ => {}
-                let range = image.get(phdr.p_vaddr as usize..
-                    (phdr.p_vaddr + phdr.p_filesz) as usize)
-                    .ok_or("program header requests and out of bounds load (in target)")?;
-                unsafe {
-                    KERNEL_EXIDX_START = range.as_ptr() as u32;
-                    KERNEL_EXIDX_END = range.as_ptr().add(range.len()) as u32;
         }
     }
+
+    let mut exidx = None;
+    // Obtain EXIDX
+    for shdr in file.section_headers() {
+        let shdr = shdr.ok_or("cannot read section header")?;
+        match shdr.sh_type as usize {
+            SHT_ARM_EXIDX => {
+                let range = shdr.sh_addr as usize..
+                    (shdr.sh_addr + shdr.sh_size) as usize;
+                let _ = image.get(range.clone())
+                    .ok_or("section header specifies EXIDX outside of image (in target)")?;
+                exidx = Some(range);
+            }
             _ => {}
         }
     }
@@ -235,8 +220,10 @@ pub fn load(
     debug!("Relocating {} rela, {} rel, {} pltrel",
            dyn_section.rela.len(), dyn_section.rel.len(), dyn_section.pltrel.len());
     let lib = Library {
+        arch,
         image,
-        dyn_section
+        dyn_section,
+        exidx: exidx.ok_or("no EXIDX section")?,
     };
 
     for rela in lib.rela() {

src/libdyld/src/reloc.rs

@@ -7,6 +7,10 @@ use super::{
     image::Image,
     Library,
 };
+use libcortex_a9::{
+    cache::{dcci_slice, iciallu, bpiall},
+    asm::{dsb, isb},
+};
 
 pub trait Relocatable {
     fn offset(&self) -> usize;
@@ -133,3 +137,34 @@ pub fn relocate<R: Relocatable>(
 
     lib.image.write(rel.offset(), value)
 }
+
+pub fn rebind(
+    arch: Arch, lib: &Library, name: &[u8], value: Elf32_Word
+) -> Result<(), Error> {
+    for rela in lib.pltrel() {
+        let rel_type = RelType::new(arch, rela.type_info())
+            .ok_or("unsupported relocation type")?;
+        match rel_type {
+            RelType::Lookup => {
+                let sym = lib.symtab().get(ELF32_R_SYM(rela.r_info) as usize)
+                    .ok_or("symbol out of bounds of symbol table")?;
+                let sym_name = lib.name_starting_at(sym.st_name as usize)?;
+
+                if sym_name == name {
+                    lib.image.write(rela.offset(), value)?
+                }
+            }
+            // No associated symbols for other relocation types.
+            _ => {}
+        }
+    }
+    // FIXME: the cache maintainance operations may be more than enough,
+    // may cause performance degradation.
+    dcci_slice(lib.image.data);
+    iciallu();
+    bpiall();
+    dsb();
+    isb();
+
+    Ok(())
+}

src/llvm_libunwind/src/Unwind-EHABI.cpp

@@ -551,6 +551,7 @@ static _Unwind_Reason_Code unwind_phase2(unw_context_t *uc, unw_cursor_t *cursor
                              static_cast<void *>(exception_object));
   int frame_count = 0;
 
+  unw_word_t prev_sp = 0x0;
   // Walk each frame until we reach where search phase said to stop.
   while (true) {
     // Ask libunwind to get next frame (skip over first which is
@@ -576,6 +577,10 @@ static _Unwind_Reason_Code unwind_phase2(unw_context_t *uc, unw_cursor_t *cursor
     unw_word_t sp;
     unw_proc_info_t frameInfo;
     __unw_get_reg(cursor, UNW_REG_SP, &sp);
+    if (sp == prev_sp) {
+        return _URC_END_OF_STACK;
+    }
+    prev_sp = sp;
     if (__unw_get_proc_info(cursor, &frameInfo) != UNW_ESUCCESS) {
       _LIBUNWIND_TRACE_UNWINDING(
           "unwind_phase2(ex_ojb=%p): __unw_get_proc_info "

src/runtime/Cargo.toml

@@ -22,12 +22,15 @@ void = { version = "1", default-features = false }
 futures = { version = "0.3", default-features = false, features = ["async-await"] }
 async-recursion = "0.3"
 fatfs = { version = "0.3", features = ["core_io"], default-features = false }
+log_buffer = { version = "1.2" }
+libm = { version = "0.2", features = ["unstable"] }
+vcell = "0.1"
 
-libboard_zynq = { git = "https://git.m-labs.hk/M-Labs/zc706.git" }
+libboard_zynq = { git = "https://git.m-labs.hk/M-Labs/zynq-rs.git" }
-libsupport_zynq = { default-features = false, git = "https://git.m-labs.hk/M-Labs/zc706.git" }
+libsupport_zynq = { default-features = false, features = ["alloc_core"], git = "https://git.m-labs.hk/M-Labs/zynq-rs.git" }
-libcortex_a9 = { git = "https://git.m-labs.hk/M-Labs/zc706.git" }
+libcortex_a9 = { git = "https://git.m-labs.hk/M-Labs/zynq-rs.git" }
-libasync = { git = "https://git.m-labs.hk/M-Labs/zc706.git" }
+libasync = { git = "https://git.m-labs.hk/M-Labs/zynq-rs.git" }
-libregister = { git = "https://git.m-labs.hk/M-Labs/zc706.git" }
+libregister = { git = "https://git.m-labs.hk/M-Labs/zynq-rs.git" }
 
 dyld = { path = "../libdyld" }
 dwarf = { path = "../libdwarf" }

src/runtime/build.rs

@@ -1,6 +1,7 @@
 use std::env;
 use std::fs::File;
 use std::io::Write;
+use std::io::{BufRead, BufReader};
 use std::path::PathBuf;
 
 fn main() {
@@ -15,4 +16,13 @@ fn main() {
     // Only re-run the build script when link.x is changed,
     // instead of when any part of the source code changes.
     println!("cargo:rerun-if-changed=link.x");
+
+    // Handle rustc-cfg file
+    let cfg_path = "../../build/rustc-cfg";
+    println!("cargo:rerun-if-changed={}", cfg_path);
+
+    let f = BufReader::new(File::open(cfg_path).unwrap());
+    for line in f.lines() {
+        println!("cargo:rustc-cfg={}", line.unwrap());
+    }
 }

src/runtime/link.x

@@ -48,9 +48,12 @@ SECTIONS
 
     .heap (NOLOAD) : ALIGN(8)
     {
-        __heap_start = .;
+        __heap0_start = .;
-        . += 0x1000000;
+        . += 0x800000;
-        __heap_end = .;
+        __heap0_end = .;
+        __heap1_start = .;
+        . += 0x800000;
+        __heap1_end = .;
     } > SDRAM
 
     .stack1 (NOLOAD) : ALIGN(8)

src/runtime/src/analyzer.rs

@@ -0,0 +1,111 @@
+use libasync::{smoltcp::TcpStream, task};
+use libboard_zynq::smoltcp::Error;
+use libcortex_a9::cache;
+use log::{debug, info, warn};
+
+use crate::proto_async::*;
+use crate::pl;
+
+const BUFFER_SIZE: usize = 512 * 1024;
+
+#[repr(align(64))]
+struct Buffer {
+    data: [u8; BUFFER_SIZE],
+}
+
+static mut BUFFER: Buffer = Buffer {
+    data: [0; BUFFER_SIZE]
+};
+
+fn arm() {
+    debug!("arming RTIO analyzer");
+    unsafe {
+        let base_addr = &mut BUFFER.data[0] as *mut _ as usize;
+        let last_addr = &mut BUFFER.data[BUFFER_SIZE - 1] as *mut _ as usize;
+        pl::csr::rtio_analyzer::message_encoder_overflow_reset_write(1);
+        pl::csr::rtio_analyzer::dma_base_address_write(base_addr as u32);
+        pl::csr::rtio_analyzer::dma_last_address_write(last_addr as u32);
+        pl::csr::rtio_analyzer::dma_reset_write(1);
+        pl::csr::rtio_analyzer::enable_write(1);
+    }
+}
+
+fn disarm() {
+    debug!("disarming RTIO analyzer");
+    unsafe {
+        pl::csr::rtio_analyzer::enable_write(0);
+        while pl::csr::rtio_analyzer::busy_read() != 0 {}
+        cache::dcci_slice(&BUFFER.data);
+    }
+    debug!("RTIO analyzer disarmed");
+}
+
+#[derive(Debug)]
+struct Header {
+    sent_bytes: u32,
+    total_byte_count: u64,
+    error_occurred: bool,
+    log_channel: u8,
+    dds_onehot_sel: bool
+}
+
+async fn write_header(stream: &mut TcpStream, header: &Header) -> Result<(), Error> {
+    write_i32(stream, header.sent_bytes as i32).await?;
+    write_i64(stream, header.total_byte_count as i64).await?;
+    write_i8(stream, header.error_occurred as i8).await?;
+    write_i8(stream, header.log_channel as i8).await?;
+    write_i8(stream, header.dds_onehot_sel as i8).await?;
+    Ok(())
+}
+
+async fn handle_connection(stream: &mut TcpStream) -> Result<(), Error> {
+    info!("received connection");
+
+    let data = unsafe { &BUFFER.data[..] };
+    let overflow_occurred = unsafe { pl::csr::rtio_analyzer::message_encoder_overflow_read() != 0 };
+    let bus_error_occurred = unsafe { pl::csr::rtio_analyzer::dma_bus_error_read() != 0 };
+    let total_byte_count = unsafe { pl::csr::rtio_analyzer::dma_byte_count_read() as u64 };
+    let pointer = (total_byte_count % BUFFER_SIZE as u64) as usize;
+    let wraparound = total_byte_count >= BUFFER_SIZE as u64;
+
+    if overflow_occurred {
+        warn!("overflow occured");
+    }
+    if bus_error_occurred {
+        warn!("bus error occured");
+    }
+
+    let header = Header {
+        total_byte_count: total_byte_count,
+        sent_bytes: if wraparound { BUFFER_SIZE as u32 } else { total_byte_count as u32 },
+        error_occurred: overflow_occurred | bus_error_occurred,
+        log_channel: pl::csr::CONFIG_RTIO_LOG_CHANNEL as u8,
+        dds_onehot_sel: true  // kept for backward compatibility of analyzer dumps
+    };
+    debug!("{:?}", header);
+
+    write_header(stream, &header).await?;
+    if wraparound {
+        stream.send(data[pointer..].iter().copied()).await?;
+        stream.send(data[..pointer].iter().copied()).await?;
+    } else {
+        stream.send(data[..pointer].iter().copied()).await?;
+    }
+
+    Ok(())
+}
+
+pub fn start() {
+    task::spawn(async move {
+        loop {
+            arm();
+            let mut stream = TcpStream::accept(1382, 2048, 2048).await.unwrap();
+            disarm();
+            let _ = handle_connection(&mut stream)
+                .await
+                .map_err(|e| warn!("connection terminated: {:?}", e));
+            let _ = stream.flush().await;
+            let _ = stream.close().await;
+        }
+    });
+}

src/runtime/src/comms.rs

@@ -1,9 +1,7 @@
 use core::fmt;
 use core::cell::RefCell;
 use core::str::Utf8Error;
-use alloc::rc::Rc;
+use alloc::{vec, vec::Vec, string::String, collections::BTreeMap, rc::Rc};
-use alloc::sync::Arc;
-use alloc::{vec, vec::Vec, string::String};
 use log::{info, warn, error};
 
 use num_derive::{FromPrimitive, ToPrimitive};
@@ -19,6 +17,8 @@ use libboard_zynq::{
     },
     timer::GlobalTimer,
 };
+use libcortex_a9::{semaphore::Semaphore, mutex::Mutex};
+use futures::{select_biased, future::FutureExt};
 use libasync::{smoltcp::{Sockets, TcpStream}, task};
 
 use crate::config;
@@ -27,6 +27,8 @@ use crate::proto_async::*;
 use crate::kernel;
 use crate::rpc;
 use crate::moninj;
+use crate::mgmt;
+use crate::analyzer;
 
 
 #[derive(Debug, Clone, Copy, PartialEq, Eq)]
@@ -80,6 +82,9 @@ enum Reply {
     ClockFailure = 15,
 }
 
+static CACHE_STORE: Mutex<BTreeMap<String, Vec<i32>>> = Mutex::new(BTreeMap::new());
+static DMA_RECORD_STORE: Mutex<BTreeMap<String, (Vec<u8>, i64)>> = Mutex::new(BTreeMap::new());
+
 async fn write_header(stream: &TcpStream, reply: Reply) -> Result<()> {
     stream.send([0x5a, 0x5a, 0x5a, 0x5a, reply.to_u8().unwrap()].iter().copied()).await?;
     Ok(())
@@ -124,7 +129,7 @@ async fn handle_run_kernel(stream: Option<&TcpStream>, control: &Rc<RefCell<kern
     control.borrow_mut().tx.async_send(kernel::Message::StartRequest).await;
     loop {
         let reply = control.borrow_mut().rx.async_recv().await;
-        match *reply {
+        match reply {
             kernel::Message::RpcSend { is_async, data } => {
                 if stream.is_none() {
                     error!("Unexpected RPC from startup/idle kernel!");
@@ -139,7 +144,7 @@ async fn handle_run_kernel(stream: Option<&TcpStream>, control: &Rc<RefCell<kern
                     match host_request {
                         Request::RPCReply => {
                             let tag = read_bytes(stream, 512).await?;
-                            let slot = match *control.borrow_mut().rx.async_recv().await {
+                            let slot = match control.borrow_mut().rx.async_recv().await {
                                 kernel::Message::RpcRecvRequest(slot) => slot,
                                 other => panic!("expected root value slot from core1, not {:?}", other),
                             };
@@ -153,7 +158,7 @@ async fn handle_run_kernel(stream: Option<&TcpStream>, control: &Rc<RefCell<kern
                                     } else {
                                         let mut control = control.borrow_mut();
                                         control.tx.async_send(kernel::Message::RpcRecvReply(Ok(size))).await;
-                                        match *control.rx.async_recv().await {
+                                        match control.rx.async_recv().await {
                                             kernel::Message::RpcRecvRequest(slot) => slot,
                                             other => panic!("expected nested value slot from kernel CPU, not {:?}", other),
                                         }
@@ -164,7 +169,7 @@ async fn handle_run_kernel(stream: Option<&TcpStream>, control: &Rc<RefCell<kern
                         },
                         Request::RPCException => {
                             let mut control = control.borrow_mut();
-                            match *control.rx.async_recv().await {
+                            match control.rx.async_recv().await {
                                 kernel::Message::RpcRecvRequest(_) => (),
                                 other => panic!("expected (ignored) root value slot from kernel CPU, not {:?}", other),
                             }
@@ -220,6 +225,25 @@ async fn handle_run_kernel(stream: Option<&TcpStream>, control: &Rc<RefCell<kern
                 }
                 break;
             }
+            kernel::Message::CachePutRequest(key, value) => {
+                CACHE_STORE.lock().insert(key, value);
+            },
+            kernel::Message::CacheGetRequest(key) => {
+                const DEFAULT: Vec<i32> = Vec::new();
+                let value = CACHE_STORE.lock().get(&key).unwrap_or(&DEFAULT).clone();
+                control.borrow_mut().tx.async_send(kernel::Message::CacheGetReply(value)).await;
+            },
+            kernel::Message::DmaPutRequest(recorder) => {
+                DMA_RECORD_STORE.lock().insert(recorder.name, (recorder.buffer, recorder.duration));
+            },
+            kernel::Message::DmaEraseRequest(name) => {
+                // prevent possible OOM when we have large DMA record replacement.
+                DMA_RECORD_STORE.lock().remove(&name);
+            },
+            kernel::Message::DmaGetRequest(name) => {
+                let result = DMA_RECORD_STORE.lock().get(&name).map(|v| v.clone());
+                control.borrow_mut().tx.async_send(kernel::Message::DmaGetReply(result)).await;
+            },
             _ => {
                 panic!("unexpected message from core1 while kernel was running: {:?}", reply);
             }
@@ -229,12 +253,12 @@ async fn handle_run_kernel(stream: Option<&TcpStream>, control: &Rc<RefCell<kern
 }
 
 
-async fn load_kernel(buffer: Vec<u8>, control: &Rc<RefCell<kernel::Control>>, stream: Option<&TcpStream>) -> Result<()> {
+async fn load_kernel(buffer: &Vec<u8>, control: &Rc<RefCell<kernel::Control>>, stream: Option<&TcpStream>) -> Result<()> {
     let mut control = control.borrow_mut();
     control.restart();
-    control.tx.async_send(kernel::Message::LoadRequest(Arc::new(buffer))).await;
+    control.tx.async_send(kernel::Message::LoadRequest(buffer.to_vec())).await;
     let reply = control.rx.async_recv().await;
-    match *reply {
+    match reply {
         kernel::Message::LoadCompleted => {
             if let Some(stream) = stream {
                 write_header(stream, Reply::LoadCompleted).await?;
@@ -262,8 +286,9 @@ async fn load_kernel(buffer: Vec<u8>, control: &Rc<RefCell<kernel::Control>>, st
 }
 
 async fn handle_connection(stream: &TcpStream, control: Rc<RefCell<kernel::Control>>) -> Result<()> {
-    expect(stream, b"ARTIQ coredev\n").await?;
+    if !expect(stream, b"ARTIQ coredev\n").await? {
-    info!("received connection");
+        return Err(Error::UnexpectedPattern);
+    }
     loop {
         let request = read_request(stream, true).await?;
         if request.is_none() {
@@ -277,7 +302,7 @@ async fn handle_connection(stream: &TcpStream, control: Rc<RefCell<kernel::Contr
             },
             Request::LoadKernel => {
                 let buffer = read_bytes(stream, 1024*1024).await?;
-                load_kernel(buffer, &control, Some(stream)).await?;
+                load_kernel(&buffer, &control, Some(stream)).await?;
             },
             Request::RunKernel => {
                 handle_run_kernel(Some(stream), &control).await?;
@@ -331,10 +356,15 @@ pub fn main(timer: GlobalTimer, cfg: &config::Config) {
 
     Sockets::init(32);
 
+    mgmt::start();
+    analyzer::start();
+    moninj::start(timer);
+
     let control: Rc<RefCell<kernel::Control>> = Rc::new(RefCell::new(kernel::Control::start()));
+    let idle_kernel = Rc::new(cfg.read("idle").ok());
     if let Ok(buffer) = cfg.read("startup") {
         info!("Loading startup kernel...");
-        if let Ok(()) = task::block_on(load_kernel(buffer, &control, None)) {
+        if let Ok(()) = task::block_on(load_kernel(&buffer, &control, None)) {
             info!("Starting startup kernel...");
             let _ = task::block_on(handle_run_kernel(None, &control));
             info!("Startup kernel finished!");
@@ -344,21 +374,49 @@ pub fn main(timer: GlobalTimer, cfg: &config::Config) {
     }
 
     task::spawn(async move {
+        let connection = Rc::new(Semaphore::new(1, 1));
+        let terminate = Rc::new(Semaphore::new(0, 1));
         loop {
             let stream = TcpStream::accept(1381, 2048, 2048).await.unwrap();
+
+            if connection.try_wait().is_none() {
+                // there is an existing connection
+                terminate.signal();
+                connection.async_wait().await;
+            }
+
             let control = control.clone();
-            task::spawn(async {
+            let idle_kernel = idle_kernel.clone();
-                let _ = handle_connection(&stream, control)
+            let connection = connection.clone();
+            let terminate = terminate.clone();
+
+            // we make sure the value of terminate is 0 before we start
+            let _ = terminate.try_wait();
+            task::spawn(async move {
+                select_biased! {
+                    _ = (async {
+                        let _ = handle_connection(&stream, control.clone())
                             .await
                             .map_err(|e| warn!("connection terminated: {}", e));
+                        if let Some(buffer) = &*idle_kernel {
+                            info!("Loading idle kernel");
+                            let _ = load_kernel(&buffer, &control, None)
+                                .await.map_err(|e| warn!("error loading idle kernel"));
+                            info!("Running idle kernel");
+                            let _ = handle_run_kernel(None, &control)
+                                .await.map_err(|e| warn!("error running idle kernel"));
+                            info!("Idle kernel terminated");
+                        }
+                    }).fuse() => (),
+                    _ = terminate.async_wait().fuse() => ()
+                }
+                connection.signal();
                 let _ = stream.flush().await;
                 let _ = stream.abort().await;
             });
         }
     });
 
-    moninj::start(timer);
-
     Sockets::run(&mut iface, || {
         Instant::from_millis(timer.get_time().0 as i32)
     });

src/runtime/src/eh_artiq.rs

@@ -128,8 +128,15 @@ pub unsafe fn artiq_personality(state: uw::_Unwind_State,
     let exception = &exception_info.exception.unwrap();
     if search_phase {
         match eh_action {
-            EHAction::None |
+            EHAction::None => return continue_unwind(exception_object, context),
-            EHAction::Cleanup(_) => return continue_unwind(exception_object, context),
+            // Actually, cleanup should not return handler found, this is to workaround
+            // the issue of terminating directly when no catch cause is found while
+            // having some cleanup routines defined by finally.
+            // The best way to handle this is to force unwind the stack in the raise
+            // function when end of stack is reached, and call terminate at the end of
+            // the unwind. Unfortunately, there is no forced unwind function defined
+            // for EHABI, and I have no idea how to implement that, so this is a hack.
+            EHAction::Cleanup(_) => return uw::_URC_HANDLER_FOUND,
             EHAction::Catch(_) => {
                 // EHABI requires the personality routine to update the
                 // SP value in the barrier cache of the exception object.
@@ -195,10 +202,9 @@ static mut INFLIGHT: ExceptionInfo = ExceptionInfo {
 };
 
 pub unsafe extern fn raise(exception: *const Exception) -> ! {
-    // Zing! The Exception<'a> to Exception<'static> transmute is not really sound in case
-    // the exception is ever captured. Fortunately, they currently aren't, and we save
-    // on the hassle of having to allocate exceptions somewhere except on stack.
     trace!("Trying to raise exception");
+    // FIXME: unsound transmute
+    // This would cause stack memory corruption.
     INFLIGHT.exception = Some(mem::transmute::<Exception, Exception<'static>>(*exception));
     INFLIGHT.handled   = false;
 
@@ -219,9 +225,8 @@ pub unsafe extern fn raise(exception: *const Exception) -> ! {
 pub unsafe extern fn reraise() -> ! {
     use cslice::AsCSlice;
 
-    trace!("Re-raise");
+    // Reraise is basically cxa_rethrow, which calls _Unwind_Resume_or_Rethrow,
-    // current implementation uses raise as _Unwind_Resume is not working now
+    // which for EHABI would always call _Unwind_RaiseException.
-    // would debug that later.
     match INFLIGHT.exception {
         Some(ref exception) => raise(exception),
         None => raise(&Exception {

src/runtime/src/irq.rs

@@ -0,0 +1,47 @@
+use libboard_zynq::{gic, mpcore, println, stdio};
+use libcortex_a9::{
+    asm,
+    regs::{MPIDR, SP},
+    spin_lock_yield, notify_spin_lock
+};
+use libregister::{RegisterR, RegisterW};
+use core::sync::atomic::{AtomicBool, Ordering};
+
+extern "C" {
+    static mut __stack1_start: u32;
+    fn main_core1() -> !;
+}
+
+static CORE1_RESTART: AtomicBool = AtomicBool::new(false);
+
+#[link_section = ".text.boot"]
+#[no_mangle]
+#[naked]
+pub unsafe extern "C" fn IRQ() {
+    if MPIDR.read().cpu_id() == 1 {
+        let mpcore = mpcore::RegisterBlock::new();
+        let mut gic = gic::InterruptController::new(mpcore);
+        let id = gic.get_interrupt_id();
+        if id.0 == 0 {
+            gic.end_interrupt(id);
+            asm::exit_irq();
+            SP.write(&mut __stack1_start as *mut _ as u32);
+            asm::enable_irq();
+            CORE1_RESTART.store(false, Ordering::Relaxed);
+            notify_spin_lock();
+            main_core1();
+        }
+    }
+    stdio::drop_uart();
+    println!("IRQ");
+    loop {}
+}
+
+pub fn restart_core1() {
+    let mut interrupt_controller = gic::InterruptController::new(mpcore::RegisterBlock::new());
+    CORE1_RESTART.store(true, Ordering::Relaxed);
+    interrupt_controller.send_sgi(gic::InterruptId(0), gic::CPUCore::Core1.into());
+    while CORE1_RESTART.load(Ordering::Relaxed) {
+        spin_lock_yield();
+    }
+}

src/runtime/src/kernel/api.rs

@@ -1,6 +1,44 @@
+use core::ffi::VaList;
+use core::ptr;
+use core::str;
+use libc::{c_char, c_int, size_t};
+use libm;
+use log::{info, warn};
+
+use alloc::vec;
+
 use crate::eh_artiq;
 use crate::rtio;
 use super::rpc::{rpc_send, rpc_send_async, rpc_recv};
+use super::dma;
+use super::cache;
+
+
+extern "C" {
+    fn vsnprintf_(buffer: *mut c_char, count: size_t, format: *const c_char, va: VaList) -> c_int;
+}
+
+unsafe extern fn core_log(fmt: *const c_char, mut args: ...) {
+    let size = vsnprintf_(ptr::null_mut(), 0, fmt, args.as_va_list()) as usize;
+    let mut buf = vec![0; size + 1];
+    vsnprintf_(buf.as_mut_ptr() as *mut i8, size + 1, fmt, args.as_va_list());
+    let buf: &[u8] = &buf.as_slice()[..size-1]; // strip \n and NUL
+    match str::from_utf8(buf) {
+        Ok(s) => info!("kernel: {}", s),
+        Err(e) => {
+            info!("kernel: {}", (str::from_utf8(&buf[..e.valid_up_to()]).unwrap()));
+            warn!("kernel: invalid utf-8");
+        }
+    }
+}
+
+unsafe extern fn rtio_log(fmt: *const c_char, mut args: ...) {
+    let size = vsnprintf_(ptr::null_mut(), 0, fmt, args.as_va_list()) as usize;
+    let mut buf = vec![0; size + 1];
+    vsnprintf_(buf.as_mut_ptr(), size + 1, fmt, args.as_va_list());
+    rtio::write_log(buf.as_slice());
+}
+
 
 macro_rules! api {
     ($i:ident) => ({
@@ -16,6 +54,15 @@ macro_rules! api {
     }
 }
 
+macro_rules! api_libm_f64f64 {
+    ($i:ident) => ({
+        extern fn $i(x: f64) -> f64 {
+            libm::$i(x)
+        }
+        api!($i = $i)
+    })
+}
+
 pub fn resolve(required: &[u8]) -> Option<u32> {
     let api = &[
         // timing
@@ -38,6 +85,21 @@ pub fn resolve(required: &[u8]) -> Option<u32> {
         api!(rtio_input_data = rtio::input_data),
         api!(rtio_input_timestamped_data = rtio::input_timestamped_data),
 
+        // log
+        api!(core_log = core_log),
+        api!(rtio_log = rtio_log),
+
+        // rtio dma
+        api!(dma_record_start = dma::dma_record_start),
+        api!(dma_record_stop = dma::dma_record_stop),
+        api!(dma_erase = dma::dma_erase),
+        api!(dma_retrieve = dma::dma_retrieve),
+        api!(dma_playback = dma::dma_playback),
+
+        // cache
+        api!(cache_get = cache::get),
+        api!(cache_put = cache::put),
+
         // Double-precision floating-point arithmetic helper functions
         // RTABI chapter 4.1.2, Table 2
         api!(__aeabi_dadd),
@@ -101,6 +163,7 @@ pub fn resolve(required: &[u8]) -> Option<u32> {
         // RTABI chapter 4.3.1
         api!(__aeabi_idiv),
         api!(__aeabi_ldivmod),
+        api!(__aeabi_idivmod),
         api!(__aeabi_uidiv),
         api!(__aeabi_uldivmod),
         // 4.3.4 Memory copying, clearing, and setting
@@ -116,14 +179,92 @@ pub fn resolve(required: &[u8]) -> Option<u32> {
         api!(__aeabi_memclr8),
         api!(__aeabi_memclr4),
         api!(__aeabi_memclr),
+
         // libc
         api!(memcmp, extern { fn memcmp(a: *const u8, b: *mut u8, size: usize); }),
+
         // exceptions
         api!(_Unwind_Resume = unwind::_Unwind_Resume),
         api!(__artiq_personality = eh_artiq::artiq_personality),
         api!(__artiq_raise = eh_artiq::raise),
         api!(__artiq_reraise = eh_artiq::reraise),
 
+        // libm
+        api_libm_f64f64!(acos),
+        api_libm_f64f64!(acosh),
+        api_libm_f64f64!(asin),
+        api_libm_f64f64!(asinh),
+        api_libm_f64f64!(atan),
+        {
+            extern fn atan2(y: f64, x: f64) -> f64 {
+                libm::atan2(y, x)
+            }
+            api!(atan2 = atan2)
+        },
+        api_libm_f64f64!(cbrt),
+        api_libm_f64f64!(ceil),
+        api_libm_f64f64!(cos),
+        api_libm_f64f64!(cosh),
+        api_libm_f64f64!(erf),
+        api_libm_f64f64!(erfc),
+        api_libm_f64f64!(exp),
+        api_libm_f64f64!(exp2),
+        api_libm_f64f64!(exp10),
+        api_libm_f64f64!(expm1),
+        api_libm_f64f64!(fabs),
+        api_libm_f64f64!(floor),
+        {
+            extern fn fma(x: f64, y: f64, z: f64) -> f64 {
+                libm::fma(x, y, z)
+            }
+            api!(fma = fma)
+        },
+        {
+            extern fn fmod(x: f64, y: f64) -> f64 {
+                libm::fmod(x, y)
+            }
+            api!(fmod = fmod)
+        },
+        {
+            extern fn hypot(x: f64, y: f64) -> f64 {
+                libm::hypot(x, y)
+            }
+            api!(hypot = hypot)
+        },
+        api_libm_f64f64!(j0),
+        api_libm_f64f64!(j1),
+        {
+            extern fn jn(n: i32, x: f64) -> f64 {
+                libm::jn(n, x)
+            }
+            api!(jn = jn)
+        },
+        api_libm_f64f64!(lgamma),
+        api_libm_f64f64!(log),
+        api_libm_f64f64!(log2),
+        api_libm_f64f64!(log10),
+        {
+            extern fn pow(x: f64, y: f64) -> f64 {
+                libm::pow(x, y)
+            }
+            api!(pow = pow)
+        },
+        api_libm_f64f64!(round),
+        api_libm_f64f64!(sin),
+        api_libm_f64f64!(sinh),
+        api_libm_f64f64!(sqrt),
+        api_libm_f64f64!(tan),
+        api_libm_f64f64!(tanh),
+        api_libm_f64f64!(tgamma),
+        api_libm_f64f64!(trunc),
+        api_libm_f64f64!(y0),
+        api_libm_f64f64!(y1),
+        {
+            extern fn yn(n: i32, x: f64) -> f64 {
+                libm::yn(n, x)
+            }
+            api!(yn = yn)
+        },
     ];
     api.iter()
        .find(|&&(exported, _)| exported.as_bytes() == required)

src/runtime/src/kernel/cache.rs

@@ -0,0 +1,26 @@
+use alloc::string::String;
+use cslice::{CSlice, AsCSlice};
+use core::mem::{transmute, forget};
+use super::{KERNEL_CHANNEL_0TO1, KERNEL_CHANNEL_1TO0, Message};
+
+pub extern fn get(key: CSlice<u8>) -> CSlice<'static, i32> {
+    let key = String::from_utf8(key.as_ref().to_vec()).unwrap();
+    KERNEL_CHANNEL_1TO0.lock().as_mut().unwrap().send(Message::CacheGetRequest(key));
+    let msg = KERNEL_CHANNEL_0TO1.lock().as_mut().unwrap().recv();
+    if let Message::CacheGetReply(v) = msg {
+        let slice = unsafe { transmute(v.as_c_slice()) };
+        // we intentionally leak the memory here,
+        // which does not matter as core1 would restart
+        forget(v);
+        slice
+    } else {
+        panic!("Expected CacheGetReply for CacheGetRequest");
+    }
+}
+
+pub extern fn put(key: CSlice<u8>, list: CSlice<i32>) {
+    let key = String::from_utf8(key.as_ref().to_vec()).unwrap();
+    let value = list.as_ref().to_vec();
+    KERNEL_CHANNEL_1TO0.lock().as_mut().unwrap().send(Message::CachePutRequest(key, value));
+}
+

src/runtime/src/kernel/control.rs

@@ -1,41 +1,55 @@
-use libcortex_a9::sync_channel::{self, sync_channel};
+use libcortex_a9::sync_channel::{Sender, Receiver};
 use libsupport_zynq::boot::Core1;
 
-use super::{CHANNEL_0TO1, CHANNEL_1TO0, Message};
+use super::{CHANNEL_0TO1, CHANNEL_1TO0, INIT_LOCK, Message};
+use crate::irq::restart_core1;
+
+use core::mem::{forget, replace};
 
 pub struct Control {
-    core1: Core1,
+    pub tx: Sender<'static, Message>,
-    pub tx: sync_channel::Sender<Message>,
+    pub rx: Receiver<'static, Message>,
-    pub rx: sync_channel::Receiver<Message>,
+}
+
+fn get_channels() -> (Sender<'static, Message>, Receiver<'static, Message>) {
+    let mut core0_tx = None;
+    while core0_tx.is_none() {
+        core0_tx = CHANNEL_0TO1.lock().take();
+    }
+    let core0_tx = core0_tx.unwrap();
+
+    let mut core0_rx = None;
+    while core0_rx.is_none() {
+        core0_rx = CHANNEL_1TO0.lock().take();
+    }
+    let core0_rx = core0_rx.unwrap();
+
+    (core0_tx, core0_rx)
 }
 
 impl Control {
     pub fn start() -> Self {
-        let core1 = Core1::start(true);
+        Core1::start(true);
-
+        let (core0_tx, core0_rx) = get_channels();
-        let (core0_tx, core1_rx) = sync_channel(4);
-        let (core1_tx, core0_rx) = sync_channel(4);
-        *CHANNEL_0TO1.lock() = Some(core1_rx);
-        *CHANNEL_1TO0.lock() = Some(core1_tx);
 
         Control {
-            core1,
             tx: core0_tx,
             rx: core0_rx,
         }
     }
 
     pub fn restart(&mut self) {
-        *CHANNEL_0TO1.lock() = None;
+        {
-        *CHANNEL_1TO0.lock() = None;
+            INIT_LOCK.lock();
-
+            restart_core1();
-        self.core1.restart();
+            unsafe {
-
+                self.tx.drop_elements();
-        let (core0_tx, core1_rx) = sync_channel(4);
-        let (core1_tx, core0_rx) = sync_channel(4);
-        *CHANNEL_0TO1.lock() = Some(core1_rx);
-        *CHANNEL_1TO0.lock() = Some(core1_tx);
-        self.tx = core0_tx;
-        self.rx = core0_rx;
             }
         }
+        let (core0_tx, core0_rx) = get_channels();
+        // dangling pointer here, so we forget it
+        forget(replace(&mut self.tx, core0_tx));
+        forget(replace(&mut self.rx, core0_rx));
+    }
+}
+

src/runtime/src/kernel/core1.rs

@@ -1,23 +1,42 @@
 //! Kernel prologue/epilogue that runs on the 2nd CPU core
 
-use core::{mem, ptr};
+use core::{mem, ptr, cell::UnsafeCell};
 use alloc::borrow::ToOwned;
 use log::{debug, info, error};
 use cslice::CSlice;
 
-use libcortex_a9::{enable_fpu, cache::dcci_slice, sync_channel};
+use libcortex_a9::{
+    enable_fpu,
+    cache::{dcci_slice, iciallu, bpiall},
+    asm::{dsb, isb},
+    sync_channel,
+};
+use libboard_zynq::{mpcore, gic};
+use libsupport_zynq::ram;
 use dyld::{self, Library};
 use crate::eh_artiq;
 use super::{
     api::resolve,
     rpc::rpc_send_async,
+    INIT_LOCK,
     CHANNEL_0TO1, CHANNEL_1TO0,
     KERNEL_CHANNEL_0TO1, KERNEL_CHANNEL_1TO0,
+    KERNEL_IMAGE,
     Message,
+    dma,
 };
 
-/// will contain the kernel image address on the heap
+// linker symbols
-static mut KERNEL_LOAD_ADDR: usize = 0;
+extern "C" {
+    #[no_mangle]
+    static __text_start: u32;
+    #[no_mangle]
+    static __text_end: u32;
+    #[no_mangle]
+    static __exidx_start: u32;
+    #[no_mangle]
+    static __exidx_end: u32;
+}
 
 unsafe fn attribute_writeback(typeinfo: *const ()) {
     struct Attr {
@@ -58,8 +77,8 @@ unsafe fn attribute_writeback(typeinfo: *const ()) {
     }
 }
 
-struct KernelImage {
+pub struct KernelImage {
-    library: Library,
+    library: UnsafeCell<Library>,
     __modinit__: u32,
     typeinfo: Option<u32>,
 }
@@ -82,16 +101,25 @@ impl KernelImage {
         }
 
         Ok(KernelImage {
-            library,
+            library: UnsafeCell::new(library),
             __modinit__,
             typeinfo,
         })
     }
 
-    pub unsafe fn exec(&mut self) {
+    pub unsafe fn rebind(&self, name: &[u8], addr: *const ()) -> Result<(), dyld::Error> {
+        let library = self.library.get().as_mut().unwrap();
+        library.rebind(name, addr)
+    }
+
+    pub unsafe fn exec(&self) {
         // Flush data cache entries for the image in DDR, including
-        // Memory/Instruction Symchronization Barriers
+        // Memory/Instruction Synchronization Barriers
-        dcci_slice(self.library.image.data);
+        dcci_slice(self.library.get().as_ref().unwrap().image.data);
+        iciallu();
+        bpiall();
+        dsb();
+        isb();
 
         (mem::transmute::<u32, fn()>(self.__modinit__))();
 
@@ -99,6 +127,12 @@ impl KernelImage {
             attribute_writeback(typeinfo as *const ());
         }
     }
+
+    pub fn get_load_addr(&self) -> usize {
+        unsafe {
+            self.library.get().as_ref().unwrap().image.as_ptr() as usize
+        }
+    }
 }
 
 #[no_mangle]
@@ -108,31 +142,30 @@ pub fn main_core1() {
     enable_fpu();
     debug!("FPU enabled on Core1");
 
-    let mut core1_tx = None;
+    ram::init_alloc_core1();
-    while core1_tx.is_none() {
+    gic::InterruptController::new(mpcore::RegisterBlock::new()).enable_interrupts();
-        core1_tx = CHANNEL_1TO0.lock().take();
-    }
-    let mut core1_tx = core1_tx.unwrap();
 
-    let mut core1_rx = None;
+    let (mut core0_tx, mut core1_rx) = sync_channel!(Message, 4);
-    while core1_rx.is_none() {
+    let (mut core1_tx, core0_rx) = sync_channel!(Message, 4);
-        core1_rx = CHANNEL_0TO1.lock().take();
+    unsafe {
+        INIT_LOCK.lock();
+        core0_tx.reset();
+        core1_tx.reset();
+        dma::init_dma_recorder();
     }
-    let mut core1_rx = core1_rx.unwrap();
+    *CHANNEL_0TO1.lock() = Some(core0_tx);
+    *CHANNEL_1TO0.lock() = Some(core0_rx);
 
     // set on load, cleared on start
     let mut loaded_kernel = None;
     loop {
         let message = core1_rx.recv();
-        match *message {
+        match message {
             Message::LoadRequest(data) => {
                 let result = dyld::load(&data, &resolve)
                     .and_then(KernelImage::new);
                 match result {
                     Ok(kernel) => {
-                        unsafe {
-                            KERNEL_LOAD_ADDR = kernel.library.image.as_ptr() as usize;
-                        }
                         loaded_kernel = Some(kernel);
                         debug!("kernel loaded");
                         core1_tx.send(Message::LoadCompleted);
@@ -145,14 +178,16 @@ pub fn main_core1() {
             },
             Message::StartRequest => {
                 info!("kernel starting");
-                if let Some(mut kernel) = loaded_kernel.take() {
+                if let Some(kernel) = loaded_kernel.take() {
+                    *KERNEL_CHANNEL_0TO1.lock() = Some(core1_rx);
+                    *KERNEL_CHANNEL_1TO0.lock() = Some(core1_tx);
                     unsafe {
-                        KERNEL_CHANNEL_0TO1 = mem::transmute(&mut core1_rx);
+                        KERNEL_IMAGE = &kernel as *const KernelImage;
-                        KERNEL_CHANNEL_1TO0 = mem::transmute(&mut core1_tx);
                         kernel.exec();
-                        KERNEL_CHANNEL_0TO1 = ptr::null_mut();
+                        KERNEL_IMAGE = ptr::null();
-                        KERNEL_CHANNEL_1TO0 = ptr::null_mut();
                     }
+                    core1_rx = core::mem::replace(&mut *KERNEL_CHANNEL_0TO1.lock(), None).unwrap();
+                    core1_tx = core::mem::replace(&mut *KERNEL_CHANNEL_1TO0.lock(), None).unwrap();
                 }
                 info!("kernel finished");
                 core1_tx.send(Message::KernelFinished);
@@ -165,7 +200,7 @@ pub fn main_core1() {
 /// Called by eh_artiq
 pub fn terminate(exception: &'static eh_artiq::Exception<'static>, backtrace: &'static mut [usize]) -> ! {
     let load_addr = unsafe {
-        KERNEL_LOAD_ADDR
+        KERNEL_IMAGE.as_ref().unwrap().get_load_addr()
     };
     let mut cursor = 0;
     // The address in the backtrace is relocated, so we have to convert it back to the address in
@@ -177,7 +212,33 @@ pub fn terminate(exception: &'static eh_artiq::Exception<'static>, backtrace: &'
         }
     }
 
-    let core1_tx: &mut sync_channel::Sender<Message> = unsafe { mem::transmute(KERNEL_CHANNEL_1TO0) };
+    {
-    core1_tx.send(Message::KernelException(exception, &backtrace[..cursor]));
+        let mut core1_tx = KERNEL_CHANNEL_1TO0.lock();
+        core1_tx.as_mut().unwrap().send(Message::KernelException(exception, &backtrace[..cursor]));
+    }
     loop {}
 }
+
+/// Called by llvm_libunwind
+#[no_mangle]
+extern fn dl_unwind_find_exidx(pc: *const u32, len_ptr: *mut u32) -> *const u32 {
+    let exidx = unsafe {
+        KERNEL_IMAGE.as_ref()
+            .expect("dl_unwind_find_exidx kernel image")
+            .library.get().as_ref().unwrap().exidx()
+    };
+
+    let length;
+    let start: *const u32;
+    unsafe {
+        if &__text_start as *const u32 <= pc && pc < &__text_end as *const u32 {
+            length = (&__exidx_end as *const u32).offset_from(&__exidx_start) as u32;
+            start = &__exidx_start;
+        } else {
+            length = exidx.len() as u32;
+            start = exidx.as_ptr();
+        }
+        *len_ptr = length;
+    }
+    start
+}

src/runtime/src/kernel/dma.rs

@@ -0,0 +1,208 @@
+use crate::{
+    pl::csr,
+    artiq_raise,
+    rtio,
+};
+use alloc::{vec::Vec, string::String, boxed::Box};
+use cslice::CSlice;
+use super::{KERNEL_IMAGE, KERNEL_CHANNEL_0TO1, KERNEL_CHANNEL_1TO0, Message};
+use core::mem;
+use log::debug;
+
+use libcortex_a9::cache::dcci_slice;
+
+const ALIGNMENT: usize = 16 * 8;
+
+#[repr(C)]
+pub struct DmaTrace {
+    duration: i64,
+    address:  i32,
+}
+
+#[derive(Clone, Debug)]
+pub struct DmaRecorder {
+    pub name: String,
+    pub buffer: Vec<u8>,
+    pub duration: i64,
+}
+
+static mut RECORDER: Option<DmaRecorder> = None;
+
+pub unsafe fn init_dma_recorder() {
+    // as static would remain after restart, we have to reset it,
+    // without running its destructor.
+    mem::forget(mem::replace(&mut RECORDER, None));
+}
+
+pub extern fn dma_record_start(name: CSlice<u8>) {
+    let name = String::from_utf8(name.as_ref().to_vec()).unwrap();
+    KERNEL_CHANNEL_1TO0.lock().as_mut().unwrap().send(Message::DmaEraseRequest(name.clone()));
+    unsafe {
+        if RECORDER.is_some() {
+            artiq_raise!("DMAError", "DMA is already recording")
+        }
+
+        let library = KERNEL_IMAGE.as_ref().unwrap();
+        library.rebind(b"rtio_output",
+                       dma_record_output as *const ()).unwrap();
+        library.rebind(b"rtio_output_wide",
+                       dma_record_output_wide as *const ()).unwrap();
+
+        RECORDER = Some(DmaRecorder {
+            name,
+            buffer: Vec::new(),
+            duration: 0,
+        });
+    }
+}
+
+pub extern fn dma_record_stop(duration: i64) {
+    unsafe {
+        if RECORDER.is_none() {
+            artiq_raise!("DMAError", "DMA is not recording")
+        }
+
+        let library = KERNEL_IMAGE.as_ref().unwrap();
+        library.rebind(b"rtio_output",
+                       rtio::output as *const ()).unwrap();
+        library.rebind(b"rtio_output_wide",
+                       rtio::output_wide as *const ()).unwrap();
+
+        let mut recorder = RECORDER.take().unwrap();
+        recorder.duration = duration;
+        KERNEL_CHANNEL_1TO0.lock().as_mut().unwrap().send(
+            Message::DmaPutRequest(recorder)
+        );
+    }
+}
+
+#[inline(always)]
+unsafe fn dma_record_output_prepare(timestamp: i64, target: i32,
+                                    words: usize) {
+    // See gateware/rtio/dma.py.
+    const HEADER_LENGTH: usize = /*length*/1 + /*channel*/3 + /*timestamp*/8 + /*address*/1;
+    let length = HEADER_LENGTH + /*data*/words * 4;
+
+    let buffer = &mut RECORDER.as_mut().unwrap().buffer;
+    buffer.reserve(length);
+    buffer.extend_from_slice(&[
+        (length    >>  0) as u8,
+        (target    >>  8) as u8,
+        (target    >>  16) as u8,
+        (target    >>  24) as u8,
+        (timestamp >>  0) as u8,
+        (timestamp >>  8) as u8,
+        (timestamp >> 16) as u8,
+        (timestamp >> 24) as u8,
+        (timestamp >> 32) as u8,
+        (timestamp >> 40) as u8,
+        (timestamp >> 48) as u8,
+        (timestamp >> 56) as u8,
+        (target    >>  0) as u8,
+    ]);
+}
+
+pub extern fn dma_record_output(target: i32, word: i32) {
+    unsafe {
+        let timestamp = rtio::now_mu();
+        dma_record_output_prepare(timestamp, target, 1);
+        RECORDER.as_mut().unwrap().buffer.extend_from_slice(&[
+            (word >>  0) as u8,
+            (word >>  8) as u8,
+            (word >> 16) as u8,
+            (word >> 24) as u8,
+        ]);
+    }
+}
+
+pub extern fn dma_record_output_wide(target: i32, words: CSlice<i32>) {
+    assert!(words.len() <= 16); // enforce the hardware limit
+
+    unsafe {
+        let timestamp = rtio::now_mu();
+        dma_record_output_prepare(timestamp, target, words.len());
+        let buffer = &mut RECORDER.as_mut().unwrap().buffer;
+        for word in words.as_ref().iter() {
+            buffer.extend_from_slice(&[
+                (word >>  0) as u8,
+                (word >>  8) as u8,
+                (word >> 16) as u8,
+                (word >> 24) as u8,
+            ]);
+        }
+    }
+}
+
+pub extern fn dma_erase(name: CSlice<u8>) {
+    let name = String::from_utf8(name.as_ref().to_vec()).unwrap();
+    KERNEL_CHANNEL_1TO0.lock().as_mut().unwrap().send(Message::DmaEraseRequest(name));
+}
+
+pub extern fn dma_retrieve(name: CSlice<u8>) -> DmaTrace {
+    let name = String::from_utf8(name.as_ref().to_vec()).unwrap();
+    KERNEL_CHANNEL_1TO0.lock().as_mut().unwrap().send(Message::DmaGetRequest(name));
+    match KERNEL_CHANNEL_0TO1.lock().as_mut().unwrap().recv() {
+        Message::DmaGetReply(None) => (),
+        Message::DmaGetReply(Some((mut v, duration))) => {
+            v.reserve(ALIGNMENT - 1);
+            let original_length = v.len();
+            let padding = ALIGNMENT - v.as_ptr() as usize % ALIGNMENT;
+            let padding = if padding == ALIGNMENT { 0 } else { padding };
+            for _ in 0..padding {
+                v.push(0);
+            }
+            // trailing zero to indicate end of buffer
+            v.push(0);
+            v.copy_within(0..original_length, padding);
+            let v = Box::new(v);
+            let address = Box::into_raw(v) as *mut Vec<u8> as i32;
+            return DmaTrace {
+                address,
+                duration,
+            };
+        },
+        _ => panic!("Expected DmaGetReply after DmaGetRequest!"),
+    }
+    // we have to defer raising error as we have to drop the message first...
+    artiq_raise!("DMAError", "DMA trace not found");
+}
+
+pub extern fn dma_playback(timestamp: i64, ptr: i32) {
+    debug!("DMA playback started");
+    unsafe {
+        let v = Box::from_raw(ptr as *mut Vec<u8>);
+        let padding = ALIGNMENT - v.as_ptr() as usize % ALIGNMENT;
+        let padding = if padding == ALIGNMENT { 0 } else { padding };
+        dcci_slice(&v[padding..]);
+        let ptr = v.as_ptr().add(padding) as i32;
+
+        csr::rtio_dma::base_address_write(ptr as u32);
+        csr::rtio_dma::time_offset_write(timestamp as u64);
+
+        csr::cri_con::selected_write(1);
+        csr::rtio_dma::enable_write(1);
+        while csr::rtio_dma::enable_read() != 0 {}
+        csr::cri_con::selected_write(0);
+
+        mem::forget(v);
+        debug!("DMA playback finished");
+
+        let error = csr::rtio_dma::error_read();
+        if error != 0 {
+            let timestamp = csr::rtio_dma::error_timestamp_read();
+            let channel = csr::rtio_dma::error_channel_read();
+            csr::rtio_dma::error_write(1);
+            if error & 1 != 0 {
+                artiq_raise!("RTIOUnderflow",
+                    "RTIO underflow at {0} mu, channel {1}",
+                    timestamp as i64, channel as i64, 0);
+            }
+            if error & 2 != 0 {
+                artiq_raise!("RTIODestinationUnreachable",
+                    "RTIO destination unreachable, output, at {0} mu, channel {1}",
+                    timestamp as i64, channel as i64, 0);
+            }
+        }
+    }
+}
+

src/runtime/src/kernel/mod.rs

@@ -1,5 +1,5 @@
 use core::ptr;
-use alloc::{vec::Vec, sync::Arc, string::String};
+use alloc::{vec::Vec, string::String};
 
 use libcortex_a9::{mutex::Mutex, sync_channel};
 use crate::eh_artiq;
@@ -9,8 +9,11 @@ pub use control::Control;
 pub mod core1;
 mod api;
 mod rpc;
+mod dma;
+pub use dma::DmaRecorder;
+mod cache;
 
-#[derive(Debug)]
+#[derive(Debug, Clone)]
 pub struct RPCException {
     pub name: String,
     pub message: String,
@@ -21,21 +24,35 @@ pub struct RPCException {
     pub function: String
 }
 
-#[derive(Debug)]
+#[derive(Debug, Clone)]
 pub enum Message {
-    LoadRequest(Arc<Vec<u8>>),
+    LoadRequest(Vec<u8>),
     LoadCompleted,
     LoadFailed,
     StartRequest,
     KernelFinished,
     KernelException(&'static eh_artiq::Exception<'static>, &'static [usize]),
-    RpcSend { is_async: bool, data: Arc<Vec<u8>> },
+    RpcSend { is_async: bool, data: Vec<u8> },
     RpcRecvRequest(*mut ()),
     RpcRecvReply(Result<usize, RPCException>),
+
+    CacheGetRequest(String),
+    CacheGetReply(Vec<i32>),
+    CachePutRequest(String, Vec<i32>),
+
+    DmaPutRequest(DmaRecorder),
+    DmaEraseRequest(String),
+    DmaGetRequest(String),
+    DmaGetReply(Option<(Vec<u8>, i64)>),
 }
 
-static CHANNEL_0TO1: Mutex<Option<sync_channel::Receiver<Message>>> = Mutex::new(None);
+static CHANNEL_0TO1: Mutex<Option<sync_channel::Sender<'static, Message>>> = Mutex::new(None);
-static CHANNEL_1TO0: Mutex<Option<sync_channel::Sender<Message>>> = Mutex::new(None);
+static CHANNEL_1TO0: Mutex<Option<sync_channel::Receiver<'static, Message>>> = Mutex::new(None);
+
+static KERNEL_CHANNEL_0TO1: Mutex<Option<sync_channel::Receiver<'static, Message>>> = Mutex::new(None);
+static KERNEL_CHANNEL_1TO0: Mutex<Option<sync_channel::Sender<'static, Message>>> = Mutex::new(None);
+
+static mut KERNEL_IMAGE: *const core1::KernelImage = ptr::null();
+
+static INIT_LOCK: Mutex<()> = Mutex::new(());
 
-static mut KERNEL_CHANNEL_0TO1: *mut () = ptr::null_mut();
-static mut KERNEL_CHANNEL_1TO0: *mut () = ptr::null_mut();

src/runtime/src/kernel/rpc.rs

@@ -1,10 +1,8 @@
 //! Kernel-side RPC API
 
-use core::mem;
+use alloc::vec::Vec;
-use alloc::{vec::Vec, sync::Arc};
 use cslice::{CSlice, AsCSlice};
 
-use libcortex_a9::sync_channel;
 use crate::eh_artiq;
 use crate::rpc::send_args;
 use super::{
@@ -13,10 +11,10 @@ use super::{
 };
 
 fn rpc_send_common(is_async: bool, service: u32, tag: &CSlice<u8>, data: *const *const ()) {
-    let core1_tx: &mut sync_channel::Sender<Message> = unsafe { mem::transmute(KERNEL_CHANNEL_1TO0) };
+    let mut core1_tx = KERNEL_CHANNEL_1TO0.lock();
     let mut buffer = Vec::<u8>::new();
     send_args(&mut buffer, service, tag.as_ref(), data).expect("RPC encoding failed");
-    core1_tx.send(Message::RpcSend { is_async: is_async, data: Arc::new(buffer) });
+    core1_tx.as_mut().unwrap().send(Message::RpcSend { is_async, data: buffer });
 }
 
 pub extern fn rpc_send(service: u32, tag: &CSlice<u8>, data: *const *const ()) {
@@ -28,11 +26,13 @@ pub extern fn rpc_send_async(service: u32, tag: &CSlice<u8>, data: *const *const
 }
 
 pub extern fn rpc_recv(slot: *mut ()) -> usize {
-    let core1_rx: &mut sync_channel::Receiver<Message> = unsafe { mem::transmute(KERNEL_CHANNEL_0TO1) };
+    let reply = {
-    let core1_tx: &mut sync_channel::Sender<Message> = unsafe { mem::transmute(KERNEL_CHANNEL_1TO0) };
+        let mut core1_rx = KERNEL_CHANNEL_0TO1.lock();
-    core1_tx.send(Message::RpcRecvRequest(slot));
+        let mut core1_tx = KERNEL_CHANNEL_1TO0.lock();
-    let reply = core1_rx.recv();
+        core1_tx.as_mut().unwrap().send(Message::RpcRecvRequest(slot));
-    match *reply {
+        core1_rx.as_mut().unwrap().recv()
+    };
+    match reply {
         Message::RpcRecvReply(Ok(alloc_size)) => alloc_size,
         Message::RpcRecvReply(Err(exception)) => unsafe {
             eh_artiq::raise(&eh_artiq::Exception {

src/runtime/src/logger.rs

@@ -0,0 +1,123 @@
+use core::cell::Cell;
+use core::fmt::Write;
+use log::{Log, LevelFilter};
+use log_buffer::LogBuffer;
+use libcortex_a9::mutex::{Mutex, MutexGuard};
+use libboard_zynq::{println, timer::GlobalTimer};
+
+pub struct LogBufferRef<'a> {
+    buffer:        MutexGuard<'a, LogBuffer<&'static mut [u8]>>,
+    old_log_level: LevelFilter
+}
+
+impl<'a> LogBufferRef<'a> {
+    fn new(buffer: MutexGuard<'a, LogBuffer<&'static mut [u8]>>) -> LogBufferRef<'a> {
+        let old_log_level = log::max_level();
+        log::set_max_level(LevelFilter::Off);
+        LogBufferRef { buffer, old_log_level }
+    }
+
+    pub fn is_empty(&self) -> bool {
+        self.buffer.is_empty()
+    }
+
+    pub fn clear(&mut self) {
+        self.buffer.clear()
+    }
+
+    pub fn extract(&mut self) -> &str {
+        self.buffer.extract()
+    }
+}
+
+impl<'a> Drop for LogBufferRef<'a> {
+    fn drop(&mut self) {
+        log::set_max_level(self.old_log_level)
+    }
+}
+
+pub struct BufferLogger {
+    buffer:      Mutex<LogBuffer<&'static mut [u8]>>,
+    uart_filter: Cell<LevelFilter>,
+    buffer_filter: Cell<LevelFilter>,
+}
+
+static mut LOGGER: Option<BufferLogger> = None;
+
+impl BufferLogger {
+    pub fn new(buffer: &'static mut [u8]) -> BufferLogger {
+        BufferLogger {
+            buffer: Mutex::new(LogBuffer::new(buffer)),
+            uart_filter: Cell::new(LevelFilter::Info),
+            buffer_filter: Cell::new(LevelFilter::Trace),
+        }
+    }
+
+    pub fn register(self) {
+        unsafe {
+            LOGGER = Some(self);
+            log::set_logger(LOGGER.as_ref().unwrap())
+                .expect("global logger can only be initialized once");
+        }
+    }
+
+    pub unsafe fn get_logger() -> &'static mut Option<BufferLogger> {
+        &mut LOGGER
+    }
+
+    pub fn buffer<'a>(&'a self) -> Option<LogBufferRef<'a>> {
+        self.buffer
+            .try_lock()
+            .map(LogBufferRef::new)
+    }
+
+    pub fn uart_log_level(&self) -> LevelFilter {
+        self.uart_filter.get()
+    }
+
+    pub fn set_uart_log_level(&self, max_level: LevelFilter) {
+        self.uart_filter.set(max_level)
+    }
+
+    pub fn buffer_log_level(&self) -> LevelFilter {
+        self.buffer_filter.get()
+    }
+
+    /// this should be reserved for mgmt module
+    pub fn set_buffer_log_level(&self, max_level: LevelFilter) {
+        self.buffer_filter.set(max_level)
+    }
+}
+
+// required for impl Log
+unsafe impl Sync for BufferLogger {}
+
+impl Log for BufferLogger {
+    fn enabled(&self, _metadata: &log::Metadata) -> bool {
+        true
+    }
+
+    fn log(&self, record: &log::Record) {
+        if self.enabled(record.metadata()) {
+            let timestamp = unsafe {
+                GlobalTimer::get()
+            }.get_us().0;
+            let seconds   = timestamp / 1_000_000;
+            let micros    = timestamp % 1_000_000;
+
+            if record.level() <= self.buffer_log_level() {
+                let mut buffer = self.buffer.lock();
+                writeln!(buffer, "[{:6}.{:06}s] {:>5}({}): {}", seconds, micros,
+                         record.level(), record.target(), record.args()).unwrap();
+            }
+
+            if record.level() <= self.uart_log_level() {
+                println!("[{:6}.{:06}s] {:>5}({}): {}", seconds, micros,
+                         record.level(), record.target(), record.args());
+            }
+        }
+    }
+
+    fn flush(&self) {
+    }
+}

src/runtime/src/main.rs

@@ -3,17 +3,24 @@
 #![recursion_limit="1024"]  // for futures_util::select!
 #![feature(alloc_error_handler)]
 #![feature(panic_info_message)]
+#![feature(c_variadic)]
+#![feature(const_btree_new)]
+#![feature(ptr_offset_from)]
+#![feature(const_in_array_repeat_expressions)]
+#![feature(naked_functions)]
 
 extern crate alloc;
 
 use core::{cmp, str};
-use log::{info, warn};
+use log::{info, warn, error};
 
-use libboard_zynq::{timer::GlobalTimer, time::Milliseconds, logger, devc, slcr};
+use libboard_zynq::{timer::GlobalTimer, devc, slcr, mpcore, gic};
+use libasync::{task, block_async};
 use libsupport_zynq::ram;
 use libregister::RegisterW;
-use nb::block;
+use nb;
-use embedded_hal::timer::CountDown;
+use void::Void;
+use embedded_hal::blocking::delay::DelayMs;
 
 mod sd_reader;
 mod config;
@@ -24,12 +31,21 @@ mod comms;
 mod rpc;
 #[path = "../../../build/pl.rs"]
 mod pl;
+#[cfg(ki_impl = "csr")]
+#[path = "rtio_csr.rs"]
+mod rtio;
+#[cfg(ki_impl = "acp")]
+#[path = "rtio_acp.rs"]
 mod rtio;
 mod kernel;
 mod moninj;
 mod load_pl;
 mod eh_artiq;
 mod panic;
+mod logger;
+mod mgmt;
+mod analyzer;
+mod irq;
 
 fn init_gateware() {
     // Set up PS->PL clocks
@@ -83,7 +99,7 @@ fn identifier_read(buf: &mut [u8]) -> &str {
     }
 }
 
-fn init_rtio(timer: GlobalTimer, cfg: &config::Config) {
+fn init_rtio(timer: &mut GlobalTimer, cfg: &config::Config) {
     let clock_sel =
         if let Ok(rtioclk) = cfg.read_str("rtioclk") {
             match rtioclk.as_ref() {
@@ -106,17 +122,21 @@ fn init_rtio(timer: GlobalTimer, cfg: &config::Config) {
             0
         };
 
+    loop {
         unsafe {
             pl::csr::rtio_crg::pll_reset_write(1);
             pl::csr::rtio_crg::clock_sel_write(clock_sel);
             pl::csr::rtio_crg::pll_reset_write(0);
         }
-    let mut countdown = timer.countdown();
+        timer.delay_ms(1);
-    countdown.start(Milliseconds(1));
-    block!(countdown.wait()).unwrap();
         let locked = unsafe { pl::csr::rtio_crg::pll_locked_read() != 0 };
-    if !locked {
+        if locked {
-        panic!("RTIO PLL failed to lock");
+            info!("RTIO PLL locked");
+            break;
+        } else {
+            warn!("RTIO PLL failed to lock, retrying...");
+            timer.delay_ms(500);
+        }
     }
 
     unsafe {
@@ -124,14 +144,55 @@ fn init_rtio(timer: GlobalTimer, cfg: &config::Config) {
     }
 }
 
+fn wait_for_async_rtio_error() -> nb::Result<(), Void> {
+    unsafe {
+        if pl::csr::rtio_core::async_error_read() != 0 {
+            Ok(())
+        } else {
+            Err(nb::Error::WouldBlock)
+        }
+    }
+}
+
+async fn report_async_rtio_errors() {
+    loop {
+        let _ = block_async!(wait_for_async_rtio_error()).await;
+        unsafe {
+            let errors = pl::csr::rtio_core::async_error_read();
+            if errors & 1 != 0 {
+                error!("RTIO collision involving channel {}",
+                       pl::csr::rtio_core::collision_channel_read());
+            }
+            if errors & 2 != 0 {
+                error!("RTIO busy error involving channel {}",
+                       pl::csr::rtio_core::busy_channel_read());
+            }
+            if errors & 4 != 0 {
+                error!("RTIO sequence error involving channel {}",
+                       pl::csr::rtio_core::sequence_error_channel_read());
+            }
+            pl::csr::rtio_core::async_error_write(errors);
+        }
+    }
+}
+
+static mut LOG_BUFFER: [u8; 1<<17] = [0; 1<<17];
+
 #[no_mangle]
 pub fn main_core0() {
-    let timer = GlobalTimer::start();
+    let mut timer = GlobalTimer::start();
-    let _ = logger::init();
+
+    let buffer_logger = unsafe {
+        logger::BufferLogger::new(&mut LOG_BUFFER[..])
+    };
+    buffer_logger.set_uart_log_level(log::LevelFilter::Debug);
+    buffer_logger.register();
     log::set_max_level(log::LevelFilter::Debug);
+
     info!("NAR3/Zynq7000 starting...");
 
-    ram::init_alloc_linker();
+    ram::init_alloc_core0();
+    gic::InterruptController::new(mpcore::RegisterBlock::new()).enable_interrupts();
 
     init_gateware();
     info!("detected gateware: {}", identifier_read(&mut [0; 64]));
@@ -144,7 +205,8 @@ pub fn main_core0() {
         }
     };
 
-    init_rtio(timer, &cfg);
+    init_rtio(&mut timer, &cfg);
+    task::spawn(report_async_rtio_errors());
 
     comms::main(timer, &cfg);
 }

src/runtime/src/mgmt.rs

@@ -0,0 +1,173 @@
+use futures::{future::poll_fn, task::Poll};
+use libasync::{smoltcp::TcpStream, task};
+use libboard_zynq::smoltcp;
+use core::cell::RefCell;
+use alloc::rc::Rc;
+use log::{self, info, warn, LevelFilter};
+
+use crate::logger::{BufferLogger, LogBufferRef};
+use crate::proto_async::*;
+use num_derive::FromPrimitive;
+use num_traits::FromPrimitive;
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub enum Error {
+    NetworkError(smoltcp::Error),
+    UnknownLogLevel(u8),
+    UnexpectedPattern,
+    UnrecognizedPacket,
+}
+
+type Result<T> = core::result::Result<T, Error>;
+
+impl core::fmt::Display for Error {
+    fn fmt(&self, f: &mut core::fmt::Formatter) -> core::fmt::Result {
+        match self {
+            &Error::NetworkError(error)  => write!(f, "network error: {}", error),
+            &Error::UnknownLogLevel(lvl) => write!(f, "unknown log level {}", lvl),
+            &Error::UnexpectedPattern    => write!(f, "unexpected pattern"),
+            &Error::UnrecognizedPacket   => write!(f, "unrecognized packet"),
+        }
+    }
+}
+
+impl From<smoltcp::Error> for Error {
+    fn from(error: smoltcp::Error) -> Self {
+        Error::NetworkError(error)
+    }
+}
+
+#[derive(Debug, FromPrimitive)]
+pub enum Request {
+    GetLog = 1,
+    ClearLog = 2,
+    PullLog = 7,
+    SetLogFilter = 3,
+    SetUartLogFilter = 6,
+}
+
+#[repr(i8)]
+pub enum Reply {
+    Success = 1,
+    LogContent = 2,
+}
+
+async fn read_log_level_filter(stream: &mut TcpStream) -> Result<log::LevelFilter> {
+    Ok(match read_i8(stream).await? {
+        0 => log::LevelFilter::Off,
+        1 => log::LevelFilter::Error,
+        2 => log::LevelFilter::Warn,
+        3 => log::LevelFilter::Info,
+        4 => log::LevelFilter::Debug,
+        5 => log::LevelFilter::Trace,
+        lv => return Err(Error::UnknownLogLevel(lv as u8)),
+    })
+}
+
+async fn get_logger_buffer_pred<F>(f: F) -> LogBufferRef<'static>
+where
+    F: Fn(&LogBufferRef) -> bool,
+{
+    poll_fn(|ctx| {
+        let logger = unsafe { BufferLogger::get_logger().as_mut().unwrap() };
+        match logger.buffer() {
+            Some(buffer) if f(&buffer) => Poll::Ready(buffer),
+            _ => {
+                ctx.waker().wake_by_ref();
+                Poll::Pending
+            }
+        }
+    })
+    .await
+}
+
+async fn get_logger_buffer() -> LogBufferRef<'static> {
+    get_logger_buffer_pred(|_| true).await
+}
+
+async fn handle_connection(stream: &mut TcpStream, pull_id: Rc<RefCell<u32>>) -> Result<()> {
+    if !expect(&stream, b"ARTIQ management\n").await? {
+        return Err(Error::UnexpectedPattern);
+    }
+
+    loop {
+        let msg = read_i8(stream).await;
+        if let Err(smoltcp::Error::Illegal) = msg {
+            return Ok(());
+        }
+        let msg: Request = FromPrimitive::from_i8(msg?).ok_or(Error::UnrecognizedPacket)?;
+        match msg {
+            Request::GetLog => {
+                let buffer = get_logger_buffer().await.extract().as_bytes().to_vec();
+                write_i8(stream, Reply::LogContent as i8).await?;
+                write_chunk(stream, &buffer).await?;
+            }
+            Request::ClearLog => {
+                let mut buffer = get_logger_buffer().await;
+                buffer.clear();
+                write_i8(stream, Reply::Success as i8).await?;
+            }
+            Request::PullLog => {
+                let id = {
+                    let mut guard = pull_id.borrow_mut();
+                    *guard += 1;
+                    *guard
+                };
+                loop {
+                    let mut buffer = get_logger_buffer_pred(|b| !b.is_empty()).await;
+                    if id != *pull_id.borrow() {
+                        // another connection attempts to pull the log...
+                        // abort this connection...
+                        break;
+                    }
+                    let bytes = buffer.extract().as_bytes().to_vec();
+                    buffer.clear();
+                    core::mem::drop(buffer);
+                    write_chunk(stream, &bytes).await?;
+                    if log::max_level() == LevelFilter::Trace {
+                        // temporarily discard all trace level log
+                        let logger = unsafe { BufferLogger::get_logger().as_mut().unwrap() };
+                        logger.set_buffer_log_level(LevelFilter::Debug);
+                        stream.flush().await?;
+                        logger.set_buffer_log_level(LevelFilter::Trace);
+                    }
+                }
+            },
+            Request::SetLogFilter => {
+                let lvl = read_log_level_filter(stream).await?;
+                info!("Changing log level to {}", lvl);
+                log::set_max_level(lvl);
+                write_i8(stream, Reply::Success as i8).await?;
+            }
+            Request::SetUartLogFilter => {
+                let lvl = read_log_level_filter(stream).await?;
+                info!("Changing UART log level to {}", lvl);
+                unsafe {
+                    BufferLogger::get_logger()
+                        .as_ref()
+                        .unwrap()
+                        .set_uart_log_level(lvl);
+                }
+                write_i8(stream, Reply::Success as i8).await?;
+            }
+        }
+    }
+}
+
+pub fn start() {
+    task::spawn(async move {
+        let pull_id = Rc::new(RefCell::new(0u32));
+        loop {
+            let mut stream = TcpStream::accept(1380, 2048, 2048).await.unwrap();
+            let pull_id = pull_id.clone();
+            task::spawn(async move {
+                info!("received connection");
+                let _ = handle_connection(&mut stream, pull_id)
+                    .await
+                    .map_err(|e| warn!("connection terminated: {:?}", e));
+                let _ = stream.flush().await;
+                let _ = stream.abort().await;
+            });
+        }
+    });
+}

src/runtime/src/moninj.rs

@@ -1,6 +1,6 @@
 use core::fmt;
 use alloc::collections::BTreeMap;
-use log::{debug, warn};
+use log::{debug, info, warn};
 use void::Void;
 
 use libboard_zynq::{smoltcp, timer::GlobalTimer, time::Milliseconds};
@@ -180,9 +180,13 @@ pub fn start(timer: GlobalTimer) {
         loop {
             let stream = TcpStream::accept(1383, 2048, 2048).await.unwrap();
             task::spawn(async move {
-                let _ = handle_connection(&stream, timer)
+                info!("received connection");
-                    .await
+                let result = handle_connection(&stream, timer).await;
-                    .map_err(|e| warn!("connection terminated: {}", e));
+                match result {
+                    Err(Error::NetworkError(smoltcp::Error::Illegal)) => info!("peer closed connection"),
+                    Err(error) => warn!("connection terminated: {}", error),
+                    _ => (),
+                }
                 let _ = stream.flush().await;
                 let _ = stream.abort().await;
             });

src/runtime/src/panic.rs

@@ -1,8 +1,21 @@
 use libboard_zynq::{print, println};
+use libregister::RegisterR;
+use libcortex_a9::regs::MPIDR;
 use unwind::backtrace;
 
+static mut PANICKED: [bool; 2] = [false; 2];
+
 #[panic_handler]
 fn panic(info: &core::panic::PanicInfo) -> ! {
+    let id = MPIDR.read().cpu_id() as usize;
+    print!("Core {} ", id);
+    unsafe {
+        if PANICKED[id] {
+            println!("nested panic!");
+            loop {}
+        }
+        PANICKED[id] = true;
+    }
     print!("panic at ");
     if let Some(location) = info.location() {
         print!("{}:{}:{}", location.file(), location.line(), location.column());
@@ -18,9 +31,9 @@ fn panic(info: &core::panic::PanicInfo) -> ! {
     let _ = backtrace(|ip| {
         // Backtrace gives us the return address, i.e. the address after the delay slot,
         // but we're interested in the call instruction.
-        println!("{:#08x}", ip - 2 * 4);
+        print!("{:#08x} ", ip - 2 * 4);
     });
-    println!("End backtrace");
+    println!("\nEnd backtrace");
 
     loop {}
 }

src/runtime/src/proto_async.rs

@@ -1,73 +1,106 @@
-use core::task::Poll;
 use core::cmp::min;
 use core::cell::RefCell;
 
 use libboard_zynq::smoltcp;
 use libasync::smoltcp::TcpStream;
 
-// TODO: use byteorder, make it more like libio
+type Result<T> = core::result::Result<T, smoltcp::Error>;
 
-pub type Result<T> = core::result::Result<T, smoltcp::Error>;
+enum RecvState<T> {
+    NeedsMore(usize, T), // bytes consumed so far, partial result
+    Completed(T),        // final result
+}
 
 pub async fn expect(stream: &TcpStream, pattern: &[u8]) -> Result<bool> {
-    stream.recv(|buf| {
+    let mut state = RecvState::NeedsMore(0, true);
-        for (i, b) in buf.iter().enumerate() {
+    loop {
-            if *b == pattern[i] {
+        state = stream.recv(|buf| {
-                if i + 1 == pattern.len() {
+            let mut consumed = 0;
-                    return Poll::Ready((i + 1, Ok(true)));
+            if let RecvState::NeedsMore(mut cur_index, _) = state {
+                for b in buf.iter() {
+                    consumed += 1;
+                    if *b == pattern[cur_index] {
+                        if cur_index + 1 == pattern.len() {
+                            return (consumed, RecvState::Completed(true));
                         }
                     } else {
-                return Poll::Ready((i + 1, Ok(false)));
+                        return (consumed, RecvState::Completed(false));
+                    }
+                    cur_index += 1;
+                }
+                (consumed, RecvState::NeedsMore(cur_index, true))
+            } else {
+                unreachable!();
+            }
+        }).await?;
+        if let RecvState::Completed(result) = state {
+            return Ok(result);
         }
     }
-        Poll::Pending
-    }).await?
 }
 
 pub async fn read_bool(stream: &TcpStream) -> Result<bool> {
     Ok(stream.recv(|buf| {
-        Poll::Ready((1, buf[0] != 0))
+        (1, buf[0] != 0)
     }).await?)
 }
 
 pub async fn read_i8(stream: &TcpStream) -> Result<i8> {
     Ok(stream.recv(|buf| {
-        Poll::Ready((1, buf[0] as i8))
+        (1, buf[0] as i8)
     }).await?)
 }
 
 pub async fn read_i32(stream: &TcpStream) -> Result<i32> {
-    Ok(stream.recv(|buf| {
+    let mut state = RecvState::NeedsMore(0, 0);
-        if buf.len() >= 4 {
+    loop {
-            let value =
+        state = stream.recv(|buf| {
-                  ((buf[0] as i32) << 24)
+            let mut consumed = 0;
-                | ((buf[1] as i32) << 16)
+            if let RecvState::NeedsMore(mut cur_index, mut cur_value) = state {
-                | ((buf[2] as i32) << 8)
+                for b in buf.iter() {
-                |  (buf[3] as i32);
+                    consumed += 1;
-            Poll::Ready((4, value))
+                    cur_index += 1;
-        } else {
+                    cur_value <<= 8;
-            Poll::Pending
+                    cur_value |= *b as i32;
+                    if cur_index == 4 {
+                        return (consumed, RecvState::Completed(cur_value));
+                    }
+                }
+                (consumed, RecvState::NeedsMore(cur_index, cur_value))
+            } else {
+                unreachable!();
+            }
+        }).await?;
+        if let RecvState::Completed(result) = state {
+            return Ok(result);
+        }
     }
-    }).await?)
 }
 
 pub async fn read_i64(stream: &TcpStream) -> Result<i64> {
-    Ok(stream.recv(|buf| {
+    let mut state = RecvState::NeedsMore(0, 0);
-        if buf.len() >= 8 {
+    loop {
-            let value =
+        state = stream.recv(|buf| {
-                  ((buf[0] as i64) << 56)
+            let mut consumed = 0;
-                | ((buf[1] as i64) << 48)
+            if let RecvState::NeedsMore(mut cur_index, mut cur_value) = state {
-                | ((buf[2] as i64) << 40)
+                for b in buf.iter() {
-                | ((buf[3] as i64) << 32)
+                    consumed += 1;
-                | ((buf[4] as i64) << 24)
+                    cur_index += 1;
-                | ((buf[5] as i64) << 16)
+                    cur_value <<= 8;
-                | ((buf[6] as i64) << 8)
+                    cur_value |= *b as i64;
-                |  (buf[7] as i64);
+                    if cur_index == 8 {
-            Poll::Ready((8, value))
+                        return (consumed, RecvState::Completed(cur_value));
-        } else {
+                    }
-            Poll::Pending
+                }
+                (consumed, RecvState::NeedsMore(cur_index, cur_value))
+            } else {
+                unreachable!();
+            }
+        }).await?;
+        if let RecvState::Completed(result) = state {
+            return Ok(result);
+        }
     }
-    }).await?)
 }
 
 pub async fn read_chunk(stream: &TcpStream, destination: &mut [u8]) -> Result<()> {
@@ -79,7 +112,7 @@ pub async fn read_chunk(stream: &TcpStream, destination: &mut [u8]) -> Result<()
             let mut destination = destination.borrow_mut();
             let count = min(total - done, buf.len());
             destination[done..done + count].copy_from_slice(&buf[..count]);
-            Poll::Ready((count, count))
+            (count, count)
         }).await?;
         done += count;
     }

src/runtime/src/rpc.rs

@@ -13,6 +13,18 @@ use crate::proto_core_io::ProtoWrite;
 use crate::proto_async;
 use self::tag::{Tag, TagIterator, split_tag};
 
+unsafe fn align_ptr<T>(ptr: *const ()) -> *const T {
+    let alignment = core::mem::align_of::<T>() as isize;
+    let fix = (alignment - (ptr as isize) % alignment) % alignment;
+    ((ptr as isize) + fix) as *const T
+}
+
+unsafe fn align_ptr_mut<T>(ptr: *mut ()) -> *mut T {
+    let alignment = core::mem::align_of::<T>() as isize;
+    let fix = (alignment - (ptr as isize) % alignment) % alignment;
+    ((ptr as isize) + fix) as *mut T
+}
+
 #[async_recursion(?Send)]
 async unsafe fn recv_value<F>(stream: &TcpStream, tag: Tag<'async_recursion>, data: &mut *mut (),
                               alloc: &(impl Fn(usize) -> F + 'async_recursion))
@@ -21,7 +33,7 @@ async unsafe fn recv_value<F>(stream: &TcpStream, tag: Tag<'async_recursion>, da
 {
     macro_rules! consume_value {
         ($ty:ty, |$ptr:ident| $map:expr) => ({
-            let $ptr = (*data) as *mut $ty;
+            let $ptr = align_ptr_mut::<$ty>(*data);
             *data = $ptr.offset(1) as *mut ();
             $map
         })
@@ -61,6 +73,7 @@ async unsafe fn recv_value<F>(stream: &TcpStream, tag: Tag<'async_recursion>, da
             Ok(())
         }
         Tag::List(it) | Tag::Array(it) => {
+            #[repr(C)]
             struct List { elements: *mut (), length: u32 };
             consume_value!(List, |ptr| {
                 (*ptr).length = proto_async::read_i32(stream).await? as u32;
@@ -108,7 +121,7 @@ unsafe fn send_value<W>(writer: &mut W, tag: Tag, data: &mut *const ())
 {
     macro_rules! consume_value {
         ($ty:ty, |$ptr:ident| $map:expr) => ({
-            let $ptr = (*data) as *const $ty;
+            let $ptr = align_ptr::<$ty>(*data);
             *data = $ptr.offset(1) as *const ();
             $map
         })
@@ -142,6 +155,7 @@ unsafe fn send_value<W>(writer: &mut W, tag: Tag, data: &mut *const ())
             Ok(())
         }
         Tag::List(it) | Tag::Array(it) => {
+            #[repr(C)]
             struct List { elements: *const (), length: u32 };
             consume_value!(List, |ptr| {
                 writer.write_u32((*ptr).length)?;
@@ -161,6 +175,7 @@ unsafe fn send_value<W>(writer: &mut W, tag: Tag, data: &mut *const ())
             Ok(())
         }
         Tag::Keyword(it) => {
+            #[repr(C)]
             struct Keyword<'a> { name: CSlice<'a, u8> };
             consume_value!(Keyword, |ptr| {
                 writer.write_string(str::from_utf8((*ptr).name.as_ref()).unwrap())?;
@@ -172,6 +187,7 @@ unsafe fn send_value<W>(writer: &mut W, tag: Tag, data: &mut *const ())
             // to accurately advance data.
         }
         Tag::Object => {
+            #[repr(C)]
             struct Object { id: u32 };
             consume_value!(*const Object, |ptr|
                 writer.write_u32((**ptr).id))

src/runtime/src/rtio_acp.rs

@@ -0,0 +1,250 @@
+use cslice::CSlice;
+use vcell::VolatileCell;
+use libcortex_a9::asm;
+
+use crate::artiq_raise;
+
+use crate::pl::csr;
+
+pub const RTIO_O_STATUS_WAIT:                      i32 = 1;
+pub const RTIO_O_STATUS_UNDERFLOW:                 i32 = 2;
+pub const RTIO_O_STATUS_DESTINATION_UNREACHABLE:   i32 = 4;
+pub const RTIO_I_STATUS_WAIT_EVENT:                i32 = 1;
+pub const RTIO_I_STATUS_OVERFLOW:                  i32 = 2;
+pub const RTIO_I_STATUS_WAIT_STATUS:               i32 = 4; // TODO
+pub const RTIO_I_STATUS_DESTINATION_UNREACHABLE:   i32 = 8;
+
+#[repr(C)]
+pub struct TimestampedData {
+    timestamp: i64,
+    data: i32,
+}
+
+#[repr(C, align(32))]
+struct Transaction {
+    request_cmd: i8,
+    padding0: i8,
+    padding1: i8,
+    padding2: i8,
+    request_target: i32,
+    request_timestamp: i64,
+    request_data: i64,
+    padding: i64,
+    reply_status: VolatileCell<i32>,
+    reply_data: VolatileCell<i32>,
+    reply_timestamp: VolatileCell<i64>
+}
+
+static mut TRANSACTION_BUFFER: Transaction = Transaction {
+    request_cmd: 0,
+    padding0: 0,
+    padding1: 0,
+    padding2: 0,
+    request_target: 0,
+    request_timestamp: 0,
+    request_data: 0,
+    padding: 0,
+    reply_status: VolatileCell::new(0),
+    reply_data: VolatileCell::new(0),
+    reply_timestamp: VolatileCell::new(0)
+};
+
+pub extern fn init() {
+    unsafe {
+        csr::rtio_core::reset_write(1);
+        csr::rtio::engine_addr_base_write(&TRANSACTION_BUFFER as *const Transaction as u32);
+        csr::rtio::enable_write(1);
+    }
+}
+
+pub extern fn get_destination_status(destination: i32) -> bool {
+    // TODO
+    destination == 0
+}
+
+pub extern fn get_counter() -> i64 {
+    unsafe {
+        csr::rtio::counter_update_write(1);
+        csr::rtio::counter_read() as i64
+    }
+}
+
+static mut NOW: i64 = 0;
+
+pub extern fn now_mu() -> i64 {
+    unsafe { NOW }
+}
+
+pub extern fn at_mu(t: i64) {
+    unsafe { NOW = t }
+}
+
+pub extern fn delay_mu(dt: i64) {
+    unsafe { NOW += dt }
+}
+
+#[inline(never)]
+unsafe fn process_exceptional_status(channel: i32, status: i32) {
+    let timestamp = now_mu();
+    if status & RTIO_O_STATUS_WAIT != 0 {
+        // FIXME: this is a kludge and probably buggy (kernel interrupted?)
+        while csr::rtio::o_status_read() as i32 & RTIO_O_STATUS_WAIT != 0 {}
+    }
+    if status & RTIO_O_STATUS_UNDERFLOW != 0 {
+        artiq_raise!("RTIOUnderflow",
+            "RTIO underflow at {0} mu, channel {1}, slack {2} mu",
+            timestamp, channel as i64, timestamp - get_counter());
+    }
+    if status & RTIO_O_STATUS_DESTINATION_UNREACHABLE != 0 {
+        artiq_raise!("RTIODestinationUnreachable",
+            "RTIO destination unreachable, output, at {0} mu, channel {1}",
+            timestamp, channel as i64, 0);
+    }
+}
+
+pub extern fn output(target: i32, data: i32) {
+    unsafe {
+        // Clear status so we can observe response
+        TRANSACTION_BUFFER.reply_status.set(0);
+
+        TRANSACTION_BUFFER.request_cmd = 0;
+        TRANSACTION_BUFFER.request_target = target;
+        TRANSACTION_BUFFER.request_timestamp = NOW;
+        TRANSACTION_BUFFER.request_data = data as i64;
+
+        asm::dmb();
+        asm::sev();
+
+        let mut status;
+        loop {
+            status = TRANSACTION_BUFFER.reply_status.get();
+            if status != 0 {
+                break
+            }
+        }
+
+        let status = status & !0x10000;
+        if status != 0 {
+            process_exceptional_status(target >> 8, status);
+        }
+    }
+}
+
+pub extern fn output_wide(target: i32, data: CSlice<i32>) {
+    // TODO
+    unimplemented!();
+}
+
+pub extern fn input_timestamp(timeout: i64, channel: i32) -> i64 {
+   unsafe {
+        // Clear status so we can observe response
+        TRANSACTION_BUFFER.reply_status.set(0);
+
+        TRANSACTION_BUFFER.request_cmd = 1;
+        TRANSACTION_BUFFER.request_timestamp = NOW;
+        TRANSACTION_BUFFER.request_target = channel << 8;
+
+        asm::dmb();
+        asm::sev();
+
+        let mut status;
+        loop {
+            status = TRANSACTION_BUFFER.reply_status.get();
+            if status != 0 {
+                break
+            }
+        }
+
+        if status & RTIO_I_STATUS_OVERFLOW != 0 {
+            artiq_raise!("RTIOOverflow",
+                         "RTIO input overflow on channel {0}",
+                         channel as i64, 0, 0);
+        }
+        if status & RTIO_I_STATUS_WAIT_EVENT != 0 {
+            return -1
+        }
+        if status & RTIO_I_STATUS_DESTINATION_UNREACHABLE != 0 {
+            artiq_raise!("RTIODestinationUnreachable",
+                         "RTIO destination unreachable, input, on channel {0}",
+                         channel as i64, 0, 0);
+        }
+
+        TRANSACTION_BUFFER.reply_timestamp.get()
+    }
+}
+
+pub extern fn input_data(channel: i32) -> i32 {
+    unsafe {
+        TRANSACTION_BUFFER.reply_status.set(0);
+
+        TRANSACTION_BUFFER.request_cmd = 1;
+        TRANSACTION_BUFFER.request_timestamp = -1;
+        TRANSACTION_BUFFER.request_target = channel << 8;
+
+        asm::dmb();
+        asm::sev();
+
+        let mut status;
+        loop {
+            status = TRANSACTION_BUFFER.reply_status.get();
+            if status != 0 {
+                break
+            }
+        }
+
+        if status & RTIO_I_STATUS_OVERFLOW != 0 {
+            artiq_raise!("RTIOOverflow",
+                         "RTIO input overflow on channel {0}",
+                         channel as i64, 0, 0);
+        }
+        if status & RTIO_I_STATUS_DESTINATION_UNREACHABLE != 0 {
+            artiq_raise!("RTIODestinationUnreachable",
+                         "RTIO destination unreachable, input, on channel {0}",
+                         channel as i64, 0, 0);
+        }
+
+        TRANSACTION_BUFFER.reply_data.get()
+    }
+}
+
+pub extern fn input_timestamped_data(timeout: i64, channel: i32) -> TimestampedData {
+    unsafe {
+        TRANSACTION_BUFFER.reply_status.set(0);
+
+        TRANSACTION_BUFFER.request_cmd = 1;
+        TRANSACTION_BUFFER.request_timestamp = timeout;
+        TRANSACTION_BUFFER.request_target = channel << 8;
+
+        asm::dmb();
+        asm::sev();
+
+        let mut status;
+        loop {
+            status = TRANSACTION_BUFFER.reply_status.get();
+            if status != 0 {
+                break
+            }
+        }
+
+        if status & RTIO_I_STATUS_OVERFLOW != 0 {
+            artiq_raise!("RTIOOverflow",
+                         "RTIO input overflow on channel {0}",
+                         channel as i64, 0, 0);
+        }
+        if status & RTIO_I_STATUS_DESTINATION_UNREACHABLE != 0 {
+            artiq_raise!("RTIODestinationUnreachable",
+                         "RTIO destination unreachable, input, on channel {0}",
+                         channel as i64, 0, 0);
+        }
+
+        TimestampedData {
+            timestamp: TRANSACTION_BUFFER.reply_timestamp.get(),
+            data: TRANSACTION_BUFFER.reply_data.get(),
+        }
+    }
+}
+
+pub fn write_log(data: &[i8]) {
+    // TODO
+    unimplemented!();
+}

src/runtime/src/rtio_csr.rs

@@ -1,6 +1,6 @@
 use core::ptr::{read_volatile, write_volatile};
 use cslice::CSlice;
-use log::error;
+
 use crate::artiq_raise;
 
 use crate::pl::csr;
@@ -124,16 +124,17 @@ pub extern fn input_timestamp(timeout: i64, channel: i32) -> i64 {
         }
 
         if status & RTIO_I_STATUS_OVERFLOW != 0 {
-            csr::rtio::i_overflow_reset_write(1);
+            artiq_raise!("RTIOOverflow",
-            error!("RTIO input overflow on channel {0}",
+                         "RTIO input overflow on channel {0}",
-                   channel as i64);
+                         channel as i64, 0, 0);
         }
         if status & RTIO_I_STATUS_WAIT_EVENT != 0 {
             return -1
         }
         if status & RTIO_I_STATUS_DESTINATION_UNREACHABLE != 0 {
-            error!("RTIO destination unreachable, input, on channel {0}",
+            artiq_raise!("RTIODestinationUnreachable",
-                   channel as i64);
+                         "RTIO destination unreachable, input, on channel {0}",
+                         channel as i64, 0, 0);
         }
 
         csr::rtio::i_timestamp_read() as i64
@@ -151,13 +152,14 @@ pub extern fn input_data(channel: i32) -> i32 {
         }
 
         if status & RTIO_I_STATUS_OVERFLOW != 0 {
-            csr::rtio::i_overflow_reset_write(1);
+            artiq_raise!("RTIOOverflow",
-            error!("RTIO input overflow on channel {0}",
+                         "RTIO input overflow on channel {0}",
-                   channel as i64);
+                         channel as i64, 0, 0);
         }
         if status & RTIO_I_STATUS_DESTINATION_UNREACHABLE != 0 {
-            error!("RTIO destination unreachable, input, on channel {0}",
+            artiq_raise!("RTIODestinationUnreachable",
-                   channel as i64);
+                         "RTIO destination unreachable, input, on channel {0}",
+                         channel as i64, 0, 0);
         }
 
         rtio_i_data_read(0) as i32
@@ -175,16 +177,17 @@ pub extern fn input_timestamped_data(timeout: i64, channel: i32) -> TimestampedD
         }
 
         if status & RTIO_I_STATUS_OVERFLOW != 0 {
-            csr::rtio::i_overflow_reset_write(1);
+            artiq_raise!("RTIOOverflow",
-            error!("RTIO input overflow on channel {0}",
+                         "RTIO input overflow on channel {0}",
-                   channel as i64);
+                         channel as i64, 0, 0);
         }
         if status & RTIO_I_STATUS_WAIT_EVENT != 0 {
             return TimestampedData { timestamp: -1, data: 0 }
         }
         if status & RTIO_I_STATUS_DESTINATION_UNREACHABLE != 0 {
-            error!("RTIO destination unreachable, input, on channel {0}",
+            artiq_raise!("RTIODestinationUnreachable",
-                   channel as i64);
+                         "RTIO destination unreachable, input, on channel {0}",
+                         channel as i64, 0, 0);
         }
 
         TimestampedData {
@@ -193,3 +196,23 @@ pub extern fn input_timestamped_data(timeout: i64, channel: i32) -> TimestampedD
         }
     }
 }
+
+pub fn write_log(data: &[i8]) {
+    unsafe {
+        csr::rtio::target_write(csr::CONFIG_RTIO_LOG_CHANNEL << 8);
+
+        let mut word: u32 = 0;
+        for i in 0..data.len() {
+            word <<= 8;
+            word |= data[i] as u32;
+            if i % 4 == 3 {
+                rtio_o_data_write(0, word);
+                word = 0;
+            }
+        }
+
+        if word != 0 {
+            rtio_o_data_write(0, word);
+        }
+    }
+}

src/szl/Cargo.toml

@@ -12,9 +12,9 @@ default = ["target_zc706"]
 [dependencies]
 log = "0.4"
 cstr_core = { version = "0.2", default-features = false }
-libboard_zynq = { git = "https://git.m-labs.hk/M-Labs/zc706.git" }
+libboard_zynq = { git = "https://git.m-labs.hk/M-Labs/zynq-rs.git" }
-libsupport_zynq = { git = "https://git.m-labs.hk/M-Labs/zc706.git" }
+libsupport_zynq = { git = "https://git.m-labs.hk/M-Labs/zynq-rs.git", default-features = false, features = ["dummy_irq_handler"] }
-libcortex_a9 = { git = "https://git.m-labs.hk/M-Labs/zc706.git" }
+libcortex_a9 = { git = "https://git.m-labs.hk/M-Labs/zynq-rs.git" }
 
 [build-dependencies]
 cc = { version = "1.0.1" }

src/szl/build.rs

@@ -32,7 +32,8 @@ pub fn compile_unlzma() {
     cfg.flag("-fPIC");
     cfg.flag("-fno-stack-protector");
     cfg.flag("--target=armv7-none-eabihf");
-    cfg.flag("-O2");
+    cfg.flag("-Oz");
+    cfg.flag("-flto=full");
 
     let sources = vec![
         "unlzma.c",

src/szl/link.x

@@ -34,6 +34,12 @@ SECTIONS
         __bss_end = .;
     } > OCM3
 
+    .heap (NOLOAD) : ALIGN(8)
+    {
+        __heap0_start = .;
+        __heap0_end = .;
+    } > OCM3
+
     .stack1 (NOLOAD) : ALIGN(8)
     {
         __stack1_end = .;

src/szl/src/main.rs

@@ -1,5 +1,6 @@
 #![no_std]
 #![no_main]
+#![feature(panic_info_message)]
 
 extern crate log;
 
@@ -7,9 +8,15 @@ use core::mem;
 use log::{debug, info, error};
 use cstr_core::CStr;
 
-use libcortex_a9::{enable_fpu, cache::dcci_slice};
+use libcortex_a9::{
+    enable_fpu,
+    cache::{dcci_slice, iciallu, bpiall},
+    asm::{dsb, isb},
+};
 use libboard_zynq::{
-    self as zynq, clocks::Clocks, clocks::source::{ClockSource, ArmPll, IoPll},
+    self as zynq, println,
+    clocks::Clocks, clocks::source::{ClockSource, ArmPll, IoPll},
+    stdio,
     logger,
     timer::GlobalTimer,
 };
@@ -23,7 +30,17 @@ extern "C" {
 }
 
 extern fn lzma_error(message: *const u8) {
-    error!("LZMA error: {}", unsafe { CStr::from_ptr(message) }.to_str().unwrap());
+    let msg = unsafe {CStr::from_ptr(message)}.to_str();
+    if let Ok(msg) = msg {
+        println!("LZMA error: {}", msg);
+    }
+}
+
+#[panic_handler]
+fn panic(_: &core::panic::PanicInfo) -> ! {
+    stdio::drop_uart();
+    println!("panicked!");
+    loop {}
 }
 
 #[no_mangle]
@@ -31,6 +48,16 @@ pub fn main_core0() {
     GlobalTimer::start();
     logger::init().unwrap();
     log::set_max_level(log::LevelFilter::Debug);
+    println!(r#"
+
+                     __________   __
+                    / ___/__  /  / /
+                    \__ \  / /  / /
+                   ___/ / / /__/ /___
+                  /____/ /____/_____/
+
+                   (C) 2020 M-Labs
+"#);
     info!("Simple Zynq Loader starting...");
 
     enable_fpu();
@@ -53,10 +80,15 @@ pub fn main_core0() {
         error!("decompression failed");
     } else {
         // Flush data cache entries for all of DDR, including
-        // Memory/Instruction Symchronization Barriers
+        // Memory/Instruction Synchronization Barriers
         dcci_slice(unsafe {
             core::slice::from_raw_parts(ddr.ptr::<u8>(), ddr.size())
         });
+        dsb();
+        iciallu();
+        bpiall();
+        dsb();
+        isb();
 
         // Start core0 only, for compatibility with FSBL.
         info!("executing payload");