Gateware: remove redundant si549.py & wrpll.py

rtio_clocking: update mmcm setting to use HIGH bandwidth

README.md

@@ -62,7 +62,7 @@ Notes:
 License
 -------
 
-Copyright (C) 2019-2023 M-Labs Limited.
+Copyright (C) 2019-2024 M-Labs Limited.
 
 ARTIQ is free software: you can redistribute it and/or modify
 it under the terms of the GNU Lesser General Public License as published by

flake.lock

@@ -11,11 +11,11 @@
         "src-pythonparser": "src-pythonparser"
       },
       "locked": {
-        "lastModified": 1706785107,
-        "narHash": "sha256-Uj72tqigiOCdewSSBBMg6zUpVKhwjAo1HeLJgvyZ3oc=",
+        "lastModified": 1716972728,
+        "narHash": "sha256-88J+eckZamtwhcCQkPpKLu6R1hmgj5+C9n2U5i+sHUE=",
         "ref": "refs/heads/master",
-        "rev": "3aaa7e04f26a495e8847e47424bfc16d76d82bf8",
-        "revCount": 8672,
+        "rev": "49e402780bebba437c6098047ab1dc68eaf5a17c",
+        "revCount": 8808,
         "type": "git",
         "url": "https://github.com/m-labs/artiq.git"
       },
@@ -37,11 +37,11 @@
         ]
       },
       "locked": {
-        "lastModified": 1701573753,
-        "narHash": "sha256-vhEtXjb9AM6/HnsgfVmhJQeqQ9JqysUm7iWNzTIbexs=",
+        "lastModified": 1707216368,
+        "narHash": "sha256-ZXoqzG2QsVsybALLYXs473avXcyKSZNh2kIgcPo60XQ=",
         "owner": "m-labs",
         "repo": "artiq-comtools",
-        "rev": "199bdabf4de49cb7ada8a4ac7133008e0f8434b7",
+        "rev": "e5d0204490bccc07ef9141b0d7c405ab01cb8273",
         "type": "github"
       },
       "original": {
@@ -103,11 +103,11 @@
     "mozilla-overlay_3": {
       "flake": false,
       "locked": {
-        "lastModified": 1695805681,
-        "narHash": "sha256-1ElPLD8eFfnuIk0G52HGGpRtQZ4QPCjChRlEOfkZ5ro=",
+        "lastModified": 1704373101,
+        "narHash": "sha256-+gi59LRWRQmwROrmE1E2b3mtocwueCQqZ60CwLG+gbg=",
         "owner": "mozilla",
         "repo": "nixpkgs-mozilla",
-        "rev": "6eabade97bc28d707a8b9d82ad13ef143836736e",
+        "rev": "9b11a87c0cc54e308fa83aac5b4ee1816d5418a2",
         "type": "github"
       },
       "original": {
@@ -118,16 +118,16 @@
     },
     "nixpkgs": {
       "locked": {
-        "lastModified": 1706515015,
-        "narHash": "sha256-eFfY5A7wlYy3jD/75lx6IJRueg4noE+jowl0a8lIlVo=",
+        "lastModified": 1716542732,
+        "narHash": "sha256-0Y9fRr0CUqWT4KgBITmaGwlnNIGMYuydu2L8iLTfHU4=",
         "owner": "NixOS",
         "repo": "nixpkgs",
-        "rev": "f4a8d6d5324c327dcc2d863eb7f3cc06ad630df4",
+        "rev": "d12251ef6e8e6a46e05689eeccd595bdbd3c9e60",
         "type": "github"
       },
       "original": {
         "owner": "NixOS",
-        "ref": "nixos-23.11",
+        "ref": "nixos-24.05",
         "repo": "nixpkgs",
         "type": "github"
       }
@@ -147,11 +147,11 @@
         ]
       },
       "locked": {
-        "lastModified": 1701572254,
-        "narHash": "sha256-ixq8dlpyOytDr+d/OmW8v1Ioy9V2G2ibOlNj8GFDSq4=",
+        "lastModified": 1708937641,
+        "narHash": "sha256-Hkb9VYFzFgkYxfbh4kYcDSn7DbMUYehoQDeTALrxo2Q=",
         "owner": "m-labs",
         "repo": "sipyco",
-        "rev": "cceac0df537887135f99aa6b1bdd82853f16b4d6",
+        "rev": "4a28b311ce0069454b4e8fe1e6049db11b9f1296",
         "type": "github"
       },
       "original": {
@@ -163,11 +163,11 @@
     "src-migen": {
       "flake": false,
       "locked": {
-        "lastModified": 1702942348,
-        "narHash": "sha256-gKIfHZxsv+jcgDFRW9mPqmwqbZXuRvXefkZcSFjOGHw=",
+        "lastModified": 1715484909,
+        "narHash": "sha256-4DCHBUBfc/VA+7NW2Hr0+JP4NnKPru2uVJyZjCCk0Ws=",
         "owner": "m-labs",
         "repo": "migen",
-        "rev": "50934ad10a87ade47219b796535978b9bdf24023",
+        "rev": "4790bb577681a8c3a8d226bc196a4e5deb39e4df",
         "type": "github"
       },
       "original": {
@@ -179,11 +179,11 @@
     "src-misoc": {
       "flake": false,
       "locked": {
-        "lastModified": 1699352904,
-        "narHash": "sha256-SglyTmXOPv8jJOjwAjJrj/WhAkItQfUbvKfUqrynwRg=",
+        "lastModified": 1715647536,
+        "narHash": "sha256-q+USDcaKHABwW56Jzq8u94iGPWlyLXMyVt0j/Gyg+IE=",
         "ref": "refs/heads/master",
-        "rev": "a53859f2167c31ab5225b6c09f30cf05527b94f4",
-        "revCount": 2452,
+        "rev": "fea9de558c730bc394a5936094ae95bb9d6fa726",
+        "revCount": 2455,
         "submodules": true,
         "type": "git",
         "url": "https://github.com/m-labs/misoc.git"
@@ -234,11 +234,11 @@
         ]
       },
       "locked": {
-        "lastModified": 1702982463,
-        "narHash": "sha256-jKR3drE2rsTaYGEgIdv5kUo2LOb1JyIb4tJhVuCXTTc=",
+        "lastModified": 1716519432,
+        "narHash": "sha256-vgKBJCQRPCutJ4n+FtJNczMZULWW7J3B8icf/PUothw=",
         "ref": "refs/heads/master",
-        "rev": "4168eb63a7e846863331ae4e656cfd82a867cca8",
-        "revCount": 636,
+        "rev": "46dc25b89e46b9043129d77e3c9348916748e325",
+        "revCount": 645,
         "type": "git",
         "url": "https://git.m-labs.hk/m-labs/zynq-rs"
       },

flake.nix

@@ -11,6 +11,7 @@
     pkgs = import artiq.inputs.nixpkgs { system = "x86_64-linux"; overlays = [ (import mozilla-overlay) ]; };
     zynqpkgs = zynq-rs.packages.x86_64-linux;
     artiqpkgs = artiq.packages.x86_64-linux;
+    llvmPackages_11 = zynq-rs.llvmPackages_11;
 
     rust = zynq-rs.rust;
     rustPlatform = zynq-rs.rustPlatform;
@@ -113,7 +114,7 @@
       "nist_clock_satellite" "nist_qc2_satellite" "acpki_nist_clock_satellite" "acpki_nist_qc2_satellite" 
       "nist_clock_satellite_100mhz" "nist_qc2_satellite_100mhz" "acpki_nist_clock_satellite_100mhz" "acpki_nist_qc2_satellite_100mhz"
     ];
-    build = { target, variant, json ? null }: let
+    board-package-set = { target, variant, json ? null }: let
       szl = zynqpkgs."${target}-szl";
       fsbl = zynqpkgs."${target}-fsbl";
       fwtype = if builtins.elem variant sat_variants then "satman" else "runtime";
@@ -132,12 +133,12 @@
           pkgs.gnumake
           (pkgs.python3.withPackages(ps: [ ps.jsonschema artiqpkgs.migen migen-axi artiqpkgs.misoc artiqpkgs.artiq ]))
           zynqpkgs.cargo-xbuild
-          pkgs.llvmPackages_9.llvm
-          pkgs.llvmPackages_9.clang-unwrapped
+          llvmPackages_11.llvm
+          llvmPackages_11.clang-unwrapped
         ];
         buildPhase = ''
           export XARGO_RUST_SRC="${rust}/lib/rustlib/src/rust/library"
-          export CLANG_EXTRA_INCLUDE_DIR="${pkgs.llvmPackages_9.clang-unwrapped.lib}/lib/clang/9.0.1/include"
+          export CLANG_EXTRA_INCLUDE_DIR="${llvmPackages_11.clang-unwrapped.lib}/lib/clang/11.1.0/include"
           export CARGO_HOME=$(mktemp -d cargo-home.XXX)
           export ZYNQ_RS=${zynq-rs}
           make TARGET=${target} GWARGS="${if json == null then "-V ${variant}" else json}" ${fwtype}
@@ -273,7 +274,7 @@
     };
 
     # for hitl-tests
-    zc706-nist_qc2 = (build { target = "zc706"; variant = "nist_qc2"; });
+    zc706-nist_qc2 = (board-package-set { target = "zc706"; variant = "nist_qc2"; });
     zc706-hitl-tests = pkgs.stdenv.mkDerivation {
       name = "zc706-hitl-tests";
 
@@ -340,29 +341,29 @@
       {
         inherit fastnumbers artiq-netboot ramda migen-axi binutils-arm;
       } //
-      (build { target = "zc706"; variant = "nist_clock"; }) //
-      (build { target = "zc706"; variant = "nist_clock_master"; }) //
-      (build { target = "zc706"; variant = "nist_clock_master_100mhz"; }) //
-      (build { target = "zc706"; variant = "nist_clock_satellite"; }) //
-      (build { target = "zc706"; variant = "nist_clock_satellite_100mhz"; }) //
-      (build { target = "zc706"; variant = "nist_qc2"; }) //
-      (build { target = "zc706"; variant = "nist_qc2_master"; }) //
-      (build { target = "zc706"; variant = "nist_qc2_master_100mhz"; }) //
-      (build { target = "zc706"; variant = "nist_qc2_satellite"; }) //
-      (build { target = "zc706"; variant = "nist_qc2_satellite_100mhz"; }) //
-      (build { target = "zc706"; variant = "acpki_nist_clock"; }) //
-      (build { target = "zc706"; variant = "acpki_nist_clock_master"; }) //
-      (build { target = "zc706"; variant = "acpki_nist_clock_master_100mhz"; }) //
-      (build { target = "zc706"; variant = "acpki_nist_clock_satellite"; }) //
-      (build { target = "zc706"; variant = "acpki_nist_clock_satellite_100mhz"; }) //
-      (build { target = "zc706"; variant = "acpki_nist_qc2"; }) //
-      (build { target = "zc706"; variant = "acpki_nist_qc2_master"; }) //
-      (build { target = "zc706"; variant = "acpki_nist_qc2_master_100mhz"; }) //
-      (build { target = "zc706"; variant = "acpki_nist_qc2_satellite"; }) //
-      (build { target = "zc706"; variant = "acpki_nist_qc2_satellite_100mhz"; }) //
-      (build { target = "kasli_soc"; variant = "demo"; json = ./demo.json; }) //
-      (build { target = "kasli_soc"; variant = "master"; json = ./kasli-soc-master.json; }) //
-      (build { target = "kasli_soc"; variant = "satellite"; json = ./kasli-soc-satellite.json; });
+      (board-package-set { target = "zc706"; variant = "nist_clock"; }) //
+      (board-package-set { target = "zc706"; variant = "nist_clock_master"; }) //
+      (board-package-set { target = "zc706"; variant = "nist_clock_master_100mhz"; }) //
+      (board-package-set { target = "zc706"; variant = "nist_clock_satellite"; }) //
+      (board-package-set { target = "zc706"; variant = "nist_clock_satellite_100mhz"; }) //
+      (board-package-set { target = "zc706"; variant = "nist_qc2"; }) //
+      (board-package-set { target = "zc706"; variant = "nist_qc2_master"; }) //
+      (board-package-set { target = "zc706"; variant = "nist_qc2_master_100mhz"; }) //
+      (board-package-set { target = "zc706"; variant = "nist_qc2_satellite"; }) //
+      (board-package-set { target = "zc706"; variant = "nist_qc2_satellite_100mhz"; }) //
+      (board-package-set { target = "zc706"; variant = "acpki_nist_clock"; }) //
+      (board-package-set { target = "zc706"; variant = "acpki_nist_clock_master"; }) //
+      (board-package-set { target = "zc706"; variant = "acpki_nist_clock_master_100mhz"; }) //
+      (board-package-set { target = "zc706"; variant = "acpki_nist_clock_satellite"; }) //
+      (board-package-set { target = "zc706"; variant = "acpki_nist_clock_satellite_100mhz"; }) //
+      (board-package-set { target = "zc706"; variant = "acpki_nist_qc2"; }) //
+      (board-package-set { target = "zc706"; variant = "acpki_nist_qc2_master"; }) //
+      (board-package-set { target = "zc706"; variant = "acpki_nist_qc2_master_100mhz"; }) //
+      (board-package-set { target = "zc706"; variant = "acpki_nist_qc2_satellite"; }) //
+      (board-package-set { target = "zc706"; variant = "acpki_nist_qc2_satellite_100mhz"; }) //
+      (board-package-set { target = "kasli_soc"; variant = "demo"; json = ./demo.json; }) //
+      (board-package-set { target = "kasli_soc"; variant = "master"; json = ./kasli-soc-master.json; }) //
+      (board-package-set { target = "kasli_soc"; variant = "satellite"; json = ./kasli-soc-satellite.json; });
 
     hydraJobs = packages.x86_64-linux // { inherit zc706-hitl-tests; inherit gateware-sim; inherit fmt-check; };
 
@@ -370,8 +371,8 @@
       name = "artiq-zynq-dev-shell";
       buildInputs = with pkgs; [
         rust
-        llvmPackages_9.llvm
-        llvmPackages_9.clang-unwrapped
+        llvmPackages_11.llvm
+        llvmPackages_11.clang-unwrapped
         gnumake
         cacert
         zynqpkgs.cargo-xbuild
@@ -384,13 +385,13 @@
         binutils-arm
       ];
       XARGO_RUST_SRC = "${rust}/lib/rustlib/src/rust/library";
-      CLANG_EXTRA_INCLUDE_DIR = "${pkgs.llvmPackages_9.clang-unwrapped.lib}/lib/clang/9.0.1/include";
+      CLANG_EXTRA_INCLUDE_DIR = "${llvmPackages_11.clang-unwrapped.lib}/lib/clang/11.1.0/include";
       ZYNQ_RS = "${zynq-rs}";
       OPENOCD_ZYNQ = "${zynq-rs}/openocd";
       SZL = "${zynqpkgs.szl}";
     };
 
-    makeArtiqZynqPackage = build;
+    makeArtiqZynqPackage = board-package-set;
 
   };
 }

src/gateware/ddmtd.py

@@ -4,51 +4,99 @@ from misoc.interconnect.csr import *
 
 
 class DDMTDSampler(Module):
-    def __init__(self, cd_ref, main_dcxo_pads):
+    def __init__(self, cd_ref, main_clk_se):
         self.ref_beating = Signal()
         self.main_beating = Signal()
 
         # # #
 
-        main_clk_se = Signal()
-        ref_beating_FF = Signal()
-        main_beating_FF = Signal()
-        self.specials += [
-            Instance("IBUFDS",
-                     i_I=main_dcxo_pads.p, i_IB=main_dcxo_pads.n,
-                     o_O=main_clk_se),
-            # Two back to back FFs are used to prevent metastability
-            Instance("FD", i_C=ClockSignal("helper"),
-                     i_D=cd_ref.clk, o_Q=ref_beating_FF),
-            Instance("FD", i_C=ClockSignal("helper"),
-                     i_D=ref_beating_FF, o_Q=self.ref_beating),
-            Instance("FD", i_C=ClockSignal("helper"),
-                     i_D=main_clk_se, o_Q=main_beating_FF),
-            Instance("FD", i_C=ClockSignal("helper"),
-                     i_D=main_beating_FF, o_Q=self.main_beating)
+        ref_clk = Signal()
+        self.specials +=[
+            # ISERDESE2 can only be driven from fabric via IDELAYE2 (see UG471)
+            Instance("IDELAYE2",
+                    p_DELAY_SRC="DATAIN",
+                    p_HIGH_PERFORMANCE_MODE="TRUE",
+                    p_REFCLK_FREQUENCY=208.3,   # REFCLK frequency from IDELAYCTRL
+                    p_IDELAY_VALUE=0,
+
+                    i_DATAIN=cd_ref.clk,
+
+                    o_DATAOUT=ref_clk
+            ),
+            Instance("ISERDESE2",
+                    p_IOBDELAY="IFD",   # use DDLY as input
+                    p_DATA_RATE="SDR",
+                    p_DATA_WIDTH=2,     # min is 2
+                    p_NUM_CE=1,
+
+                    i_DDLY=ref_clk,
+                    i_CE1=1,
+                    i_CLK=ClockSignal("helper"),
+                    i_CLKDIV=ClockSignal("helper"),
+
+                    o_Q1=self.ref_beating
+            ),
+            Instance("ISERDESE2",
+                    p_DATA_RATE="SDR",
+                    p_DATA_WIDTH=2,     # min is 2
+                    p_NUM_CE=1,
+
+                    i_D=main_clk_se,
+                    i_CE1=1,
+                    i_CLK=ClockSignal("helper"),
+                    i_CLKDIV=ClockSignal("helper"),
+
+                    o_Q1=self.main_beating,
+            ),
         ]
 
 
-class DDMTDDeglitcherFirstEdge(Module):
-    def __init__(self, input_signal, blind_period=300):
+class DDMTDDeglitcherMedianEdge(Module):
+    def __init__(self, counter, input_signal, stable_0_period=100, stable_1_period=100):
+        self.tag = Signal(len(counter))
         self.detect = Signal()
-        rising = Signal()
-        input_signal_r = Signal()
+
+        stable_0_counter = Signal(reset=stable_0_period - 1, max=stable_0_period)
+        stable_1_counter = Signal(reset=stable_1_period - 1, max=stable_1_period)
 
         # # #
 
-        self.sync.helper += [
-            input_signal_r.eq(input_signal),
-            rising.eq(input_signal & ~input_signal_r)
-        ]
-
-        blind_counter = Signal(max=blind_period)
-        self.sync.helper += [
-            If(blind_counter != 0, blind_counter.eq(blind_counter - 1)),
-            If(input_signal_r, blind_counter.eq(blind_period - 1)),
-            self.detect.eq(rising & (blind_counter == 0))
-        ]
+        # Based on CERN's median edge deglitcher FSM
+        # https://white-rabbit.web.cern.ch/documents/Precise_time_and_frequency_transfer_in_a_White_Rabbit_network.pdf (p.72)
+        fsm = ClockDomainsRenamer("helper")(FSM(reset_state="WAIT_STABLE_0"))
+        self.submodules += fsm
 
+        fsm.act("WAIT_STABLE_0",
+            If(stable_0_counter != 0,
+                NextValue(stable_0_counter, stable_0_counter - 1)
+            ).Else(
+                NextValue(stable_0_counter, stable_0_period - 1),
+                NextState("WAIT_EDGE")
+            ),
+            If(input_signal,
+                NextValue(stable_0_counter, stable_0_period - 1)
+            ),
+        )
+        fsm.act("WAIT_EDGE",
+            If(input_signal,
+                NextValue(self.tag, counter),
+                NextState("GOT_EDGE")
+            )
+        )
+        fsm.act("GOT_EDGE",
+            If(stable_1_counter != 0,
+                NextValue(stable_1_counter, stable_1_counter - 1)
+            ).Else(
+                NextValue(stable_1_counter, stable_1_period - 1),
+                self.detect.eq(1),
+                NextState("WAIT_STABLE_0")
+            ),
+            If(~input_signal,
+                NextValue(self.tag, self.tag + 1),
+                NextValue(stable_1_counter, stable_1_period - 1)
+            ),
+        )
+        
 
 class DDMTD(Module):
     def __init__(self, counter, input_signal):
@@ -59,13 +107,13 @@ class DDMTD(Module):
 
         # # #
 
-        deglitcher = DDMTDDeglitcherFirstEdge(input_signal)
+        deglitcher = DDMTDDeglitcherMedianEdge(counter, input_signal)
         self.submodules += deglitcher
 
         self.sync.helper += [
             self.h_tag_update.eq(0),
             If(deglitcher.detect,
                 self.h_tag_update.eq(1),
-                self.h_tag.eq(counter)
+                self.h_tag.eq(deglitcher.tag)
                )
-        ]
+        ]

src/gateware/drtio_aux_controller.py

@@ -1,12 +1,12 @@
 """Auxiliary controller, common to satellite and master"""
 
-from artiq.gateware.drtio.aux_controller import Transmitter, Receiver
+from artiq.gateware.drtio.aux_controller import (max_packet, aux_buffer_count,
+    Transmitter, Receiver)
 from migen.fhdl.simplify import FullMemoryWE
 from misoc.interconnect.csr import *
 from migen_axi.interconnect.sram import SRAM
 from migen_axi.interconnect import axi
 
-max_packet = 1024
 
 class _DRTIOAuxControllerBase(Module):
     def __init__(self, link_layer):
@@ -27,12 +27,12 @@ class DRTIOAuxControllerAxi(_DRTIOAuxControllerBase):
         tx_sdram_if = SRAM(self.transmitter.mem, read_only=False)
         rx_sdram_if = SRAM(self.receiver.mem, read_only=True)
         aw_decoder = axi.AddressDecoder(self.bus.aw,
-            [(lambda a: a[log2_int(max_packet)] == 0, tx_sdram_if.bus.aw),
-             (lambda a: a[log2_int(max_packet)] == 1, rx_sdram_if.bus.aw)],
+            [(lambda a: a[log2_int(max_packet*aux_buffer_count)] == 0, tx_sdram_if.bus.aw),
+             (lambda a: a[log2_int(max_packet*aux_buffer_count)] == 1, rx_sdram_if.bus.aw)],
             register=True)
         ar_decoder = axi.AddressDecoder(self.bus.ar,
-            [(lambda a: a[log2_int(max_packet)] == 0, tx_sdram_if.bus.ar),
-             (lambda a: a[log2_int(max_packet)] == 1, rx_sdram_if.bus.ar)],
+            [(lambda a: a[log2_int(max_packet*aux_buffer_count)] == 0, tx_sdram_if.bus.ar),
+             (lambda a: a[log2_int(max_packet*aux_buffer_count)] == 1, rx_sdram_if.bus.ar)],
             register=True)
         # unlike wb, axi address decoder only connects ar/aw lanes,
         # the rest must also be connected!
@@ -82,4 +82,4 @@ class DRTIOAuxControllerBare(_DRTIOAuxControllerBase):
         return self.receiver.mem.get_port(write_capable=False)
 
     def get_mem_size(self):
-        return max_packet
+        return max_packet*aux_buffer_count

src/gateware/kasli_soc.py

@@ -20,14 +20,13 @@ from artiq.gateware.drtio.transceiver import gtx_7series, eem_serdes
 from artiq.gateware.drtio.siphaser import SiPhaser7Series
 from artiq.gateware.drtio.rx_synchronizer import XilinxRXSynchronizer
 from artiq.gateware.drtio import *
+from artiq.gateware.wrpll import wrpll
 
 import dma
 import analyzer
 import acpki
 import drtio_aux_controller
 import zynq_clocking
-import wrpll
-import si549
 from config import write_csr_file, write_mem_file, write_rustc_cfg_file
 
 eem_iostandard_dict = {
@@ -107,9 +106,10 @@ class GTPBootstrapClock(Module):
 
 
 class GenericStandalone(SoCCore):
-    def __init__(self, description, acpki=False, with_wrpll=False):
+    def __init__(self, description, acpki=False):
         self.acpki = acpki
         clk_freq = description["rtio_frequency"]
+        with_wrpll = description["enable_wrpll"]
 
         platform = kasli_soc.Platform()
         platform.toolchain.bitstream_commands.extend([
@@ -121,13 +121,6 @@ class GenericStandalone(SoCCore):
         SoCCore.__init__(self, platform=platform, csr_data_width=32, ident=ident, ps_cd_sys=False)
 
         self.config["HW_REV"] = description["hw_rev"]
-        
-
-        self.submodules += SMAClkinForward(self.platform)
-
-        self.config["HAS_SI5324"] = None
-        self.config["SI5324_SOFT_RESET"] = None
-
         clk_synth = platform.request("cdr_clk_clean_fabric")
         clk_synth_se = Signal()
         clk_synth_se_buf = Signal()
@@ -142,6 +135,7 @@ class GenericStandalone(SoCCore):
         ]
         fix_serdes_timing_path(platform)
         self.submodules.bootstrap = GTPBootstrapClock(self.platform, clk_freq)
+        self.config["RTIO_FREQUENCY"] = str(clk_freq/1e6)
         self.config["CLOCK_FREQUENCY"] = int(clk_freq)
 
         self.submodules.sys_crg = zynq_clocking.SYSCRG(self.platform, self.ps7, clk_synth_se_buf)
@@ -151,6 +145,23 @@ class GenericStandalone(SoCCore):
         self.crg = self.ps7 # HACK for eem_7series to find the clock
         self.crg.cd_sys = self.sys_crg.cd_sys
 
+        if with_wrpll:
+            self.submodules.wrpll_refclk = wrpll.FrequencyMultiplier(platform.request("sma_clkin"))
+            self.submodules.wrpll = wrpll.WRPLL(
+                platform=self.platform,
+                cd_ref=self.wrpll_refclk.cd_ref,
+                main_clk_se=clk_synth_se)
+            self.csr_devices.append("wrpll_refclk")
+            self.csr_devices.append("wrpll")
+            self.comb += self.ps7.core.core0.nfiq.eq(self.wrpll.ev.irq)
+            self.config["HAS_SI549"] = None
+            self.config["WRPLL_REF_CLK"] = "SMA_CLKIN"
+        else:
+            self.submodules += SMAClkinForward(self.platform)
+            self.config["HAS_SI5324"] = None
+            self.config["SI5324_SOFT_RESET"] = None
+
+
         self.rtio_channels = []
         has_grabber = any(peripheral["type"] == "grabber" for peripheral in description["peripherals"])
         if has_grabber:
@@ -207,8 +218,9 @@ class GenericStandalone(SoCCore):
 
 
 class GenericMaster(SoCCore):
-    def __init__(self, description, acpki=False, with_wrpll=False):
+    def __init__(self, description, acpki=False):
         clk_freq = description["rtio_frequency"]
+        with_wrpll = description["enable_wrpll"]
 
         has_drtio_over_eem = any(peripheral["type"] == "shuttler" for peripheral in description["peripherals"])
         self.acpki = acpki
@@ -259,16 +271,16 @@ class GenericMaster(SoCCore):
         self.specials += MultiReg(self.sys_crg.clk_sw_fsm.o_clk_sw & self.sys_crg.mmcm_locked, self.gt_drtio.clk_path_ready, odomain="bootstrap")
 
         if with_wrpll:
-            self.submodules.wrpll_refclk = wrpll.SMAFrequencyMultiplier(platform.request("sma_clkin"))
-            self.submodules.main_dcxo = si549.Si549(platform.request("ddmtd_main_dcxo_i2c"))
-            self.submodules.helper_dcxo = si549.Si549(platform.request("ddmtd_helper_dcxo_i2c"))
+            clk_synth = platform.request("cdr_clk_clean_fabric")
+            clk_synth_se = Signal()
+            platform.add_period_constraint(clk_synth.p, 8.0)
+            self.specials += Instance("IBUFGDS", p_DIFF_TERM="TRUE", p_IBUF_LOW_PWR="FALSE", i_I=clk_synth.p, i_IB=clk_synth.n, o_O=clk_synth_se)
+            self.submodules.wrpll_refclk = wrpll.FrequencyMultiplier(platform.request("sma_clkin"))
             self.submodules.wrpll = wrpll.WRPLL(
+                platform=self.platform,
                 cd_ref=self.wrpll_refclk.cd_ref,
-                main_dcxo_pads=platform.request("cdr_clk_clean_fabric"),
-                helper_dcxo_pads=platform.request("ddmtd_helper_clk"))
+                main_clk_se=clk_synth_se)
             self.csr_devices.append("wrpll_refclk")
-            self.csr_devices.append("main_dcxo")
-            self.csr_devices.append("helper_dcxo")
             self.csr_devices.append("wrpll")
             self.comb += self.ps7.core.core0.nfiq.eq(self.wrpll.ev.irq)
             self.config["HAS_SI549"] = None
@@ -416,8 +428,9 @@ class GenericMaster(SoCCore):
 
 
 class GenericSatellite(SoCCore):
-    def __init__(self, description, acpki=False, with_wrpll=False):
+    def __init__(self, description, acpki=False):
         clk_freq = description["rtio_frequency"]
+        with_wrpll = description["enable_wrpll"]
 
         self.acpki = acpki
 
@@ -570,18 +583,20 @@ class GenericSatellite(SoCCore):
         self.config["CLOCK_FREQUENCY"] = int(clk_freq)
 
         if with_wrpll:
-            self.submodules.main_dcxo = si549.Si549(platform.request("ddmtd_main_dcxo_i2c"))
-            self.submodules.helper_dcxo = si549.Si549(platform.request("ddmtd_helper_dcxo_i2c"))
+            clk_synth = platform.request("cdr_clk_clean_fabric")
+            clk_synth_se = Signal()
+            platform.add_period_constraint(clk_synth.p, 8.0)
+            self.specials += Instance("IBUFGDS", p_DIFF_TERM="TRUE", p_IBUF_LOW_PWR="FALSE", i_I=clk_synth.p, i_IB=clk_synth.n, o_O=clk_synth_se)
             self.submodules.wrpll = wrpll.WRPLL(
+                platform=self.platform,
                 cd_ref=self.gt_drtio.cd_rtio_rx0,
-                main_dcxo_pads=platform.request("cdr_clk_clean_fabric"),
-                helper_dcxo_pads=platform.request("ddmtd_helper_clk"))
-            self.csr_devices.append("main_dcxo")
-            self.csr_devices.append("helper_dcxo")
+                main_clk_se=clk_synth_se)
+            self.submodules.wrpll_skewtester = wrpll.SkewTester(self.rx_synchronizer)
+            self.csr_devices.append("wrpll_skewtester")
             self.csr_devices.append("wrpll")
             self.comb += self.ps7.core.core0.nfiq.eq(self.wrpll.ev.irq)
             self.config["HAS_SI549"] = None
-            self.config["WRPLL_REF_CLK"] = "GTX_CDR"
+            self.config["WRPLL_REF_CLK"] = "GT_CDR"
         else:
             self.submodules.siphaser = SiPhaser7Series(
                 si5324_clkin=platform.request("cdr_clk"),
@@ -620,8 +635,6 @@ def main():
         help="build gateware into the specified directory")
     parser.add_argument("--acpki", default=False, action="store_true",
         help="enable ACPKI")
-    parser.add_argument("--with-wrpll", default=True, action="store_true",
-        help="enable WRPLL")
     parser.add_argument("description", metavar="DESCRIPTION",
                         help="JSON system description file")
     args = parser.parse_args()
@@ -639,7 +652,7 @@ def main():
     else:
         raise ValueError("Invalid DRTIO role")
 
-    soc = cls(description, acpki=args.acpki, with_wrpll=args.with_wrpll)
+    soc = cls(description, acpki=args.acpki)
     soc.finalize()
 
     if args.r is not None:

src/gateware/si549.py

@@ -1,278 +0,0 @@
-from migen import *
-from migen.genlib.fsm import *
-
-from misoc.interconnect.csr import *
-
-
-class I2CClockGen(Module):
-    def __init__(self, width):
-        self.load = Signal(width)
-        self.clk2x = Signal()
-
-        cnt = Signal.like(self.load)
-        self.comb += [
-            self.clk2x.eq(cnt == 0),
-        ]
-        self.sync += [
-            If(self.clk2x,
-                cnt.eq(self.load),
-            ).Else(
-                cnt.eq(cnt - 1),
-            )
-        ]
-
-
-class I2CMasterMachine(Module):
-    def __init__(self, clock_width):
-        self.scl = Signal(reset=1)
-        self.sda_o = Signal(reset=1)
-        self.sda_i = Signal()
-
-        self.submodules.cg = CEInserter()(I2CClockGen(clock_width))
-        self.start = Signal()
-        self.stop = Signal()
-        self.write = Signal()
-        self.ack = Signal()
-        self.data = Signal(8)
-        self.ready = Signal()
-
-        # # #
-
-        bits = Signal(4)
-        data = Signal(8)
-
-        fsm = CEInserter()(FSM("IDLE"))
-        self.submodules += fsm
-
-        fsm.act("IDLE",
-            self.ready.eq(1),
-            If(self.start,
-                NextState("START0"),
-            ).Elif(self.stop,
-                NextState("STOP0"),
-            ).Elif(self.write,
-                NextValue(bits, 8),
-                NextValue(data, self.data),
-                NextState("WRITE0")
-            )
-        )
-
-        fsm.act("START0",
-            NextValue(self.scl, 1),
-            NextState("START1")
-        )
-        fsm.act("START1",
-            NextValue(self.sda_o, 0),
-            NextState("IDLE")
-        )
-
-        fsm.act("STOP0",
-            NextValue(self.scl, 0),
-            NextState("STOP1")
-        )
-        fsm.act("STOP1",
-            NextValue(self.sda_o, 0),
-            NextState("STOP2")
-        )
-        fsm.act("STOP2",
-            NextValue(self.scl, 1), 
-            NextState("STOP3")
-        )
-        fsm.act("STOP3",
-            NextValue(self.sda_o, 1),
-            NextState("IDLE")
-        )
-
-        fsm.act("WRITE0",
-            NextValue(self.scl, 0),
-            NextState("WRITE1")
-        )
-        fsm.act("WRITE1",
-            If(bits == 0,
-                NextValue(self.sda_o, 1),
-                NextState("READACK0"),
-            ).Else(
-                NextValue(self.sda_o, data[7]),
-                NextState("WRITE2"),
-            )
-        )
-        fsm.act("WRITE2",
-            NextValue(self.scl, 1),
-            NextValue(data[1:], data[:-1]),
-            NextValue(bits, bits - 1),
-            NextState("WRITE0"),
-        )
-        fsm.act("READACK0",
-            NextValue(self.scl, 1),
-            NextState("READACK1"),
-        )
-        fsm.act("READACK1",
-            NextValue(self.ack, ~self.sda_i),
-            NextState("IDLE")
-        )
-
-        run = Signal()
-        idle = Signal()
-        self.comb += [
-            run.eq((self.start | self.stop | self.write) & self.ready),
-            idle.eq(~run & fsm.ongoing("IDLE")),
-            self.cg.ce.eq(~idle),
-            fsm.ce.eq(run | self.cg.clk2x),
-        ]
-
-
-class ADPLLProgrammer(Module):
-    def __init__(self):
-        self.i2c_divider = Signal(16)
-        self.i2c_address = Signal(7)
-
-        self.adpll = Signal(24)
-        self.stb = Signal()
-        self.busy = Signal()
-        self.nack = Signal()
-
-        self.scl = Signal()
-        self.sda_i = Signal()
-        self.sda_o = Signal()
-
-        # # #
-
-        master = I2CMasterMachine(16)
-        self.submodules += master
-
-        self.comb += [
-            master.cg.load.eq(self.i2c_divider),
-            self.scl.eq(master.scl),
-            master.sda_i.eq(self.sda_i),
-            self.sda_o.eq(master.sda_o)
-        ]
-
-        fsm = FSM()
-        self.submodules += fsm
-
-        adpll = Signal.like(self.adpll)
-
-        fsm.act("IDLE",
-            If(self.stb,
-                NextValue(adpll, self.adpll),
-                NextState("START")
-            )
-        )
-        fsm.act("START",
-            master.start.eq(1),
-            If(master.ready, NextState("DEVADDRESS"))
-        )
-        fsm.act("DEVADDRESS",
-            master.data.eq(self.i2c_address << 1),
-            master.write.eq(1),
-            If(master.ready, NextState("REGADRESS"))
-        )
-        fsm.act("REGADRESS",
-            master.data.eq(231),
-            master.write.eq(1),
-            If(master.ready,
-                If(master.ack,
-                    NextState("DATA0")
-                ).Else(
-                    self.nack.eq(1),
-                    NextState("STOP")
-                )
-            )
-        )
-        fsm.act("DATA0",
-            master.data.eq(adpll[0:8]),
-            master.write.eq(1),
-            If(master.ready,
-                If(master.ack,
-                    NextState("DATA1")
-                ).Else(
-                    self.nack.eq(1),
-                    NextState("STOP")
-                )
-            )
-        )
-        fsm.act("DATA1",
-            master.data.eq(adpll[8:16]),
-            master.write.eq(1),
-            If(master.ready,
-                If(master.ack,
-                    NextState("DATA2")
-                ).Else(
-                    self.nack.eq(1),
-                    NextState("STOP")
-                )
-            )
-        )
-        fsm.act("DATA2",
-            master.data.eq(adpll[16:24]),
-            master.write.eq(1),
-            If(master.ready,
-                If(~master.ack, self.nack.eq(1)),
-                NextState("STOP")
-            )
-        )
-        fsm.act("STOP",
-            master.stop.eq(1),
-            If(master.ready,
-                If(~master.ack, self.nack.eq(1)),
-                NextState("IDLE")
-            )
-        )
-
-        self.comb += self.busy.eq(~fsm.ongoing("IDLE"))
-
-
-class Si549(Module, AutoCSR):
-    def __init__(self, pads):
-        self.i2c_divider = CSRStorage(16, reset=75)
-        self.i2c_address = CSRStorage(7)
-
-        self.adpll = CSRStorage(24)
-        self.adpll_stb = CSRStorage()
-        self.adpll_busy = CSRStatus()
-        self.nack = CSRStatus()
-
-        self.bitbang_enable = CSRStorage()
-
-        self.sda_oe = CSRStorage()
-        self.sda_out = CSRStorage()
-        self.sda_in = CSRStatus()
-        self.scl_oe = CSRStorage()
-        self.scl_out = CSRStorage()
-
-        # # #
-
-        self.submodules.programmer = ADPLLProgrammer()
-
-        self.comb += [
-            self.programmer.i2c_divider.eq(self.i2c_divider.storage),
-            self.programmer.i2c_address.eq(self.i2c_address.storage),
-            self.programmer.adpll.eq(self.adpll.storage),
-            self.programmer.stb.eq(self.adpll_stb.storage),
-            self.adpll_busy.status.eq(self.programmer.busy),
-            self.nack.status.eq(self.programmer.nack)
-        ]
-
-        # I2C with bitbang/gateware mode select
-        sda_t = TSTriple(1)
-        scl_t = TSTriple(1)
-        self.specials += [
-            sda_t.get_tristate(pads.sda),
-            scl_t.get_tristate(pads.scl)
-        ]
-
-        self.comb += [
-            If(self.bitbang_enable.storage,
-                sda_t.oe.eq(self.sda_oe.storage),
-                sda_t.o.eq(self.sda_out.storage),
-                self.sda_in.status.eq(sda_t.i),
-                scl_t.oe.eq(self.scl_oe.storage),
-                scl_t.o.eq(self.scl_out.storage)
-            ).Else(
-                sda_t.oe.eq(~self.programmer.sda_o),
-                sda_t.o.eq(0),
-                self.programmer.sda_i.eq(sda_t.i),
-                scl_t.oe.eq(~self.programmer.scl),
-                scl_t.o.eq(0),
-            )
-        ]

src/gateware/wrpll.py

@@ -1,185 +0,0 @@
-from migen import *
-from migen.genlib.cdc import MultiReg, AsyncResetSynchronizer, PulseSynchronizer
-from misoc.interconnect.csr import *
-from misoc.interconnect.csr_eventmanager import *
-
-from ddmtd import DDMTDSampler, DDMTD
-
-
-class FrequencyCounter(Module, AutoCSR):
-    def __init__(self, domains, counter_width=24):
-        for domain in domains:
-            name = "counter_" + domain
-            counter = CSRStatus(counter_width, name=name)
-            setattr(self, name, counter)
-        self.update_en = CSRStorage()
-
-        timer = Signal(counter_width)
-        timer_tick = Signal()
-        self.sync += Cat(timer, timer_tick).eq(timer + 1)
-
-        for domain in domains:
-            sync_domain = getattr(self.sync, domain)
-            divider = Signal(2)
-            sync_domain += divider.eq(divider + 1)
-
-            divided = Signal()
-            sync_domain += divided.eq(divider[-1])
-            divided_sys = Signal()
-            self.specials += MultiReg(divided, divided_sys)
-
-            divided_sys_r = Signal()
-            divided_tick = Signal()
-            self.sync += divided_sys_r.eq(divided_sys)
-            self.comb += divided_tick.eq(divided_sys & ~divided_sys_r)
-
-            counter = Signal(counter_width)
-            counter_csr = getattr(self, "counter_" + domain)
-            self.sync += [
-                If(timer_tick,
-                    If(self.update_en.storage, counter_csr.status.eq(counter)),
-                    counter.eq(0),
-                ).Else(
-                    If(divided_tick, counter.eq(counter + 1))
-                )
-            ]
-
-
-class WRPLL(Module, AutoCSR):
-    def __init__(self, cd_ref, main_dcxo_pads, helper_dcxo_pads, COUNTER_BIT=32):
-
-        self.ref_tag = CSRStatus(COUNTER_BIT)
-        self.main_tag = CSRStatus(COUNTER_BIT)
-
-        ddmtd_counter = Signal(COUNTER_BIT)
-
-        ref_tag_sys = Signal(COUNTER_BIT)
-        main_tag_sys = Signal(COUNTER_BIT)
-        ref_tag_stb_sys = Signal()
-        main_tag_stb_sys = Signal()
-
-        # # #
-
-        self.helper_reset = CSRStorage()
-        self.clock_domains.cd_helper = ClockDomain()
-        self.specials += [
-            Instance("IBUFGDS",
-                     i_I=helper_dcxo_pads.p, i_IB=helper_dcxo_pads.n,
-                     o_O=self.cd_helper.clk),
-            AsyncResetSynchronizer(self.cd_helper, self.helper_reset.storage)
-        ]
-
-        self.submodules.frequency_counter = FrequencyCounter(["sys", cd_ref.name])
-
-        self.submodules.ddmtd_sampler = DDMTDSampler(cd_ref, main_dcxo_pads)
-
-        self.sync.helper += ddmtd_counter.eq(ddmtd_counter + 1)
-        self.submodules.ddmtd_ref = DDMTD(ddmtd_counter, self.ddmtd_sampler.ref_beating)
-        self.submodules.ddmtd_main = DDMTD(ddmtd_counter, self.ddmtd_sampler.main_beating)
-
-        # DDMTD tags collection
-
-        self.specials += [
-            MultiReg(self.ddmtd_ref.h_tag, ref_tag_sys),
-            MultiReg(self.ddmtd_main.h_tag, main_tag_sys)
-        ]
-
-        ref_tag_stb_ps = PulseSynchronizer("helper", "sys")
-        main_tag_stb_ps = PulseSynchronizer("helper", "sys")
-        self.submodules += [
-            ref_tag_stb_ps,
-            main_tag_stb_ps
-        ]
-        self.sync.helper += [
-            ref_tag_stb_ps.i.eq(self.ddmtd_ref.h_tag_update),
-            main_tag_stb_ps.i.eq(self.ddmtd_main.h_tag_update)
-        ]
-        self.sync += [
-            ref_tag_stb_sys.eq(ref_tag_stb_ps.o),
-            main_tag_stb_sys.eq(main_tag_stb_ps.o)
-        ]
-
-        self.sync += [
-            If(ref_tag_stb_sys,
-                self.ref_tag.status.eq(ref_tag_sys),
-            ),
-            If(main_tag_stb_sys,
-                self.main_tag.status.eq(main_tag_sys)
-            )
-        ]
-
-        # PL->PS interrupt
-
-        self.submodules.ref_tag_ev = EventManager()
-        self.ref_tag_ev.stb = EventSourcePulse()
-        self.ref_tag_ev.finalize()
-
-        self.submodules.main_tag_ev = EventManager()
-        self.main_tag_ev.stb = EventSourcePulse()
-        self.main_tag_ev.finalize()
-
-        self.sync += [
-            self.ref_tag_ev.stb.trigger.eq(ref_tag_stb_sys),
-            self.main_tag_ev.stb.trigger.eq(main_tag_stb_sys)
-        ]
-
-        self.submodules.ev = SharedIRQ(self.ref_tag_ev, self.main_tag_ev)
-
-
-class SMAFrequencyMultiplier(Module, AutoCSR):
-    def __init__(self, sma_clkin):
-        freq = 125e6
-        period = 1e9/freq # ns
-
-        sma_clkin_se = Signal()
-        mmcm_locked = Signal()
-        mmcm_fb_clk = Signal()
-        ref_clk = Signal()
-        self.clock_domains.cd_ref = ClockDomain()
-
-        self.mmcm_locked = CSRStatus()
-        self.mmcm_reset = CSRStorage()
-
-        self.mmcm_daddr = CSRStorage(7)
-        self.mmcm_din = CSRStorage(16)
-        self.mmcm_dwen = CSRStorage()
-        self.mmcm_den = CSRStorage()
-        self.mmcm_dclk = CSRStorage()
-        self.mmcm_dout = CSRStatus(16)
-        self.mmcm_dready = CSRStatus()
-
-        # # #
-
-        self.specials += [
-            Instance("IBUFDS",
-                i_I=sma_clkin.p, i_IB=sma_clkin.n,
-                o_O=sma_clkin_se),
-            # MMCME2 is capable to accept 10Mhz input while PLLE2 only support down to 19Mhz input (DS191)
-            Instance("MMCME2_ADV",
-                p_BANDWIDTH="LOW",  # lower jitter
-                o_LOCKED=self.mmcm_locked.status,
-                i_RST=self.mmcm_reset.storage,
-                
-                p_CLKIN1_PERIOD=period,
-                i_CLKIN1=sma_clkin_se,
-                i_CLKINSEL=1,  # 1=CLKIN1 0=CLKIN2
-
-                # VCO @ 1.25Ghz
-                p_CLKFBOUT_MULT_F=10, p_DIVCLK_DIVIDE=1,
-                i_CLKFBIN=mmcm_fb_clk, o_CLKFBOUT=mmcm_fb_clk, 
-
-                # 125Mhz for WRPLL
-                p_CLKOUT0_DIVIDE_F=10, p_CLKOUT0_PHASE=0.0, o_CLKOUT0=ref_clk,
-
-                # Dynamic Reconfiguration Port
-                i_DADDR = self.mmcm_daddr.storage,
-                i_DI = self.mmcm_din.storage,
-                i_DWE = self.mmcm_dwen.storage,
-                i_DEN = self.mmcm_den.storage,
-                i_DCLK = self.mmcm_dclk.storage,
-                o_DO = self.mmcm_dout.status,
-                o_DRDY = self.mmcm_dready.status
-            ),
-            Instance("BUFG", i_I=ref_clk, o_O=self.cd_ref.clk),
-            AsyncResetSynchronizer(self.cd_ref, ~self.mmcm_locked.status),
-        ]

src/libboard_artiq/Cargo.toml.tpl

@@ -10,6 +10,7 @@ name = "libboard_artiq"
 [features]
 target_zc706 = ["libboard_zynq/target_zc706", "libconfig/target_zc706"]
 target_kasli_soc = ["libboard_zynq/target_kasli_soc", "libconfig/target_kasli_soc"]
+calibrate_wrpll_skew = []
 
 [build-dependencies]
 build_zynq = { path = "../libbuild_zynq" }

src/libboard_artiq/src/drtioaux.rs

@@ -1,9 +1,10 @@
+use core::slice;
+
 use core_io::{Error as IoError, ErrorKind as IoErrorKind};
 use crc;
 use io::{proto::{ProtoRead, ProtoWrite},
          Cursor};
 use libboard_zynq::{time::Milliseconds, timer::GlobalTimer};
-use libcortex_a9::asm::dmb;
 
 pub use crate::drtioaux_proto::Packet;
 use crate::{drtioaux_proto::Error as ProtocolError, mem::mem::DRTIOAUX_MEM, pl::csr::DRTIOAUX};
@@ -56,30 +57,14 @@ pub fn has_rx_error(linkno: u8) -> bool {
     }
 }
 
-pub fn copy_work_buffer(src: *mut u32, dst: *mut u32, len: isize) {
-    // AXI writes must be 4-byte aligned (drtio proto doesn't care for that),
-    // and AXI burst reads/writes are not implemented yet in gateware
-    // thus the need for a work buffer for transmitting and copying it over
-    unsafe {
-        for i in 0..(len / 4) {
-            *dst.offset(i) = *src.offset(i);
-            //data memory barrier to prevent bursts
-            dmb();
-        }
-    }
-}
-
 fn receive<F, T>(linkno: u8, f: F) -> Result<Option<T>, Error>
 where F: FnOnce(&[u8]) -> Result<T, Error> {
     let linkidx = linkno as usize;
     unsafe {
         if (DRTIOAUX[linkidx].aux_rx_present_read)() == 1 {
-            let ptr = (DRTIOAUX_MEM[linkidx].base + DRTIOAUX_MEM[linkidx].size / 2) as *mut u32;
-            let len = (DRTIOAUX[linkidx].aux_rx_length_read)() as usize;
-            // work buffer to accomodate axi burst reads
-            let mut buf: [u8; 1024] = [0; 1024];
-            copy_work_buffer(ptr, buf.as_mut_ptr() as *mut u32, len as isize);
-            let result = f(&buf[0..len]);
+            let read_ptr = (DRTIOAUX[linkidx].aux_read_pointer_read)() as usize;
+            let ptr = (DRTIOAUX_MEM[linkidx].base + DRTIOAUX_MEM[linkidx].size / 2 + read_ptr * 0x400) as *mut u32;
+            let result = f(slice::from_raw_parts(ptr as *mut u8, 0x400 as usize));
             (DRTIOAUX[linkidx].aux_rx_present_write)(1);
             Ok(Some(result?))
         } else {
@@ -100,15 +85,15 @@ pub fn recv(linkno: u8) -> Result<Option<Packet>, Error> {
 
         let mut reader = Cursor::new(buffer);
 
-        let checksum_at = buffer.len() - 4;
+        let packet = Packet::read_from(&mut reader)?;
+        let padding = (12 - (reader.position() % 8)) % 8;
+        let checksum_at = reader.position() + padding;
         let checksum = crc::crc32::checksum_ieee(&reader.get_ref()[0..checksum_at]);
         reader.set_position(checksum_at);
         if reader.read_u32()? != checksum {
             return Err(Error::CorruptedPacket);
         }
-        reader.set_position(0);
-
-        Ok(Packet::read_from(&mut reader)?)
+        Ok(packet)
     })
 }
 
@@ -130,11 +115,7 @@ where F: FnOnce(&mut [u8]) -> Result<usize, Error> {
     unsafe {
         while (DRTIOAUX[linkno].aux_tx_read)() != 0 {}
         let ptr = DRTIOAUX_MEM[linkno].base as *mut u32;
-        let len = DRTIOAUX_MEM[linkno].size / 2;
-        // work buffer, works with unaligned mem access
-        let mut buf: [u8; 1024] = [0; 1024];
-        let len = f(&mut buf[0..len])?;
-        copy_work_buffer(buf.as_mut_ptr() as *mut u32, ptr, len as isize);
+        let len = f(slice::from_raw_parts_mut(ptr as *mut u8, 0x400 as usize))?;
         (DRTIOAUX[linkno].aux_tx_length_write)(len as u16);
         (DRTIOAUX[linkno].aux_tx_write)(1);
         Ok(())

src/libboard_artiq/src/drtioaux_async.rs

@@ -1,3 +1,5 @@
+use core::slice;
+
 use core_io::{Error as IoError, ErrorKind as IoErrorKind};
 use crc;
 use io::{proto::{ProtoRead, ProtoWrite},
@@ -8,7 +10,7 @@ use nb;
 use void::Void;
 
 pub use crate::drtioaux_proto::Packet;
-use crate::{drtioaux::{copy_work_buffer, has_rx_error, Error},
+use crate::{drtioaux::{has_rx_error, Error},
             mem::mem::DRTIOAUX_MEM,
             pl::csr::DRTIOAUX};
 
@@ -38,12 +40,9 @@ where F: FnOnce(&[u8]) -> Result<T, Error> {
     let linkidx = linkno as usize;
     unsafe {
         if (DRTIOAUX[linkidx].aux_rx_present_read)() == 1 {
-            let ptr = (DRTIOAUX_MEM[linkidx].base + DRTIOAUX_MEM[linkidx].size / 2) as *mut u32;
-            let len = (DRTIOAUX[linkidx].aux_rx_length_read)() as usize;
-            // work buffer to accomodate axi burst reads
-            let mut buf: [u8; 1024] = [0; 1024];
-            copy_work_buffer(ptr, buf.as_mut_ptr() as *mut u32, len as isize);
-            let result = f(&buf[0..len]);
+            let read_ptr = (DRTIOAUX[linkidx].aux_read_pointer_read)() as usize;
+            let ptr = (DRTIOAUX_MEM[linkidx].base + DRTIOAUX_MEM[linkidx].size / 2 + read_ptr * 0x400) as *mut u32;
+            let result = f(slice::from_raw_parts(ptr as *mut u8, 0x400 as usize));
             (DRTIOAUX[linkidx].aux_rx_present_write)(1);
             Ok(Some(result?))
         } else {
@@ -64,15 +63,15 @@ pub async fn recv(linkno: u8) -> Result<Option<Packet>, Error> {
 
         let mut reader = Cursor::new(buffer);
 
-        let checksum_at = buffer.len() - 4;
+        let packet = Packet::read_from(&mut reader)?;
+        let padding = (12 - (reader.position() % 8)) % 8;
+        let checksum_at = reader.position() + padding;
         let checksum = crc::crc32::checksum_ieee(&reader.get_ref()[0..checksum_at]);
         reader.set_position(checksum_at);
         if reader.read_u32()? != checksum {
             return Err(Error::CorruptedPacket);
         }
-        reader.set_position(0);
-
-        Ok(Packet::read_from(&mut reader)?)
+        Ok(packet)
     })
     .await
 }
@@ -103,11 +102,7 @@ where F: FnOnce(&mut [u8]) -> Result<usize, Error> {
     unsafe {
         let _ = block_async!(tx_ready(linkno)).await;
         let ptr = DRTIOAUX_MEM[linkno].base as *mut u32;
-        let len = DRTIOAUX_MEM[linkno].size / 2;
-        // work buffer, works with unaligned mem access
-        let mut buf: [u8; 1024] = [0; 1024];
-        let len = f(&mut buf[0..len])?;
-        copy_work_buffer(buf.as_mut_ptr() as *mut u32, ptr, len as isize);
+        let len = f(slice::from_raw_parts_mut(ptr as *mut u8, 0x400 as usize))?;
         (DRTIOAUX[linkno].aux_tx_length_write)(len as u16);
         (DRTIOAUX[linkno].aux_tx_write)(1);
         Ok(())

src/libboard_artiq/src/drtioaux_proto.rs

@@ -1,9 +1,11 @@
 use core_io::{Error as IoError, Read, Write};
 use io::proto::{ProtoRead, ProtoWrite};
 
+const MAX_PACKET: usize = 1024;
+
 // maximum size of arbitrary payloads
 // used by satellite -> master analyzer, subkernel exceptions
-pub const SAT_PAYLOAD_MAX_SIZE: usize  = /*max size*/512 - /*CRC*/4 - /*packet ID*/1 - /*last*/1 - /*length*/2;
+pub const SAT_PAYLOAD_MAX_SIZE: usize  = /*max size*/MAX_PACKET - /*CRC*/4 - /*packet ID*/1 - /*last*/1 - /*length*/2;
 // used by DDMA, subkernel program data (need to provide extra ID and destination)
 pub const MASTER_PAYLOAD_MAX_SIZE: usize = SAT_PAYLOAD_MAX_SIZE - /*source*/1 - /*destination*/1 - /*ID*/4;
 
@@ -89,8 +91,6 @@ pub enum Packet {
     RoutingSetRank {
         rank: u8,
     },
-    RoutingRetrievePackets,
-    RoutingNoPackets,
     RoutingAck,
 
     MonitorRequest {
@@ -325,8 +325,6 @@ impl Packet {
                 rank: reader.read_u8()?,
             },
             0x32 => Packet::RoutingAck,
-            0x33 => Packet::RoutingRetrievePackets,
-            0x34 => Packet::RoutingNoPackets,
 
             0x40 => Packet::MonitorRequest {
                 destination: reader.read_u8()?,
@@ -602,8 +600,6 @@ impl Packet {
                 writer.write_u8(rank)?;
             }
             Packet::RoutingAck => writer.write_u8(0x32)?,
-            Packet::RoutingRetrievePackets => writer.write_u8(0x33)?,
-            Packet::RoutingNoPackets => writer.write_u8(0x34)?,
 
             Packet::MonitorRequest {
                 destination,

src/libboard_artiq/src/fiq.rs

@@ -1,15 +1,15 @@
-#[cfg(has_wrpll)]
-use libboard_artiq::si549;
 use libboard_zynq::{println, stdio};
 use libcortex_a9::{interrupt_handler, regs::MPIDR};
 use libregister::RegisterR;
 
+#[cfg(has_si549)]
+use crate::si549;
+
 interrupt_handler!(FIQ, fiq, __irq_stack0_start, __irq_stack1_start, {
-    // println!("hello?");
     match MPIDR.read().cpu_id() {
         0 => {
             // nFIQ is driven directly and bypass GIC
-            #[cfg(has_wrpll)]
+            #[cfg(has_si549)]
             si549::wrpll::interrupt_handler();
             return;
         }

src/libboard_artiq/src/io_expander.rs

@@ -1,6 +1,7 @@
 use libboard_zynq::i2c;
 use log::info;
 
+#[cfg(has_virtual_leds)]
 use crate::pl::csr;
 
 // Only the bare minimum registers. Bits/IO connections equivalent between IC types.
@@ -37,6 +38,7 @@ const IODIR1: [u8; 2] = [
 
 pub struct IoExpander {
     address: u8,
+    #[cfg(has_virtual_leds)]
     virtual_led_mapping: &'static [(u8, u8, u8)],
     iodir: [u8; 2],
     out_current: [u8; 2],
@@ -46,17 +48,18 @@ pub struct IoExpander {
 
 impl IoExpander {
     pub fn new(i2c: &mut i2c::I2c, index: u8) -> Result<Self, &'static str> {
-        #[cfg(hw_rev = "v1.0")]
+        #[cfg(all(hw_rev = "v1.0", has_virtual_leds))]
         const VIRTUAL_LED_MAPPING0: [(u8, u8, u8); 2] = [(0, 0, 6), (1, 1, 6)];
-        #[cfg(hw_rev = "v1.1")]
+        #[cfg(all(hw_rev = "v1.1", has_virtual_leds))]
         const VIRTUAL_LED_MAPPING0: [(u8, u8, u8); 2] = [(0, 0, 7), (1, 1, 6)];
-
+        #[cfg(has_virtual_leds)]
         const VIRTUAL_LED_MAPPING1: [(u8, u8, u8); 2] = [(2, 0, 6), (3, 1, 6)];
 
         // Both expanders on SHARED I2C bus
         let mut io_expander = match index {
             0 => IoExpander {
                 address: 0x40,
+                #[cfg(has_virtual_leds)]
                 virtual_led_mapping: &VIRTUAL_LED_MAPPING0,
                 iodir: IODIR0,
                 out_current: [0; 2],
@@ -70,6 +73,7 @@ impl IoExpander {
             },
             1 => IoExpander {
                 address: 0x42,
+                #[cfg(has_virtual_leds)]
                 virtual_led_mapping: &VIRTUAL_LED_MAPPING1,
                 iodir: IODIR1,
                 out_current: [0; 2],

src/libboard_artiq/src/lib.rs

@@ -1,5 +1,7 @@
 #![no_std]
 #![feature(never_type)]
+#![feature(naked_functions)]
+#![feature(asm)]
 
 extern crate core_io;
 extern crate crc;
@@ -19,7 +21,8 @@ pub mod drtioaux;
 #[cfg(has_drtio)]
 pub mod drtioaux_async;
 pub mod drtioaux_proto;
-#[cfg(all(feature = "target_kasli_soc", has_drtio))]
+pub mod fiq;
+#[cfg(feature = "target_kasli_soc")]
 pub mod io_expander;
 pub mod logger;
 #[cfg(has_drtio)]

src/libboard_artiq/src/si549.rs

@@ -7,17 +7,16 @@ use crate::pl::csr;
 #[cfg(feature = "target_kasli_soc")]
 const ADDRESS: u8 = 0x67;
 
-#[derive(Clone, Copy)]
+const ADPLL_MAX: i32 = (950.0 / 0.0001164) as i32;
+
 pub struct DividerConfig {
     pub hsdiv: u16,
     pub lsdiv: u8,
     pub fbdiv: u64,
 }
 
-#[derive(Clone, Copy)]
 pub struct FrequencySetting {
     pub main: DividerConfig,
-    #[cfg(has_wrpll)]
     pub helper: DividerConfig,
 }
 
@@ -27,7 +26,6 @@ mod i2c {
     #[derive(Clone, Copy)]
     pub enum DCXO {
         Main,
-        #[cfg(has_wrpll)]
         Helper,
     }
 
@@ -37,45 +35,40 @@ mod i2c {
 
     fn sda_i(dcxo: DCXO) -> bool {
         match dcxo {
-            DCXO::Main => unsafe { csr::main_dcxo::sda_in_read() == 1 },
-            #[cfg(has_wrpll)]
-            DCXO::Helper => unsafe { csr::helper_dcxo::sda_in_read() == 1 },
+            DCXO::Main => unsafe { csr::wrpll::main_dcxo_sda_in_read() == 1 },
+            DCXO::Helper => unsafe { csr::wrpll::helper_dcxo_sda_in_read() == 1 },
         }
     }
 
     fn sda_oe(dcxo: DCXO, oe: bool) {
         let val = if oe { 1 } else { 0 };
         match dcxo {
-            DCXO::Main => unsafe { csr::main_dcxo::sda_oe_write(val) },
-            #[cfg(has_wrpll)]
-            DCXO::Helper => unsafe { csr::helper_dcxo::sda_oe_write(val) },
+            DCXO::Main => unsafe { csr::wrpll::main_dcxo_sda_oe_write(val) },
+            DCXO::Helper => unsafe { csr::wrpll::helper_dcxo_sda_oe_write(val) },
         };
     }
 
     fn sda_o(dcxo: DCXO, o: bool) {
         let val = if o { 1 } else { 0 };
         match dcxo {
-            DCXO::Main => unsafe { csr::main_dcxo::sda_out_write(val) },
-            #[cfg(has_wrpll)]
-            DCXO::Helper => unsafe { csr::helper_dcxo::sda_out_write(val) },
+            DCXO::Main => unsafe { csr::wrpll::main_dcxo_sda_out_write(val) },
+            DCXO::Helper => unsafe { csr::wrpll::helper_dcxo_sda_out_write(val) },
         };
     }
 
     fn scl_oe(dcxo: DCXO, oe: bool) {
         let val = if oe { 1 } else { 0 };
         match dcxo {
-            DCXO::Main => unsafe { csr::main_dcxo::scl_oe_write(val) },
-            #[cfg(has_wrpll)]
-            DCXO::Helper => unsafe { csr::helper_dcxo::scl_oe_write(val) },
+            DCXO::Main => unsafe { csr::wrpll::main_dcxo_scl_oe_write(val) },
+            DCXO::Helper => unsafe { csr::wrpll::helper_dcxo_scl_oe_write(val) },
         };
     }
 
     fn scl_o(dcxo: DCXO, o: bool) {
         let val = if o { 1 } else { 0 };
         match dcxo {
-            DCXO::Main => unsafe { csr::main_dcxo::scl_out_write(val) },
-            #[cfg(has_wrpll)]
-            DCXO::Helper => unsafe { csr::helper_dcxo::scl_out_write(val) },
+            DCXO::Main => unsafe { csr::wrpll::main_dcxo_scl_out_write(val) },
+            DCXO::Helper => unsafe { csr::wrpll::helper_dcxo_scl_out_write(val) },
         };
     }
 
@@ -219,7 +212,7 @@ fn read(dcxo: i2c::DCXO, reg: u8, timer: &mut GlobalTimer) -> Result<u8, &'stati
     Ok(val)
 }
 
-fn setup(dcxo: i2c::DCXO, config: DividerConfig, timer: &mut GlobalTimer) -> Result<(), &'static str> {
+fn setup(dcxo: i2c::DCXO, config: &DividerConfig, timer: &mut GlobalTimer) -> Result<(), &'static str> {
     i2c::init(dcxo, timer)?;
 
     write(dcxo, 255, 0x00, timer)?; // PAGE
@@ -252,47 +245,137 @@ fn setup(dcxo: i2c::DCXO, config: DividerConfig, timer: &mut GlobalTimer) -> Res
     Ok(())
 }
 
-pub fn main_setup(timer: &mut GlobalTimer, settings: FrequencySetting) -> Result<(), &'static str> {
+pub fn main_setup(timer: &mut GlobalTimer, settings: &FrequencySetting) -> Result<(), &'static str> {
     unsafe {
-        csr::main_dcxo::bitbang_enable_write(1);
-        csr::main_dcxo::i2c_address_write(ADDRESS);
+        csr::wrpll::main_dcxo_bitbang_enable_write(1);
+        csr::wrpll::main_dcxo_i2c_address_write(ADDRESS);
     }
 
-    setup(i2c::DCXO::Main, settings.main, timer)?;
+    setup(i2c::DCXO::Main, &settings.main, timer)?;
 
     // Si549 maximum settling time for large frequency change.
     timer.delay_us(40_000);
 
     unsafe {
-        csr::main_dcxo::bitbang_enable_write(0);
+        csr::wrpll::main_dcxo_bitbang_enable_write(0);
     }
 
     info!("Main Si549 started");
     Ok(())
 }
 
+pub fn helper_setup(timer: &mut GlobalTimer, settings: &FrequencySetting) -> Result<(), &'static str> {
+    unsafe {
+        csr::wrpll::helper_reset_write(1);
+        csr::wrpll::helper_dcxo_bitbang_enable_write(1);
+        csr::wrpll::helper_dcxo_i2c_address_write(ADDRESS);
+    }
+
+    setup(i2c::DCXO::Helper, &settings.helper, timer)?;
+
+    // Si549 maximum settling time for large frequency change.
+    timer.delay_us(40_000);
+
+    unsafe {
+        csr::wrpll::helper_reset_write(0);
+        csr::wrpll::helper_dcxo_bitbang_enable_write(0);
+    }
+    info!("Helper Si549 started");
+    Ok(())
+}
+
+fn set_adpll(dcxo: i2c::DCXO, adpll: i32) -> Result<(), &'static str> {
+    if adpll.abs() > ADPLL_MAX {
+        return Err("adpll is too large");
+    }
+
+    match dcxo {
+        i2c::DCXO::Main => unsafe {
+            if csr::wrpll::main_dcxo_bitbang_enable_read() == 1 {
+                return Err("Main si549 bitbang mode is active when using gateware i2c");
+            }
+
+            while csr::wrpll::main_dcxo_adpll_busy_read() == 1 {}
+            if csr::wrpll::main_dcxo_nack_read() == 1 {
+                return Err("Main si549 failed to ack adpll write");
+            }
+
+            csr::wrpll::main_dcxo_i2c_address_write(ADDRESS);
+            csr::wrpll::main_dcxo_adpll_write(adpll as u32);
+
+            csr::wrpll::main_dcxo_adpll_stb_write(1);
+        },
+        i2c::DCXO::Helper => unsafe {
+            if csr::wrpll::helper_dcxo_bitbang_enable_read() == 1 {
+                return Err("Helper si549 bitbang mode is active when using gateware i2c");
+            }
+
+            while csr::wrpll::helper_dcxo_adpll_busy_read() == 1 {}
+            if csr::wrpll::helper_dcxo_nack_read() == 1 {
+                return Err("Helper si549 failed to ack adpll write");
+            }
+
+            csr::wrpll::helper_dcxo_i2c_address_write(ADDRESS);
+            csr::wrpll::helper_dcxo_adpll_write(adpll as u32);
+
+            csr::wrpll::helper_dcxo_adpll_stb_write(1);
+        },
+    };
+
+    Ok(())
+}
+
 #[cfg(has_wrpll)]
 pub mod wrpll {
 
-    use embedded_hal::blocking::delay::DelayMs;
-    use libcortex_a9::mutex::Mutex;
-
     use super::*;
 
-    const TIMER_WIDTH: u32 = 24;
-    const COUNTER_DIV: u32 = 2;
+    const BEATING_PERIOD: i32 = 0x8000;
+    const BEATING_HALFPERIOD: i32 = 0x4000;
+    const COUNTER_WIDTH: u32 = 24;
+    const DIV_WIDTH: u32 = 2;
 
-    const ADPLL_MAX: i32 = (950.0 / 0.0001164) as i32;
+    // y[n] = b0*x[n] + b1*x[n-1] + b2*x[n-2] - a1*y[n-1] - a2*y[n-2]
+    struct FilterParameters {
+        pub b0: f64,
+        pub b1: f64,
+        pub b2: f64,
+        pub a1: f64,
+        pub a2: f64,
+    }
 
-    const KP: i32 = 6;
-    const KI: i32 = 2;
+    #[cfg(rtio_frequency = "100.0")]
+    const LPF: FilterParameters = FilterParameters {
+        b0: 0.03967479060647884,
+        b1: 0.07934958121295768,
+        b2: 0.03967479060647884,
+        a1: -1.3865593741228928,
+        a2: 0.5452585365488082,
+    };
 
-    static BASE_ADPLL: Mutex<i32> = Mutex::new(0);
-    static H_INTEGRATOR: Mutex<i32> = Mutex::new(0);
-    static M_INTEGRATOR: Mutex<i32> = Mutex::new(0);
+    #[cfg(rtio_frequency = "125.0")]
+    const LPF: FilterParameters = FilterParameters {
+        b0: 0.07209205036273991,
+        b1: 0.14418410072547982,
+        b2: 0.07209205036273991,
+        a1: -0.6114078511562919,
+        a2: -0.10022394739274834,
+    };
+
+    static mut H_ADPLL1: i32 = 0;
+    static mut H_ADPLL2: i32 = 0;
+    static mut PERIOD_ERR1: i32 = 0;
+    static mut PERIOD_ERR2: i32 = 0;
+
+    static mut M_ADPLL1: i32 = 0;
+    static mut M_ADPLL2: i32 = 0;
+    static mut PHASE_ERR1: i32 = 0;
+    static mut PHASE_ERR2: i32 = 0;
+
+    static mut BASE_ADPLL: i32 = 0;
 
     #[derive(Clone, Copy)]
-    pub enum FIQ {
+    pub enum ISR {
         RefTag,
         MainTag,
     }
@@ -300,46 +383,62 @@ pub mod wrpll {
     mod tag_collector {
         use super::*;
 
-        const BEATING_PERIOD: i32 = 0x8000;
-        const BEATING_HALFPERIOD: i32 = 0x4000;
-
-        static REF_TAG: Mutex<u32> = Mutex::new(0);
-        static REF_TAG_READY: Mutex<bool> = Mutex::new(false);
-        static MAIN_TAG: Mutex<u32> = Mutex::new(0);
-        static MAIN_TAG_READY: Mutex<bool> = Mutex::new(false);
+        #[cfg(wrpll_ref_clk = "GT_CDR")]
+        static mut TAG_OFFSET: u32 = 8382;
+        #[cfg(wrpll_ref_clk = "SMA_CLKIN")]
+        static mut TAG_OFFSET: u32 = 0;
+        static mut REF_TAG: u32 = 0;
+        static mut REF_TAG_READY: bool = false;
+        static mut MAIN_TAG: u32 = 0;
+        static mut MAIN_TAG_READY: bool = false;
 
         pub fn reset() {
             clear_phase_diff_ready();
-            *REF_TAG.lock() = 0;
-            *MAIN_TAG.lock() = 0;
+            unsafe {
+                REF_TAG = 0;
+                MAIN_TAG = 0;
+            }
         }
 
         pub fn clear_phase_diff_ready() {
-            *REF_TAG_READY.lock() = false;
-            *MAIN_TAG_READY.lock() = false;
+            unsafe {
+                REF_TAG_READY = false;
+                MAIN_TAG_READY = false;
+            }
         }
 
-        pub fn collect_tags(interrupt: FIQ) {
+        pub fn collect_tags(interrupt: ISR) {
             match interrupt {
-                FIQ::RefTag => {
-                    *REF_TAG.lock() = unsafe { csr::wrpll::ref_tag_read() };
-                    *REF_TAG_READY.lock() = true;
-                }
-                FIQ::MainTag => {
-                    *MAIN_TAG.lock() = unsafe { csr::wrpll::main_tag_read() };
-                    *MAIN_TAG_READY.lock() = true;
-                }
+                ISR::RefTag => unsafe {
+                    REF_TAG = csr::wrpll::ref_tag_read();
+                    REF_TAG_READY = true;
+                },
+                ISR::MainTag => unsafe {
+                    MAIN_TAG = csr::wrpll::main_tag_read();
+                    MAIN_TAG_READY = true;
+                },
             }
         }
 
         pub fn phase_diff_ready() -> bool {
-            *REF_TAG_READY.lock() && *MAIN_TAG_READY.lock()
+            unsafe { REF_TAG_READY && MAIN_TAG_READY }
+        }
+
+        #[cfg(feature = "calibrate_wrpll_skew")]
+        pub fn set_tag_offset(offset: u32) {
+            unsafe {
+                TAG_OFFSET = offset;
+            }
+        }
+
+        #[cfg(feature = "calibrate_wrpll_skew")]
+        pub fn get_tag_offset() -> u32 {
+            unsafe { TAG_OFFSET }
         }
 
         pub fn get_period_error() -> i32 {
             // n * BEATING_PERIOD - REF_TAG(n) mod BEATING_PERIOD
-            let mut period_error = (*REF_TAG.lock()).overflowing_neg().0.rem_euclid(BEATING_PERIOD as u32) as i32;
-
+            let mut period_error = unsafe { REF_TAG.overflowing_neg().0.rem_euclid(BEATING_PERIOD as u32) as i32 };
             // mapping tags from [0, 2π] -> [-π, π]
             if period_error > BEATING_HALFPERIOD {
                 period_error -= BEATING_PERIOD
@@ -348,11 +447,13 @@ pub mod wrpll {
         }
 
         pub fn get_phase_error() -> i32 {
-            // MAIN_TAG(n) - REF_TAG(n) mod BEATING_PERIOD
-            let mut phase_error = (*MAIN_TAG.lock())
-                .overflowing_sub(*REF_TAG.lock())
-                .0
-                .rem_euclid(BEATING_PERIOD as u32) as i32;
+            // MAIN_TAG(n) - REF_TAG(n) - TAG_OFFSET mod BEATING_PERIOD
+            let mut phase_error = unsafe {
+                MAIN_TAG
+                    .overflowing_sub(REF_TAG + TAG_OFFSET)
+                    .0
+                    .rem_euclid(BEATING_PERIOD as u32) as i32
+            };
 
             // mapping tags from [0, 2π] -> [-π, π]
             if phase_error > BEATING_HALFPERIOD {
@@ -362,29 +463,7 @@ pub mod wrpll {
         }
     }
 
-    pub fn helper_setup(timer: &mut GlobalTimer, settings: FrequencySetting) -> Result<(), &'static str> {
-        unsafe {
-            csr::wrpll::helper_reset_write(1);
-            csr::helper_dcxo::bitbang_enable_write(1);
-            csr::helper_dcxo::i2c_address_write(ADDRESS);
-        }
-
-        setup(i2c::DCXO::Helper, settings.helper, timer)?;
-
-        // Si549 maximum settling time for large frequency change.
-        timer.delay_us(40_000);
-
-        unsafe {
-            csr::wrpll::helper_reset_write(0);
-            csr::helper_dcxo::bitbang_enable_write(0);
-        }
-        info!("Helper Si549 started");
-
-        timer.delay_ms(5_000);
-        Ok(())
-    }
-
-    fn set_fiq(en: bool) {
+    fn set_isr(en: bool) {
         let val = if en { 1 } else { 0 };
         unsafe {
             csr::wrpll::ref_tag_ev_enable_write(val);
@@ -392,71 +471,24 @@ pub mod wrpll {
         }
     }
 
-    /// set adpll using gateware i2c
-    /// Note: disable main/helper i2c bitbang before using this function
-    fn set_adpll(dcxo: i2c::DCXO, adpll: i32) -> Result<(), &'static str> {
-        if adpll.abs() > ADPLL_MAX {
-            return Err("adpll is too large");
-        }
-
-        match dcxo {
-            i2c::DCXO::Main => unsafe {
-                if csr::main_dcxo::bitbang_enable_read() == 1 {
-                    return Err("Main si549 bitbang mode is active when using gateware i2c");
-                }
-
-                while csr::main_dcxo::adpll_busy_read() == 1 {}
-                csr::main_dcxo::i2c_address_write(ADDRESS);
-                csr::main_dcxo::adpll_write(adpll as u32);
-
-                csr::main_dcxo::adpll_stb_write(1);
-                csr::main_dcxo::adpll_stb_write(0);
-
-                if csr::main_dcxo::nack_read() == 1 {
-                    return Err("Main si549 failed to ack adpll write");
-                }
-            },
-            i2c::DCXO::Helper => unsafe {
-                if csr::helper_dcxo::bitbang_enable_read() == 1 {
-                    return Err("Helper si549 bitbang mode is active when using gateware i2c");
-                }
-
-                while csr::helper_dcxo::adpll_busy_read() == 1 {}
-                csr::helper_dcxo::i2c_address_write(ADDRESS);
-                csr::helper_dcxo::adpll_write(adpll as u32);
-
-                csr::helper_dcxo::adpll_stb_write(1);
-                csr::helper_dcxo::adpll_stb_write(0);
-
-                if csr::helper_dcxo::nack_read() == 1 {
-                    return Err("Helper si549 failed to ack adpll write");
-                }
-            },
-        };
-
-        Ok(())
-    }
-
-    /// To get within capture range
-    fn set_base_adpll(timer: &mut GlobalTimer) -> Result<(), &'static str> {
+    fn set_base_adpll() -> Result<(), &'static str> {
         let count2adpll =
-            |error: i32| (((error) as f64 * 1e6) / (0.0001164 * (1 << (TIMER_WIDTH - COUNTER_DIV)) as f64)) as i32;
-
-        let (ref_count, main_count) = get_freq_counts(timer);
-        let mut base_adpll_lock = BASE_ADPLL.lock();
-        *base_adpll_lock = count2adpll(ref_count as i32 - main_count as i32);
-        set_adpll(i2c::DCXO::Main, *base_adpll_lock)?;
-        set_adpll(i2c::DCXO::Helper, *base_adpll_lock)?;
+            |error: i32| ((error as f64 * 1e6) / (0.0001164 * (1 << (COUNTER_WIDTH - DIV_WIDTH)) as f64)) as i32;
 
+        let (ref_count, main_count) = get_freq_counts();
+        unsafe {
+            BASE_ADPLL = count2adpll(ref_count as i32 - main_count as i32);
+            set_adpll(i2c::DCXO::Main, BASE_ADPLL)?;
+            set_adpll(i2c::DCXO::Helper, BASE_ADPLL)?;
+        }
         Ok(())
     }
 
-    pub fn get_freq_counts(timer: &mut GlobalTimer) -> (u32, u32) {
+    fn get_freq_counts() -> (u32, u32) {
         unsafe {
-            csr::wrpll::frequency_counter_update_en_write(1);
-            timer.delay_us(150_000); // 8ns << TIMER_WIDTH
-            csr::wrpll::frequency_counter_update_en_write(0);
-            #[cfg(wrpll_ref_clk = "GTX_CDR")]
+            csr::wrpll::frequency_counter_update_write(1);
+            while csr::wrpll::frequency_counter_busy_read() == 1 {}
+            #[cfg(wrpll_ref_clk = "GT_CDR")]
             let ref_count = csr::wrpll::frequency_counter_counter_rtio_rx0_read();
             #[cfg(wrpll_ref_clk = "SMA_CLKIN")]
             let ref_count = csr::wrpll::frequency_counter_counter_ref_read();
@@ -466,39 +498,48 @@ pub mod wrpll {
         }
     }
 
-    fn reset_plls() -> Result<(), &'static str> {
-        *H_INTEGRATOR.lock() = 0;
-        *M_INTEGRATOR.lock() = 0;
+    fn reset_plls(timer: &mut GlobalTimer) -> Result<(), &'static str> {
+        unsafe {
+            H_ADPLL1 = 0;
+            H_ADPLL2 = 0;
+            PERIOD_ERR1 = 0;
+            PERIOD_ERR2 = 0;
+            M_ADPLL1 = 0;
+            M_ADPLL2 = 0;
+            PHASE_ERR1 = 0;
+            PHASE_ERR2 = 0;
+        }
         set_adpll(i2c::DCXO::Main, 0)?;
         set_adpll(i2c::DCXO::Helper, 0)?;
+        // wait for adpll to transfer and DCXO to settle
+        timer.delay_us(200);
         Ok(())
     }
 
-    fn clear_pending(interrupt: FIQ) {
+    fn clear_pending(interrupt: ISR) {
         match interrupt {
-            FIQ::RefTag => unsafe { csr::wrpll::ref_tag_ev_pending_write(1) },
-            FIQ::MainTag => unsafe { csr::wrpll::main_tag_ev_pending_write(1) },
+            ISR::RefTag => unsafe { csr::wrpll::ref_tag_ev_pending_write(1) },
+            ISR::MainTag => unsafe { csr::wrpll::main_tag_ev_pending_write(1) },
         };
     }
 
-    fn is_pending(interrupt: FIQ) -> bool {
+    fn is_pending(interrupt: ISR) -> bool {
         match interrupt {
-            FIQ::RefTag => unsafe { csr::wrpll::ref_tag_ev_pending_read() == 1 },
-            FIQ::MainTag => unsafe { csr::wrpll::main_tag_ev_pending_read() == 1 },
+            ISR::RefTag => unsafe { csr::wrpll::ref_tag_ev_pending_read() == 1 },
+            ISR::MainTag => unsafe { csr::wrpll::main_tag_ev_pending_read() == 1 },
         }
     }
 
     pub fn interrupt_handler() {
-        if is_pending(FIQ::RefTag) {
-            tag_collector::collect_tags(FIQ::RefTag);
-            clear_pending(FIQ::RefTag);
-
+        if is_pending(ISR::RefTag) {
+            tag_collector::collect_tags(ISR::RefTag);
+            clear_pending(ISR::RefTag);
             helper_pll().expect("failed to run helper DCXO PLL");
         }
 
-        if is_pending(FIQ::MainTag) {
-            tag_collector::collect_tags(FIQ::MainTag);
-            clear_pending(FIQ::MainTag);
+        if is_pending(ISR::MainTag) {
+            tag_collector::collect_tags(ISR::MainTag);
+            clear_pending(ISR::MainTag);
         }
 
         if tag_collector::phase_diff_ready() {
@@ -508,125 +549,165 @@ pub mod wrpll {
     }
 
     fn helper_pll() -> Result<(), &'static str> {
-        // unsafe {
-        //     use libboard_zynq::println;
-
-        //     ERR_ARR[COUNTER] = tag_collector::get_period_error();
-
-        //     if COUNTER == SIZE - 1 {
-        //         for i in 0..SIZE {
-        //             println!("{}", ERR_ARR[i])
-        //         }
-        //         COUNTER = 0;
-        //         let mut timer = GlobalTimer::get();
-        //         timer.delay_us(20_000_000);
-        //     }
-        //     COUNTER += 1;
-        // }
         let period_err = tag_collector::get_period_error();
-        let mut integrator_lock = H_INTEGRATOR.lock();
-
-        *integrator_lock += period_err * KI;
-        let mut h_adpll = *BASE_ADPLL.lock() + period_err * KP + *integrator_lock;
-
-        h_adpll = h_adpll.clamp(-ADPLL_MAX, ADPLL_MAX);
-        set_adpll(i2c::DCXO::Helper, h_adpll)?;
-
+        unsafe {
+            let adpll = ((LPF.b0 * period_err as f64) + (LPF.b1 * PERIOD_ERR1 as f64) + (LPF.b2 * PERIOD_ERR2 as f64)
+                - (LPF.a1 * H_ADPLL1 as f64)
+                - (LPF.a2 * H_ADPLL2 as f64)) as i32;
+            set_adpll(i2c::DCXO::Helper, BASE_ADPLL + adpll)?;
+            H_ADPLL2 = H_ADPLL1;
+            PERIOD_ERR2 = PERIOD_ERR1;
+            H_ADPLL1 = adpll;
+            PERIOD_ERR1 = period_err;
+        };
         Ok(())
     }
 
-    const SIZE: usize = 2000;
-    static mut COUNTER: usize = 0;
-    static mut ERR_ARR: [i32; SIZE] = [0; SIZE];
-    static mut FIN_ADPLL: i32 = 0;
-
     fn main_pll() -> Result<(), &'static str> {
-        // unsafe {
-        //     use libboard_zynq::println;
-
-        //     ERR_ARR[COUNTER] = tag_collector::get_phase_error();
-
-        //     if COUNTER == SIZE - 1 {
-        //         for i in 0..SIZE {
-        //             println!("{}", ERR_ARR[i])
-        //         }
-        //         println!("{:>3} Zero crossing adpll = {:>5}", SIZE, FIN_ADPLL);
-        //         COUNTER = 0;
-        //         let mut timer = GlobalTimer::get();
-        //         timer.delay_us(20_000_000);
-        //     }
-        //     COUNTER += 1;
-        // }
-
         let phase_err = tag_collector::get_phase_error();
-        let mut integrator_lock = M_INTEGRATOR.lock();
-
-        *integrator_lock += phase_err * KI;
-        let mut m_adpll = *BASE_ADPLL.lock() + phase_err * KP + *integrator_lock;
-
-        m_adpll = m_adpll.clamp(-ADPLL_MAX, ADPLL_MAX);
-        set_adpll(i2c::DCXO::Main, m_adpll)?;
-
         unsafe {
-            if ERR_ARR[COUNTER] == 0 {
-                FIN_ADPLL = m_adpll;
+            let adpll = ((LPF.b0 * phase_err as f64) + (LPF.b1 * PHASE_ERR1 as f64) + (LPF.b2 * PHASE_ERR2 as f64)
+                - (LPF.a1 * M_ADPLL1 as f64)
+                - (LPF.a2 * M_ADPLL2 as f64)) as i32;
+            set_adpll(i2c::DCXO::Main, BASE_ADPLL + adpll)?;
+            M_ADPLL2 = M_ADPLL1;
+            PHASE_ERR2 = PHASE_ERR1;
+            M_ADPLL1 = adpll;
+            PHASE_ERR1 = phase_err;
+        };
+        Ok(())
+    }
+
+    #[cfg(wrpll_ref_clk = "GT_CDR")]
+    fn test_skew(timer: &mut GlobalTimer) -> Result<(), &'static str> {
+        // wait for PLL to stabilize
+        timer.delay_us(20_000);
+
+        info!("testing the skew of SYS CLK...");
+        if has_timing_error(timer) {
+            return Err("the skew cannot satisfy setup/hold time constraint of RX synchronizer");
+        }
+        info!("the skew of SYS CLK met the timing constraint");
+        Ok(())
+    }
+
+    #[cfg(wrpll_ref_clk = "GT_CDR")]
+    fn has_timing_error(timer: &mut GlobalTimer) -> bool {
+        unsafe {
+            csr::wrpll_skewtester::error_write(1);
+        }
+        timer.delay_us(5_000);
+        unsafe { csr::wrpll_skewtester::error_read() == 1 }
+    }
+
+    #[cfg(feature = "calibrate_wrpll_skew")]
+    fn find_edge(target: bool, timer: &mut GlobalTimer) -> Result<u32, &'static str> {
+        const STEP: u32 = 8;
+        const STABLE_THRESHOLD: u32 = 10;
+
+        enum FSM {
+            Init,
+            WaitEdge,
+            GotEdge,
+        }
+
+        let mut state: FSM = FSM::Init;
+        let mut offset: u32 = tag_collector::get_tag_offset();
+        let mut median_edge: u32 = 0;
+        let mut stable_counter: u32 = 0;
+
+        for _ in 0..(BEATING_PERIOD as u32 / STEP) as usize {
+            tag_collector::set_tag_offset(offset);
+            offset += STEP;
+            // wait for PLL to stabilize
+            timer.delay_us(20_000);
+
+            let error = has_timing_error(timer);
+            // A median edge deglitcher
+            match state {
+                FSM::Init => {
+                    if error != target {
+                        stable_counter += 1;
+                    } else {
+                        stable_counter = 0;
+                    }
+
+                    if stable_counter >= STABLE_THRESHOLD {
+                        state = FSM::WaitEdge;
+                        stable_counter = 0;
+                    }
+                }
+                FSM::WaitEdge => {
+                    if error == target {
+                        state = FSM::GotEdge;
+                        median_edge = offset;
+                    }
+                }
+                FSM::GotEdge => {
+                    if error != target {
+                        median_edge += STEP;
+                        stable_counter = 0;
+                    } else {
+                        stable_counter += 1;
+                    }
+
+                    if stable_counter >= STABLE_THRESHOLD {
+                        return Ok(median_edge);
+                    }
+                }
             }
         }
+        return Err("failed to find timing error edge");
+    }
+
+    #[cfg(feature = "calibrate_wrpll_skew")]
+    fn calibrate_skew(timer: &mut GlobalTimer) -> Result<(), &'static str> {
+        info!("calibrating skew to meet timing constraint...");
+
+        // clear calibrated value
+        tag_collector::set_tag_offset(0);
+        let rising = find_edge(true, timer)? as i32;
+        let falling = find_edge(false, timer)? as i32;
+
+        let width = BEATING_PERIOD - (falling - rising);
+        let result = falling + width / 2;
+        tag_collector::set_tag_offset(result as u32);
+
+        info!(
+            "calibration successful, error zone: {} -> {}, width: {} ({}deg), middle of working region: {}",
+            rising,
+            falling,
+            width,
+            360 * width / BEATING_PERIOD,
+            result,
+        );
 
         Ok(())
     }
 
     pub fn select_recovered_clock(rc: bool, timer: &mut GlobalTimer) {
-        set_fiq(false);
+        set_isr(false);
 
         if rc {
             tag_collector::reset();
-            reset_plls().expect("failed to reset main and helper PLL");
+            reset_plls(timer).expect("failed to reset main and helper PLL");
 
-            info!("warming up refclk...");
-            // refclk need a couple seconds for freq counter to read it properly
-            // timer.delay_us(20_000_000);
-            set_base_adpll(timer).expect("failed to set base adpll");
-
-            let ppm = 0.0;
-            *BASE_ADPLL.lock() += (ppm / 0.0001164) as i32;
-            timer.delay_us(200);
-            info!("KP = {}, KI = {}", KP, KI);
-            info!("adding {}ppm to main & helper base adpll ({})", ppm, *BASE_ADPLL.lock());
+            // get within capture range
+            set_base_adpll().expect("failed to set base adpll");
 
             // clear gateware pending flag
-            clear_pending(FIQ::RefTag);
-            clear_pending(FIQ::MainTag);
-
-            unsafe {
-                info!("ref tag = {} ", csr::wrpll::ref_tag_read());
-                info!("main tag = {} ", csr::wrpll::main_tag_read());
-            }
+            clear_pending(ISR::RefTag);
+            clear_pending(ISR::MainTag);
 
             // use nFIQ to avoid IRQ being disabled by mutex lock and mess up PLL
-            set_fiq(true);
+            set_isr(true);
             info!("WRPLL interrupt enabled");
 
-            timer.delay_ms(500);
-            unsafe {
-                info!("main_tag_ev_enable_read() = {}", csr::wrpll::main_tag_ev_enable_read());
-                info!("main_tag_ev_status_read() = {}", csr::wrpll::main_tag_ev_status_read());
-                info!("ref_tag_ev_enable_read() = {}", csr::wrpll::ref_tag_ev_enable_read());
-                info!("ref_tag_ev_status_read() = {}", csr::wrpll::ref_tag_ev_status_read());
+            #[cfg(feature = "calibrate_wrpll_skew")]
+            calibrate_skew(timer).expect("failed to set the correct skew");
 
-                // loop {
-                //     if is_pending(FIQ::RefTag) {
-                //         info!("REF tag is pending")
-                //     }
-                //     if is_pending(FIQ::MainTag) {
-                //         info!("Main tag is pending")
-                //     }
-
-                //     // info!("ref tag = {} ", csr::wrpll::ref_tag_read());
-                //     // info!("main tag = {} ", csr::wrpll::main_tag_read());
-                //     timer.delay_ms(500);
-                // }
-            }
+            #[cfg(wrpll_ref_clk = "GT_CDR")]
+            test_skew(timer).expect("skew test failed");
         }
     }
 }
@@ -649,12 +730,10 @@ pub mod wrpll_refclk {
         pub filt_reg2: u16,     //0x4F
     }
 
-    fn one_clock_cycle(timer: &mut GlobalTimer) {
+    fn one_clock_cycle() {
         unsafe {
             csr::wrpll_refclk::mmcm_dclk_write(1);
-            timer.delay_us(1);
             csr::wrpll_refclk::mmcm_dclk_write(0);
-            timer.delay_us(1);
         }
     }
 
@@ -692,33 +771,34 @@ pub mod wrpll_refclk {
         unsafe { csr::wrpll_refclk::mmcm_dready_read() == 1 }
     }
 
-    fn read(timer: &mut GlobalTimer, address: u8) -> u16 {
+    #[allow(dead_code)]
+    fn read(address: u8) -> u16 {
         set_addr(address);
         set_enable(true);
-        // Set DADDR on the MMCM and assert DEN for one clock cycle
-        one_clock_cycle(timer);
+        // Set DADDR on the mmcm and assert DEN for one clock cycle
+        one_clock_cycle();
 
         set_enable(false);
         while !drp_ready() {
             // keep the clock signal until data is ready
-            one_clock_cycle(timer);
+            one_clock_cycle();
         }
         get_data()
     }
 
-    fn write(timer: &mut GlobalTimer, address: u8, value: u16) {
+    fn write(address: u8, value: u16) {
         set_addr(address);
         set_data(value);
         set_write_enable(true);
         set_enable(true);
-        // Set DADDR, DI on the MMCM and assert DWE, DEN for one clock cycle
-        one_clock_cycle(timer);
+        // Set DADDR, DI on the mmcm and assert DWE, DEN for one clock cycle
+        one_clock_cycle();
 
         set_write_enable(false);
         set_enable(false);
         while !drp_ready() {
             // keep the clock signal until write is finished
-            one_clock_cycle(timer);
+            one_clock_cycle();
         }
     }
 
@@ -729,49 +809,45 @@ pub mod wrpll_refclk {
         }
     }
 
-    pub fn setup(timer: &mut GlobalTimer, settings: MmcmSetting) -> Result<(), &'static str> {
-        // Based on "DRP State Machine" from XAPP888
-
-        // hold reset HIGH during mmcm config
-        reset(true);
-        write(timer, 0x08, settings.clkout0_reg1);
-        write(timer, 0x09, settings.clkout0_reg2);
-        write(timer, 0x14, settings.clkfbout_reg1);
-        write(timer, 0x15, settings.clkfbout_reg2);
-        write(timer, 0x16, settings.div_reg);
-        write(timer, 0x18, settings.lock_reg1);
-        write(timer, 0x19, settings.lock_reg2);
-        write(timer, 0x1A, settings.lock_reg3);
-        write(timer, 0x28, settings.power_reg);
-        write(timer, 0x4E, settings.filt_reg1);
-        write(timer, 0x4F, settings.filt_reg2);
-        reset(false);
-
-        // wait for the mmcm to lock
-        timer.delay_us(100);
-
-        let locked = unsafe { csr::wrpll_refclk::mmcm_locked_read() == 1 };
-        if !locked {
-            return Err("failed to generate 125Mhz ref clock from SMA CLKIN");
+    pub fn setup(timer: &mut GlobalTimer, settings: MmcmSetting, mmcm_bypass: bool) -> Result<(), &'static str> {
+        unsafe {
+            csr::wrpll_refclk::refclk_reset_write(1);
         }
 
-        // const TIMER_WIDTH: u32 = 24;
-        // const COUNTER_DIV: u32 = 2;
+        if mmcm_bypass {
+            info!("Bypassing mmcm");
+            unsafe {
+                csr::wrpll_refclk::mmcm_bypass_write(1);
+            }
+        } else {
+            // Based on "DRP State Machine" from XAPP888
+            // hold reset HIGH during mmcm config
+            reset(true);
+            write(0x08, settings.clkout0_reg1);
+            write(0x09, settings.clkout0_reg2);
+            write(0x14, settings.clkfbout_reg1);
+            write(0x15, settings.clkfbout_reg2);
+            write(0x16, settings.div_reg);
+            write(0x18, settings.lock_reg1);
+            write(0x19, settings.lock_reg2);
+            write(0x1A, settings.lock_reg3);
+            write(0x28, settings.power_reg);
+            write(0x4E, settings.filt_reg1);
+            write(0x4F, settings.filt_reg2);
+            reset(false);
 
-        // let (ref_count, main_count) = wrpll::get_freq_counts(timer);
+            // wait for the mmcm to lock
+            timer.delay_us(100);
 
-        // let f_sys = 125e6;
+            let locked = unsafe { csr::wrpll_refclk::mmcm_locked_read() == 1 };
+            if !locked {
+                return Err("mmcm failed to generate 125MHz ref clock from SMA CLKIN");
+            }
+        }
 
-        // info!(
-        //     "ref counter = {} freq = {}",
-        //     ref_count,
-        //     (ref_count as f64 * f_sys) as f64 / (1 << (TIMER_WIDTH - COUNTER_DIV)) as f64
-        // );
-        // info!(
-        //     "Main counter = {} freq = {}",
-        //     main_count,
-        //     (main_count as f64 * f_sys) as f64 / (1 << (TIMER_WIDTH - COUNTER_DIV)) as f64
-        // );
+        unsafe {
+            csr::wrpll_refclk::refclk_reset_write(0);
+        }
 
         Ok(())
     }

src/libbuild_zynq/link.x

@@ -10,7 +10,9 @@ SECTIONS
     __text_start = .;
     .text :
     {
+        __exceptions_start = .;
         KEEP(*(.text.exceptions));
+        __exceptions_end = .;
         *(.text.boot);
         *(.text .text.*);
     } > SDRAM

src/libio/cursor.rs

@@ -45,7 +45,10 @@ impl<T: AsRef<[u8]>> Read for Cursor<T> {
     fn read(&mut self, buf: &mut [u8]) -> Result<usize, IoError> {
         let data = &self.inner.as_ref()[self.pos..];
         let len = buf.len().min(data.len());
-        buf[..len].copy_from_slice(&data[..len]);
+        // ``copy_from_slice`` generates AXI bursts, use a regular loop instead
+        for i in 0..len {
+            buf[i] = data[i];
+        }
         self.pos += len;
         Ok(len)
     }
@@ -55,7 +58,9 @@ impl Write for Cursor<&mut [u8]> {
     fn write(&mut self, buf: &[u8]) -> Result<usize, IoError> {
         let data = &mut self.inner[self.pos..];
         let len = buf.len().min(data.len());
-        data[..len].copy_from_slice(&buf[..len]);
+        for i in 0..len {
+            data[i] = buf[i];
+        }
         self.pos += len;
         Ok(len)
     }
@@ -68,7 +73,6 @@ impl Write for Cursor<&mut [u8]> {
 
 #[cfg(feature = "alloc")]
 impl Write for Cursor<Vec<u8>> {
-    #[inline]
     fn write(&mut self, buf: &[u8]) -> Result<usize, IoError> {
         self.inner.extend_from_slice(buf);
         Ok(buf.len())

src/libksupport/src/lib.rs

@@ -21,7 +21,6 @@ pub use pl::csr::rtio_core;
 use void::Void;
 
 pub mod eh_artiq;
-pub mod fiq;
 pub mod i2c;
 pub mod irq;
 pub mod kernel;

src/runtime/src/analyzer.rs

@@ -60,7 +60,7 @@ pub mod remote_analyzer {
         routing_table: &drtio_routing::RoutingTable,
         up_destinations: &Rc<RefCell<[bool; drtio_routing::DEST_COUNT]>>,
         timer: GlobalTimer,
-    ) -> Result<RemoteBuffer, &'static str> {
+    ) -> Result<RemoteBuffer, drtio::Error> {
         // gets data from satellites and returns consolidated data
         let mut remote_data: Vec<u8> = Vec::new();
         let mut remote_error = false;

src/runtime/src/main.rs

@@ -8,20 +8,20 @@
 #[macro_use]
 extern crate alloc;
 
-#[cfg(all(feature = "target_kasli_soc", has_drtio))]
+#[cfg(all(feature = "target_kasli_soc", has_virtual_leds))]
 use core::cell::RefCell;
 
 use ksupport;
 use libasync::task;
 #[cfg(has_drtio_eem)]
 use libboard_artiq::drtio_eem;
-#[cfg(all(feature = "target_kasli_soc", has_drtio))]
+#[cfg(feature = "target_kasli_soc")]
 use libboard_artiq::io_expander;
 use libboard_artiq::{identifier_read, logger, pl};
 use libboard_zynq::{gic, mpcore, timer::GlobalTimer};
 use libconfig::Config;
 use libcortex_a9::l2c::enable_l2_cache;
-use libsupport_zynq::ram;
+use libsupport_zynq::{exception_vectors, ram};
 use log::{info, warn};
 
 mod analyzer;
@@ -38,7 +38,12 @@ mod rtio_mgt;
 #[cfg(has_drtio)]
 mod subkernel;
 
-#[cfg(all(feature = "target_kasli_soc", has_drtio))]
+// linker symbols
+extern "C" {
+    static __exceptions_start: u32;
+}
+
+#[cfg(all(feature = "target_kasli_soc", has_virtual_leds))]
 async fn io_expanders_service(
     i2c_bus: RefCell<&mut libboard_zynq::i2c::I2c>,
     io_expander0: RefCell<io_expander::IoExpander>,
@@ -77,6 +82,9 @@ static mut LOG_BUFFER: [u8; 1 << 17] = [0; 1 << 17];
 
 #[no_mangle]
 pub fn main_core0() {
+    unsafe {
+        exception_vectors::set_vector_table(&__exceptions_start as *const u32 as u32);
+    }
     enable_l2_cache(0x8);
     let mut timer = GlobalTimer::start();
 
@@ -93,7 +101,7 @@ pub fn main_core0() {
     info!("gateware ident: {}", identifier_read(&mut [0; 64]));
 
     ksupport::i2c::init();
-    #[cfg(all(feature = "target_kasli_soc", has_drtio))]
+    #[cfg(feature = "target_kasli_soc")]
     {
         let i2c_bus = unsafe { (ksupport::i2c::I2C_BUS).as_mut().unwrap() };
         let mut io_expander0 = io_expander::IoExpander::new(i2c_bus, 0).unwrap();
@@ -116,6 +124,7 @@ pub fn main_core0() {
         io_expander1.set(1, 1, false);
         io_expander0.service(i2c_bus).unwrap();
         io_expander1.service(i2c_bus).unwrap();
+        #[cfg(has_virtual_leds)]
         task::spawn(io_expanders_service(
             RefCell::new(i2c_bus),
             RefCell::new(io_expander0),

src/runtime/src/moninj.rs

@@ -58,10 +58,11 @@ mod remote_moninj {
     use log::error;
 
     use super::*;
-    use crate::rtio_mgt::drtio;
+    use crate::rtio_mgt::{drtio, drtio::Error as DrtioError};
 
     pub async fn read_probe(
         aux_mutex: &Rc<Mutex<bool>>,
+        routing_table: &drtio_routing::RoutingTable,
         timer: GlobalTimer,
         linkno: u8,
         destination: u8,
@@ -71,6 +72,7 @@ mod remote_moninj {
         let reply = drtio::aux_transact(
             aux_mutex,
             linkno,
+            routing_table,
             &drtioaux_async::Packet::MonitorRequest {
                 destination: destination,
                 channel: channel as _,
@@ -82,8 +84,8 @@ mod remote_moninj {
         match reply {
             Ok(drtioaux_async::Packet::MonitorReply { value }) => return value as i64,
             Ok(packet) => error!("received unexpected aux packet: {:?}", packet),
-            Err("link went down") => {
-                debug!("link is down");
+            Err(DrtioError::LinkDown) => {
+                warn!("link is down");
             }
             Err(e) => error!("aux packet error ({})", e),
         }
@@ -92,6 +94,7 @@ mod remote_moninj {
 
     pub async fn inject(
         aux_mutex: &Rc<Mutex<bool>>,
+        _routing_table: &drtio_routing::RoutingTable,
         _timer: GlobalTimer,
         linkno: u8,
         destination: u8,
@@ -115,6 +118,7 @@ mod remote_moninj {
 
     pub async fn read_injection_status(
         aux_mutex: &Rc<Mutex<bool>>,
+        routing_table: &drtio_routing::RoutingTable,
         timer: GlobalTimer,
         linkno: u8,
         destination: u8,
@@ -124,6 +128,7 @@ mod remote_moninj {
         let reply = drtio::aux_transact(
             aux_mutex,
             linkno,
+            routing_table,
             &drtioaux_async::Packet::InjectionStatusRequest {
                 destination: destination,
                 channel: channel as _,
@@ -135,8 +140,8 @@ mod remote_moninj {
         match reply {
             Ok(drtioaux_async::Packet::InjectionStatusReply { value }) => return value as i8,
             Ok(packet) => error!("received unexpected aux packet: {:?}", packet),
-            Err("link went down") => {
-                debug!("link is down");
+            Err(DrtioError::LinkDown) => {
+                warn!("link is down");
             }
             Err(e) => error!("aux packet error ({})", e),
         }
@@ -183,7 +188,7 @@ macro_rules! dispatch {
             local_moninj::$func(channel.into(), $($param, )*)
         } else {
             let linkno = hop - 1 as u8;
-            remote_moninj::$func($aux_mutex, $timer, linkno, destination, channel, $($param, )*).await
+            remote_moninj::$func($aux_mutex, $routing_table, $timer, linkno, destination, channel, $($param, )*).await
         }
     }}
 }

src/runtime/src/rtio_clocking.rs

@@ -6,8 +6,6 @@ use libboard_artiq::pl;
 use libboard_artiq::si5324;
 #[cfg(has_si549)]
 use libboard_artiq::si549;
-#[cfg(has_wrpll)]
-use libboard_artiq::si549::wrpll_refclk;
 #[cfg(has_si5324)]
 use libboard_zynq::i2c::I2c;
 use libboard_zynq::timer::GlobalTimer;
@@ -264,71 +262,87 @@ fn setup_si5324(i2c: &mut I2c, timer: &mut GlobalTimer, clk: RtioClock) {
     si5324::setup(i2c, &si5324_settings, si5324_ref_input, timer).expect("cannot initialize Si5324");
 }
 
-#[cfg(has_wrpll)]
-fn wrpll_setup(timer: &mut GlobalTimer, clk: RtioClock, si549_settings: si549::FrequencySetting) {
-    let mmcm_setting = match clk {
-        RtioClock::Ext0_Synth0_10to125 => si549::wrpll_refclk::MmcmSetting {
-            clkout0_reg1: 0x1083,
-            clkout0_reg2: 0x0080,
-            clkfbout_reg1: 0x179e,
-            clkfbout_reg2: 0x4c00,
-            div_reg: 0x1041,
-            lock_reg1: 0x00fa,
-            lock_reg2: 0x7c01,
-            lock_reg3: 0xffe9,
-            power_reg: 0x9900,
-            filt_reg1: 0x0808,
-            filt_reg2: 0x0800,
-        },
-        RtioClock::Ext0_Synth0_80to125 => si549::wrpll_refclk::MmcmSetting {
-            clkout0_reg1: 0x1145,
-            clkout0_reg2: 0x0000,
-            clkfbout_reg1: 0x11c7,
-            clkfbout_reg2: 0x5880,
-            div_reg: 0x1041,
-            lock_reg1: 0x028a,
-            lock_reg2: 0x7c01,
-            lock_reg3: 0xffe9,
-            power_reg: 0x9900,
-            filt_reg1: 0x0808,
-            filt_reg2: 0x9800,
-        },
-        RtioClock::Ext0_Synth0_100to125 => si549::wrpll_refclk::MmcmSetting {
-            clkout0_reg1: 0x1145,
-            clkout0_reg2: 0x0000,
-            clkfbout_reg1: 0x1145,
-            clkfbout_reg2: 0x4c00,
-            div_reg: 0x1041,
-            lock_reg1: 0x0339,
-            lock_reg2: 0x7c01,
-            lock_reg3: 0xffe9,
-            power_reg: 0x9900,
-            filt_reg1: 0x0808,
-            filt_reg2: 0x9800,
-        },
-        RtioClock::Ext0_Synth0_125to125 => si549::wrpll_refclk::MmcmSetting {
-            clkout0_reg1: 0x1145,
-            clkout0_reg2: 0x0000,
-            clkfbout_reg1: 0x1145,
-            clkfbout_reg2: 0x0000,
-            div_reg: 0x1041,
-            lock_reg1: 0x03e8,
-            lock_reg2: 0x7001,
-            lock_reg3: 0xf3e9,
-            power_reg: 0x0100,
-            filt_reg1: 0x0808,
-            filt_reg2: 0x1100,
-        },
+#[cfg(all(has_si549, has_wrpll))]
+fn wrpll_setup(timer: &mut GlobalTimer, clk: RtioClock, si549_settings: &si549::FrequencySetting) {
+    // register values are directly copied from preconfigured mmcm
+    let (mmcm_setting, mmcm_bypass) = match clk {
+        RtioClock::Ext0_Synth0_10to125 => (
+            si549::wrpll_refclk::MmcmSetting {
+                // CLKFBOUT_MULT = 62.5, DIVCLK_DIVIDE = 1 , CLKOUT0_DIVIDE = 5
+                clkout0_reg1: 0x1083,
+                clkout0_reg2: 0x0080,
+                clkfbout_reg1: 0x179e,
+                clkfbout_reg2: 0x4c00,
+                div_reg: 0x1041,
+                lock_reg1: 0x00fa,
+                lock_reg2: 0x7c01,
+                lock_reg3: 0xffe9,
+                power_reg: 0x9900,
+                filt_reg1: 0x1008,
+                filt_reg2: 0x8800,
+            },
+            false,
+        ),
+        RtioClock::Ext0_Synth0_80to125 => (
+            si549::wrpll_refclk::MmcmSetting {
+                // CLKFBOUT_MULT = 15.625, DIVCLK_DIVIDE = 1 , CLKOUT0_DIVIDE = 10
+                clkout0_reg1: 0x1145,
+                clkout0_reg2: 0x0000,
+                clkfbout_reg1: 0x11c7,
+                clkfbout_reg2: 0x5880,
+                div_reg: 0x1041,
+                lock_reg1: 0x028a,
+                lock_reg2: 0x7c01,
+                lock_reg3: 0xffe9,
+                power_reg: 0x9900,
+                filt_reg1: 0x9908,
+                filt_reg2: 0x8100,
+            },
+            false,
+        ),
+        RtioClock::Ext0_Synth0_100to125 => (
+            si549::wrpll_refclk::MmcmSetting {
+                // CLKFBOUT_MULT = 12.5, DIVCLK_DIVIDE = 1 , CLKOUT0_DIVIDE = 10
+                clkout0_reg1: 0x1145,
+                clkout0_reg2: 0x0000,
+                clkfbout_reg1: 0x1145,
+                clkfbout_reg2: 0x4c00,
+                div_reg: 0x1041,
+                lock_reg1: 0x0339,
+                lock_reg2: 0x7c01,
+                lock_reg3: 0xffe9,
+                power_reg: 0x9900,
+                filt_reg1: 0x9108,
+                filt_reg2: 0x0100,
+            },
+            false,
+        ),
+        RtioClock::Ext0_Synth0_125to125 => (
+            si549::wrpll_refclk::MmcmSetting {
+                // CLKFBOUT_MULT = 10, DIVCLK_DIVIDE = 1 , CLKOUT0_DIVIDE = 10
+                clkout0_reg1: 0x1145,
+                clkout0_reg2: 0x0000,
+                clkfbout_reg1: 0x1145,
+                clkfbout_reg2: 0x0000,
+                div_reg: 0x1041,
+                lock_reg1: 0x03e8,
+                lock_reg2: 0x7001,
+                lock_reg3: 0xf3e9,
+                power_reg: 0x0100,
+                filt_reg1: 0x9908,
+                filt_reg2: 0x1100,
+            },
+            true,
+        ),
         _ => unreachable!(),
     };
 
-    si549::main_setup(timer, si549_settings).expect("cannot initialize main Si549");
-    si549::wrpll::helper_setup(timer, si549_settings).expect("cannot initialize helper Si549");
-
-    si549::wrpll_refclk::setup(timer, mmcm_setting).expect("cannot initialize ref clk for wrpll");
-    // si549::wrpll::select_recovered_clock(true, timer);
+    si549::helper_setup(timer, &si549_settings).expect("cannot initialize helper Si549");
+    si549::wrpll_refclk::setup(timer, mmcm_setting, mmcm_bypass).expect("cannot initialize ref clk for wrpll");
+    si549::wrpll::select_recovered_clock(true, timer);
 }
 
+#[cfg(has_si549)]
 fn get_si549_setting(clk: RtioClock) -> si549::FrequencySetting {
     match clk {
         RtioClock::Ext0_Synth0_10to125 => {
@@ -343,9 +357,6 @@ fn get_si549_setting(clk: RtioClock) -> si549::FrequencySetting {
         RtioClock::Ext0_Synth0_125to125 => {
             info!("using 125MHz reference to make 125MHz RTIO clock with WRPLL");
         }
-        RtioClock::Int_150 => {
-            info!("using internal 150MHz RTIO clock");
-        }
         RtioClock::Int_100 => {
             info!("using internal 100MHz RTIO clock");
         }
@@ -368,9 +379,8 @@ fn get_si549_setting(clk: RtioClock) -> si549::FrequencySetting {
                     lsdiv: 0,
                     fbdiv: 0x046C5F49797,
                 },
-                #[cfg(has_wrpll)]
                 helper: si549::DividerConfig {
-                    // 100Mhz*32767/32768
+                    // 100MHz*32767/32768
                     hsdiv: 0x06C,
                     lsdiv: 0,
                     fbdiv: 0x046C5670BBD,
@@ -378,16 +388,15 @@ fn get_si549_setting(clk: RtioClock) -> si549::FrequencySetting {
             }
         }
         _ => {
-            // Everything else use 125Mhz
+            // Everything else use 125MHz
             si549::FrequencySetting {
                 main: si549::DividerConfig {
                     hsdiv: 0x058,
                     lsdiv: 0,
                     fbdiv: 0x04815791F25,
                 },
-                #[cfg(has_wrpll)]
                 helper: si549::DividerConfig {
-                    // 125Mhz*32767/32768
+                    // 125MHz*32767/32768
                     hsdiv: 0x058,
                     lsdiv: 0,
                     fbdiv: 0x04814E8F442,
@@ -412,22 +421,10 @@ pub fn init(timer: &mut GlobalTimer, cfg: &Config) {
     }
 
     #[cfg(has_si549)]
-    {
-        let si549_settings = get_si549_setting(clk);
+    let si549_settings = get_si549_setting(clk);
 
-        match clk {
-            RtioClock::Ext0_Synth0_10to125
-            | RtioClock::Ext0_Synth0_80to125
-            | RtioClock::Ext0_Synth0_100to125
-            | RtioClock::Ext0_Synth0_125to125 => {
-                wrpll_setup(timer, clk, si549_settings);
-            }
-            _ => {
-                wrpll_setup(timer, RtioClock::Ext0_Synth0_125to125, si549_settings);
-                // si549::main_setup(timer, si549_settings).expect("cannot initialize main Si549");
-            }
-        }
-    }
+    #[cfg(has_si549)]
+    si549::main_setup(timer, &si549_settings).expect("cannot initialize main Si549");
 
     #[cfg(has_drtio)]
     init_drtio(timer);
@@ -435,6 +432,17 @@ pub fn init(timer: &mut GlobalTimer, cfg: &Config) {
     #[cfg(not(has_drtio))]
     init_rtio(timer);
 
-    // !! move back to cfg after testing
-    si549::wrpll::select_recovered_clock(true, timer);
+    #[cfg(all(has_si549, has_wrpll))]
+    {
+        // SYS CLK switch will reset CSRs that are used by WRPLL
+        match clk {
+            RtioClock::Ext0_Synth0_10to125
+            | RtioClock::Ext0_Synth0_80to125
+            | RtioClock::Ext0_Synth0_100to125
+            | RtioClock::Ext0_Synth0_125to125 => {
+                wrpll_setup(timer, clk, &si549_settings);
+            }
+            _ => {}
+        }
+    }
 }

src/runtime/src/rtio_mgt.rs

@@ -1,19 +1,20 @@
 use alloc::rc::Rc;
 use core::cell::RefCell;
 
-use libboard_artiq::{drtio_routing, pl::csr};
+use libboard_artiq::{drtio_routing, drtio_routing::RoutingTable, pl::csr};
 use libboard_zynq::timer::GlobalTimer;
 use libcortex_a9::mutex::Mutex;
 
 #[cfg(has_drtio)]
 pub mod drtio {
     use alloc::vec::Vec;
+    use core::fmt;
 
     use embedded_hal::blocking::delay::DelayMs;
     use ksupport::{resolve_channel_name, ASYNC_ERROR_BUSY, ASYNC_ERROR_COLLISION, ASYNC_ERROR_SEQUENCE_ERROR,
                    SEEN_ASYNC_ERRORS};
     use libasync::{delay, task};
-    use libboard_artiq::{drtioaux::Error,
+    use libboard_artiq::{drtioaux::Error as DrtioError,
                          drtioaux_async,
                          drtioaux_async::Packet,
                          drtioaux_proto::{PayloadStatus, MASTER_PAYLOAD_MAX_SIZE}};
@@ -23,9 +24,44 @@ pub mod drtio {
     use super::*;
     use crate::{analyzer::remote_analyzer::RemoteBuffer, rtio_dma::remote_dma, subkernel};
 
+    #[derive(Debug, PartialEq, Eq, Clone, Copy)]
+    pub enum Error {
+        Timeout,
+        AuxError,
+        LinkDown,
+        UnexpectedReply,
+        DmaAddTraceFail(u8),
+        DmaEraseFail(u8),
+        DmaPlaybackFail(u8),
+        SubkernelAddFail(u8),
+        SubkernelRunFail(u8),
+    }
+
+    impl fmt::Display for Error {
+        fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+            match self {
+                Error::Timeout => write!(f, "timed out"),
+                Error::AuxError => write!(f, "aux packet error"),
+                Error::LinkDown => write!(f, "link down"),
+                Error::UnexpectedReply => write!(f, "unexpected reply"),
+                Error::DmaAddTraceFail(dest) => write!(f, "error adding DMA trace on satellite #{}", dest),
+                Error::DmaEraseFail(dest) => write!(f, "error erasing DMA trace on satellite #{}", dest),
+                Error::DmaPlaybackFail(dest) => write!(f, "error playing back DMA trace on satellite #{}", dest),
+                Error::SubkernelAddFail(dest) => write!(f, "error adding subkernel on satellite #{}", dest),
+                Error::SubkernelRunFail(dest) => write!(f, "error on subkernel run request on satellite #{}", dest),
+            }
+        }
+    }
+
+    impl From<DrtioError> for Error {
+        fn from(_error: DrtioError) -> Self {
+            Error::AuxError
+        }
+    }
+
     pub fn startup(
         aux_mutex: &Rc<Mutex<bool>>,
-        routing_table: &Rc<RefCell<drtio_routing::RoutingTable>>,
+        routing_table: &Rc<RefCell<RoutingTable>>,
         up_destinations: &Rc<RefCell<[bool; drtio_routing::DEST_COUNT]>>,
         timer: GlobalTimer,
     ) {
@@ -43,128 +79,104 @@ pub mod drtio {
         unsafe { (csr::DRTIO[linkno].rx_up_read)() == 1 }
     }
 
-    async fn link_has_async_ready(linkno: u8) -> bool {
-        let linkno = linkno as usize;
-        let async_ready;
-        unsafe {
-            async_ready = (csr::DRTIO[linkno].async_messages_ready_read)() == 1;
-            (csr::DRTIO[linkno].async_messages_ready_write)(1);
-        }
-        async_ready
-    }
-
-    async fn process_async_packets(
-        aux_mutex: &Mutex<bool>,
-        linkno: u8,
-        routing_table: &drtio_routing::RoutingTable,
-        timer: GlobalTimer,
-    ) {
-        if link_has_async_ready(linkno).await {
-            loop {
-                let reply = aux_transact(aux_mutex, linkno, &Packet::RoutingRetrievePackets, timer).await;
-                if let Ok(packet) = reply {
-                    match packet {
-                        Packet::DmaPlaybackStatus {
-                            id,
-                            source,
-                            destination: 0,
-                            error,
-                            channel,
-                            timestamp,
-                        } => {
-                            remote_dma::playback_done(id, source, error, channel, timestamp).await;
-                        }
-                        Packet::SubkernelFinished {
-                            id,
-                            destination: 0,
-                            with_exception,
-                            exception_src,
-                        } => {
-                            subkernel::subkernel_finished(id, with_exception, exception_src).await;
-                        }
-                        Packet::SubkernelMessage {
-                            id,
-                            source,
-                            destination: 0,
-                            status,
-                            length,
-                            data,
-                        } => {
-                            subkernel::message_handle_incoming(id, status, length as usize, &data).await;
-                            // acknowledge receiving part of the message
-                            let _lock = aux_mutex.async_lock().await;
-                            drtioaux_async::send(linkno, &Packet::SubkernelMessageAck { destination: source })
-                                .await
-                                .unwrap();
-                            let mut countdown = timer.countdown();
-                            // give the satellites some time to process
-                            delay(&mut countdown, Milliseconds(10)).await;
-                        }
-                        // routable packets
-                        Packet::DmaAddTraceRequest { destination, .. }
-                        | Packet::DmaAddTraceReply { destination, .. }
-                        | Packet::DmaRemoveTraceRequest { destination, .. }
-                        | Packet::DmaRemoveTraceReply { destination, .. }
-                        | Packet::DmaPlaybackRequest { destination, .. }
-                        | Packet::DmaPlaybackReply { destination, .. }
-                        | Packet::SubkernelLoadRunRequest { destination, .. }
-                        | Packet::SubkernelLoadRunReply { destination, .. }
-                        | Packet::SubkernelMessage { destination, .. }
-                        | Packet::SubkernelMessageAck { destination, .. }
-                        | Packet::DmaPlaybackStatus { destination, .. }
-                        | Packet::SubkernelFinished { destination, .. } => {
-                            let dest_link = routing_table.0[destination as usize][0] - 1;
-                            if dest_link == linkno {
-                                warn!(
-                                    "[LINK#{}] Re-routed packet would return to the same link, dropping: {:?}",
-                                    linkno, packet
-                                );
-                            } else if destination == 0 {
-                                warn!("[LINK#{}] Received invalid routable packet: {:?}", linkno, packet)
-                            } else {
-                                drtioaux_async::send(dest_link, &packet).await.unwrap();
-                            }
-                        }
-
-                        Packet::RoutingNoPackets => break,
-
-                        other => warn!("[LINK#{}] Received an unroutable packet: {:?}", linkno, other),
-                    }
+    async fn process_async_packets(linkno: u8, routing_table: &RoutingTable, packet: Packet) -> Option<Packet> {
+        match packet {
+            Packet::DmaPlaybackStatus {
+                id,
+                source,
+                destination: 0,
+                error,
+                channel,
+                timestamp,
+            } => {
+                remote_dma::playback_done(id, source, error, channel, timestamp).await;
+                None
+            }
+            Packet::SubkernelFinished {
+                id,
+                destination: 0,
+                with_exception,
+                exception_src,
+            } => {
+                subkernel::subkernel_finished(id, with_exception, exception_src).await;
+                None
+            }
+            Packet::SubkernelMessage {
+                id,
+                source,
+                destination: 0,
+                status,
+                length,
+                data,
+            } => {
+                subkernel::message_handle_incoming(id, status, length as usize, &data).await;
+                // acknowledge receiving part of the message
+                drtioaux_async::send(linkno, &Packet::SubkernelMessageAck { destination: source })
+                    .await
+                    .unwrap();
+                None
+            }
+            // routable packets
+            Packet::DmaAddTraceRequest { destination, .. }
+            | Packet::DmaAddTraceReply { destination, .. }
+            | Packet::DmaRemoveTraceRequest { destination, .. }
+            | Packet::DmaRemoveTraceReply { destination, .. }
+            | Packet::DmaPlaybackRequest { destination, .. }
+            | Packet::DmaPlaybackReply { destination, .. }
+            | Packet::SubkernelLoadRunRequest { destination, .. }
+            | Packet::SubkernelLoadRunReply { destination, .. }
+            | Packet::SubkernelMessage { destination, .. }
+            | Packet::SubkernelMessageAck { destination, .. }
+            | Packet::DmaPlaybackStatus { destination, .. }
+            | Packet::SubkernelFinished { destination, .. } => {
+                if destination == 0 {
+                    Some(packet)
                 } else {
-                    warn!(
-                        "[LINK#{}] Error handling async packets ({})",
-                        linkno,
-                        reply.unwrap_err()
-                    );
-                    return;
+                    let dest_link = routing_table.0[destination as usize][0] - 1;
+                    if dest_link == linkno {
+                        warn!(
+                            "[LINK#{}] Re-routed packet would return to the same link, dropping: {:?}",
+                            linkno, packet
+                        );
+                    } else {
+                        drtioaux_async::send(dest_link, &packet).await.unwrap();
+                    }
+                    None
                 }
             }
+            other => Some(other),
         }
     }
 
-    async fn recv_aux_timeout(linkno: u8, timeout: u64, timer: GlobalTimer) -> Result<Packet, &'static str> {
+    async fn recv_aux_timeout(linkno: u8, timeout: u64, timer: GlobalTimer) -> Result<Packet, Error> {
         if !link_rx_up(linkno).await {
-            return Err("link went down");
+            return Err(Error::LinkDown);
         }
         match drtioaux_async::recv_timeout(linkno, Some(timeout), timer).await {
             Ok(packet) => return Ok(packet),
-            Err(Error::TimedOut) => return Err("timed out"),
-            Err(_) => return Err("aux packet error"),
+            Err(DrtioError::TimedOut) => return Err(Error::Timeout),
+            Err(_) => return Err(Error::AuxError),
         }
     }
 
     pub async fn aux_transact(
         aux_mutex: &Mutex<bool>,
         linkno: u8,
+        routing_table: &RoutingTable,
         request: &Packet,
         timer: GlobalTimer,
-    ) -> Result<Packet, &'static str> {
+    ) -> Result<Packet, Error> {
         if !link_rx_up(linkno).await {
-            return Err("link went down");
+            return Err(Error::LinkDown);
         }
         let _lock = aux_mutex.async_lock().await;
         drtioaux_async::send(linkno, request).await.unwrap();
-        Ok(recv_aux_timeout(linkno, 200, timer).await?)
+        loop {
+            let packet = recv_aux_timeout(linkno, 200, timer).await?;
+            if let Some(packet) = process_async_packets(linkno, routing_table, packet).await {
+                return Ok(packet);
+            }
+        }
     }
 
     async fn drain_buffer(linkno: u8, draining_time: Milliseconds, timer: GlobalTimer) {
@@ -172,12 +184,17 @@ pub mod drtio {
         loop {
             if timer.get_time() > max_time {
                 return;
-            } //could this be cut short?
+            }
             let _ = drtioaux_async::recv(linkno).await;
         }
     }
 
-    async fn ping_remote(aux_mutex: &Rc<Mutex<bool>>, linkno: u8, timer: GlobalTimer) -> u32 {
+    async fn ping_remote(
+        aux_mutex: &Rc<Mutex<bool>>,
+        linkno: u8,
+        routing_table: &RoutingTable,
+        timer: GlobalTimer,
+    ) -> u32 {
         let mut count = 0;
         loop {
             if !link_rx_up(linkno).await {
@@ -187,7 +204,7 @@ pub mod drtio {
             if count > 100 {
                 return 0;
             }
-            let reply = aux_transact(aux_mutex, linkno, &Packet::EchoRequest, timer).await;
+            let reply = aux_transact(aux_mutex, linkno, routing_table, &Packet::EchoRequest, timer).await;
             match reply {
                 Ok(Packet::EchoReply) => {
                     // make sure receive buffer is drained
@@ -200,7 +217,7 @@ pub mod drtio {
         }
     }
 
-    async fn sync_tsc(aux_mutex: &Rc<Mutex<bool>>, linkno: u8, timer: GlobalTimer) -> Result<(), &'static str> {
+    async fn sync_tsc(aux_mutex: &Rc<Mutex<bool>>, linkno: u8, timer: GlobalTimer) -> Result<(), Error> {
         let _lock = aux_mutex.async_lock().await;
 
         unsafe {
@@ -211,22 +228,23 @@ pub mod drtio {
         // by the satellite, in response to a TSC set on the RT link.
         let reply = recv_aux_timeout(linkno, 10000, timer).await?;
         if reply == Packet::TSCAck {
-            return Ok(());
+            Ok(())
         } else {
-            return Err("unexpected reply");
+            Err(Error::UnexpectedReply)
         }
     }
 
     async fn load_routing_table(
         aux_mutex: &Rc<Mutex<bool>>,
         linkno: u8,
-        routing_table: &drtio_routing::RoutingTable,
+        routing_table: &RoutingTable,
         timer: GlobalTimer,
-    ) -> Result<(), &'static str> {
+    ) -> Result<(), Error> {
         for i in 0..drtio_routing::DEST_COUNT {
             let reply = aux_transact(
                 aux_mutex,
                 linkno,
+                routing_table,
                 &Packet::RoutingSetPath {
                     destination: i as u8,
                     hops: routing_table.0[i],
@@ -235,7 +253,7 @@ pub mod drtio {
             )
             .await?;
             if reply != Packet::RoutingAck {
-                return Err("unexpected reply");
+                return Err(Error::UnexpectedReply);
             }
         }
         Ok(())
@@ -245,13 +263,21 @@ pub mod drtio {
         aux_mutex: &Rc<Mutex<bool>>,
         linkno: u8,
         rank: u8,
+        routing_table: &RoutingTable,
         timer: GlobalTimer,
-    ) -> Result<(), &'static str> {
-        let reply = aux_transact(aux_mutex, linkno, &Packet::RoutingSetRank { rank: rank }, timer).await?;
-        if reply != Packet::RoutingAck {
-            return Err("unexpected reply");
+    ) -> Result<(), Error> {
+        let reply = aux_transact(
+            aux_mutex,
+            linkno,
+            routing_table,
+            &Packet::RoutingSetRank { rank: rank },
+            timer,
+        )
+        .await?;
+        match reply {
+            Packet::RoutingAck => Ok(()),
+            _ => Err(Error::UnexpectedReply),
         }
-        Ok(())
     }
 
     async fn init_buffer_space(destination: u8, linkno: u8) {
@@ -270,10 +296,14 @@ pub mod drtio {
         }
     }
 
-    async fn process_unsolicited_aux(aux_mutex: &Rc<Mutex<bool>>, linkno: u8) {
+    async fn process_unsolicited_aux(aux_mutex: &Mutex<bool>, linkno: u8, routing_table: &RoutingTable) {
         let _lock = aux_mutex.async_lock().await;
         match drtioaux_async::recv(linkno).await {
-            Ok(Some(packet)) => warn!("[LINK#{}] unsolicited aux packet: {:?}", linkno, packet),
+            Ok(Some(packet)) => {
+                if let Some(packet) = process_async_packets(linkno, routing_table, packet).await {
+                    warn!("[LINK#{}] unsolicited aux packet: {:?}", linkno, packet);
+                }
+            }
             Ok(None) => (),
             Err(_) => warn!("[LINK#{}] aux packet error", linkno),
         }
@@ -301,7 +331,7 @@ pub mod drtio {
     }
 
     async fn destination_set_up(
-        routing_table: &drtio_routing::RoutingTable,
+        routing_table: &RoutingTable,
         up_destinations: &Rc<RefCell<[bool; drtio_routing::DEST_COUNT]>>,
         destination: u8,
         up: bool,
@@ -324,7 +354,7 @@ pub mod drtio {
 
     async fn destination_survey(
         aux_mutex: &Rc<Mutex<bool>>,
-        routing_table: &drtio_routing::RoutingTable,
+        routing_table: &RoutingTable,
         up_links: &[bool],
         up_destinations: &Rc<RefCell<[bool; drtio_routing::DEST_COUNT]>>,
         timer: GlobalTimer,
@@ -345,6 +375,7 @@ pub mod drtio {
                         let reply = aux_transact(
                             aux_mutex,
                             linkno,
+                            routing_table,
                             &Packet::DestinationStatusRequest {
                                 destination: destination,
                             },
@@ -400,6 +431,7 @@ pub mod drtio {
                         let reply = aux_transact(
                             aux_mutex,
                             linkno,
+                            routing_table,
                             &Packet::DestinationStatusRequest {
                                 destination: destination,
                             },
@@ -427,7 +459,7 @@ pub mod drtio {
 
     pub async fn link_task(
         aux_mutex: &Rc<Mutex<bool>>,
-        routing_table: &drtio_routing::RoutingTable,
+        routing_table: &RoutingTable,
         up_destinations: &Rc<RefCell<[bool; drtio_routing::DEST_COUNT]>>,
         timer: GlobalTimer,
     ) {
@@ -438,8 +470,7 @@ pub mod drtio {
                 if up_links[linkno as usize] {
                     /* link was previously up */
                     if link_rx_up(linkno).await {
-                        process_async_packets(aux_mutex, linkno, routing_table, timer).await;
-                        process_unsolicited_aux(aux_mutex, linkno).await;
+                        process_unsolicited_aux(aux_mutex, linkno, routing_table).await;
                         process_local_errors(linkno).await;
                     } else {
                         info!("[LINK#{}] link is down", linkno);
@@ -449,7 +480,7 @@ pub mod drtio {
                     /* link was previously down */
                     if link_rx_up(linkno).await {
                         info!("[LINK#{}] link RX became up, pinging", linkno);
-                        let ping_count = ping_remote(aux_mutex, linkno, timer).await;
+                        let ping_count = ping_remote(aux_mutex, linkno, routing_table, timer).await;
                         if ping_count > 0 {
                             info!("[LINK#{}] remote replied after {} packets", linkno, ping_count);
                             up_links[linkno as usize] = true;
@@ -459,7 +490,7 @@ pub mod drtio {
                             if let Err(e) = load_routing_table(aux_mutex, linkno, routing_table, timer).await {
                                 error!("[LINK#{}] failed to load routing table ({})", linkno, e);
                             }
-                            if let Err(e) = set_rank(aux_mutex, linkno, 1 as u8, timer).await {
+                            if let Err(e) = set_rank(aux_mutex, linkno, 1 as u8, routing_table, timer).await {
                                 error!("[LINK#{}] failed to set rank ({})", linkno, e);
                             }
                             info!("[LINK#{}] link initialization completed", linkno);
@@ -476,7 +507,7 @@ pub mod drtio {
     }
 
     #[allow(dead_code)]
-    pub fn reset(aux_mutex: Rc<Mutex<bool>>, mut timer: GlobalTimer) {
+    pub fn reset(aux_mutex: Rc<Mutex<bool>>, routing_table: &RoutingTable, mut timer: GlobalTimer) {
         for linkno in 0..csr::DRTIO.len() {
             unsafe {
                 (csr::DRTIO[linkno].reset_write)(1);
@@ -492,7 +523,13 @@ pub mod drtio {
         for linkno in 0..csr::DRTIO.len() {
             let linkno = linkno as u8;
             if task::block_on(link_rx_up(linkno)) {
-                let reply = task::block_on(aux_transact(&aux_mutex, linkno, &Packet::ResetRequest, timer));
+                let reply = task::block_on(aux_transact(
+                    &aux_mutex,
+                    linkno,
+                    routing_table,
+                    &Packet::ResetRequest,
+                    timer,
+                ));
                 match reply {
                     Ok(Packet::ResetAck) => (),
                     Ok(_) => error!("[LINK#{}] reset failed, received unexpected aux packet", linkno),
@@ -505,14 +542,15 @@ pub mod drtio {
     async fn partition_data<PacketF, HandlerF>(
         linkno: u8,
         aux_mutex: &Rc<Mutex<bool>>,
+        routing_table: &RoutingTable,
         timer: GlobalTimer,
         data: &[u8],
         packet_f: PacketF,
         reply_handler_f: HandlerF,
-    ) -> Result<(), &'static str>
+    ) -> Result<(), Error>
     where
         PacketF: Fn(&[u8; MASTER_PAYLOAD_MAX_SIZE], PayloadStatus, usize) -> Packet,
-        HandlerF: Fn(&Packet) -> Result<(), &'static str>,
+        HandlerF: Fn(&Packet) -> Result<(), Error>,
     {
         let mut i = 0;
         while i < data.len() {
@@ -528,7 +566,7 @@ pub mod drtio {
             i += len;
             let status = PayloadStatus::from_status(first, last);
             let packet = packet_f(&slice, status, len);
-            let reply = aux_transact(aux_mutex, linkno, &packet, timer).await?;
+            let reply = aux_transact(aux_mutex, linkno, routing_table, &packet, timer).await?;
             reply_handler_f(&reply)?;
         }
         Ok(())
@@ -536,16 +574,17 @@ pub mod drtio {
 
     pub async fn ddma_upload_trace(
         aux_mutex: &Rc<Mutex<bool>>,
-        routing_table: &drtio_routing::RoutingTable,
+        routing_table: &RoutingTable,
         timer: GlobalTimer,
         id: u32,
         destination: u8,
         trace: &Vec<u8>,
-    ) -> Result<(), &'static str> {
+    ) -> Result<(), Error> {
         let linkno = routing_table.0[destination as usize][0] - 1;
         partition_data(
             linkno,
             aux_mutex,
+            routing_table,
             timer,
             trace,
             |slice, status, len| Packet::DmaAddTraceRequest {
@@ -566,8 +605,8 @@ pub mod drtio {
                     destination: 0,
                     succeeded: false,
                     ..
-                } => Err("error adding trace on satellite"),
-                _ => Err("adding DMA trace failed, unexpected aux packet"),
+                } => Err(Error::DmaAddTraceFail(destination)),
+                _ => Err(Error::UnexpectedReply),
             },
         )
         .await
@@ -575,15 +614,16 @@ pub mod drtio {
 
     pub async fn ddma_send_erase(
         aux_mutex: &Rc<Mutex<bool>>,
-        routing_table: &drtio_routing::RoutingTable,
+        routing_table: &RoutingTable,
         timer: GlobalTimer,
         id: u32,
         destination: u8,
-    ) -> Result<(), &'static str> {
+    ) -> Result<(), Error> {
         let linkno = routing_table.0[destination as usize][0] - 1;
         let reply = aux_transact(
             aux_mutex,
             linkno,
+            routing_table,
             &Packet::DmaRemoveTraceRequest {
                 id: id,
                 source: 0,
@@ -591,33 +631,33 @@ pub mod drtio {
             },
             timer,
         )
-        .await;
+        .await?;
         match reply {
-            Ok(Packet::DmaRemoveTraceReply {
+            Packet::DmaRemoveTraceReply {
                 destination: 0,
                 succeeded: true,
-            }) => Ok(()),
-            Ok(Packet::DmaRemoveTraceReply {
+            } => Ok(()),
+            Packet::DmaRemoveTraceReply {
                 destination: 0,
                 succeeded: false,
-            }) => Err("satellite DMA erase error"),
-            Ok(_) => Err("adding trace failed, unexpected aux packet"),
-            Err(_) => Err("erasing trace failed, aux error"),
+            } => Err(Error::DmaEraseFail(destination)),
+            _ => Err(Error::UnexpectedReply),
         }
     }
 
     pub async fn ddma_send_playback(
         aux_mutex: &Rc<Mutex<bool>>,
-        routing_table: &drtio_routing::RoutingTable,
+        routing_table: &RoutingTable,
         timer: GlobalTimer,
         id: u32,
         destination: u8,
         timestamp: u64,
-    ) -> Result<(), &'static str> {
+    ) -> Result<(), Error> {
         let linkno = routing_table.0[destination as usize][0] - 1;
         let reply = aux_transact(
             aux_mutex,
             linkno,
+            routing_table,
             &Packet::DmaPlaybackRequest {
                 id: id,
                 source: 0,
@@ -626,45 +666,44 @@ pub mod drtio {
             },
             timer,
         )
-        .await;
+        .await?;
         match reply {
-            Ok(Packet::DmaPlaybackReply {
+            Packet::DmaPlaybackReply {
                 destination: 0,
                 succeeded: true,
-            }) => Ok(()),
-            Ok(Packet::DmaPlaybackReply {
+            } => Ok(()),
+            Packet::DmaPlaybackReply {
                 destination: 0,
                 succeeded: false,
-            }) => Err("error on DMA playback request"),
-            Ok(_) => Err("received unexpected aux packet during DMA playback"),
-            Err(_) => Err("aux error on DMA playback"),
+            } => Err(Error::DmaPlaybackFail(destination)),
+            _ => Err(Error::UnexpectedReply),
         }
     }
 
     async fn analyzer_get_data(
         aux_mutex: &Rc<Mutex<bool>>,
-        routing_table: &drtio_routing::RoutingTable,
+        routing_table: &RoutingTable,
         timer: GlobalTimer,
         destination: u8,
-    ) -> Result<RemoteBuffer, &'static str> {
+    ) -> Result<RemoteBuffer, Error> {
         let linkno = routing_table.0[destination as usize][0] - 1;
         let reply = aux_transact(
             aux_mutex,
             linkno,
+            routing_table,
             &Packet::AnalyzerHeaderRequest {
                 destination: destination,
             },
             timer,
         )
-        .await;
+        .await?;
         let (sent, total, overflow) = match reply {
-            Ok(Packet::AnalyzerHeader {
+            Packet::AnalyzerHeader {
                 sent_bytes,
                 total_byte_count,
                 overflow_occurred,
-            }) => (sent_bytes, total_byte_count, overflow_occurred),
-            Ok(_) => return Err("received unexpected aux packet during remote analyzer header request"),
-            Err(e) => return Err(e),
+            } => (sent_bytes, total_byte_count, overflow_occurred),
+            _ => return Err(Error::UnexpectedReply),
         };
 
         let mut remote_data: Vec<u8> = Vec::new();
@@ -674,19 +713,19 @@ pub mod drtio {
                 let reply = aux_transact(
                     aux_mutex,
                     linkno,
+                    routing_table,
                     &Packet::AnalyzerDataRequest {
                         destination: destination,
                     },
                     timer,
                 )
-                .await;
+                .await?;
                 match reply {
-                    Ok(Packet::AnalyzerData { last, length, data }) => {
+                    Packet::AnalyzerData { last, length, data } => {
                         last_packet = last;
                         remote_data.extend(&data[0..length as usize]);
                     }
-                    Ok(_) => return Err("received unexpected aux packet during remote analyzer data request"),
-                    Err(e) => return Err(e),
+                    _ => return Err(Error::UnexpectedReply),
                 }
             }
         }
@@ -701,10 +740,10 @@ pub mod drtio {
 
     pub async fn analyzer_query(
         aux_mutex: &Rc<Mutex<bool>>,
-        routing_table: &drtio_routing::RoutingTable,
+        routing_table: &RoutingTable,
         up_destinations: &Rc<RefCell<[bool; drtio_routing::DEST_COUNT]>>,
         timer: GlobalTimer,
-    ) -> Result<Vec<RemoteBuffer>, &'static str> {
+    ) -> Result<Vec<RemoteBuffer>, Error> {
         let mut remote_buffers: Vec<RemoteBuffer> = Vec::new();
         for i in 1..drtio_routing::DEST_COUNT {
             if destination_up(up_destinations, i as u8).await {
@@ -716,16 +755,17 @@ pub mod drtio {
 
     pub async fn subkernel_upload(
         aux_mutex: &Rc<Mutex<bool>>,
-        routing_table: &drtio_routing::RoutingTable,
+        routing_table: &RoutingTable,
         timer: GlobalTimer,
         id: u32,
         destination: u8,
         data: &Vec<u8>,
-    ) -> Result<(), &'static str> {
+    ) -> Result<(), Error> {
         let linkno = routing_table.0[destination as usize][0] - 1;
         partition_data(
             linkno,
             aux_mutex,
+            routing_table,
             timer,
             data,
             |slice, status, len| Packet::SubkernelAddDataRequest {
@@ -737,8 +777,8 @@ pub mod drtio {
             },
             |reply| match reply {
                 Packet::SubkernelAddDataReply { succeeded: true } => Ok(()),
-                Packet::SubkernelAddDataReply { succeeded: false } => Err("error adding subkernel on satellite"),
-                _ => Err("adding subkernel failed, unexpected aux packet"),
+                Packet::SubkernelAddDataReply { succeeded: false } => Err(Error::SubkernelAddFail(destination)),
+                _ => Err(Error::UnexpectedReply),
             },
         )
         .await
@@ -746,16 +786,17 @@ pub mod drtio {
 
     pub async fn subkernel_load(
         aux_mutex: &Rc<Mutex<bool>>,
-        routing_table: &drtio_routing::RoutingTable,
+        routing_table: &RoutingTable,
         timer: GlobalTimer,
         id: u32,
         destination: u8,
         run: bool,
-    ) -> Result<(), &'static str> {
+    ) -> Result<(), Error> {
         let linkno = routing_table.0[destination as usize][0] - 1;
         let reply = aux_transact(
             aux_mutex,
             linkno,
+            routing_table,
             &Packet::SubkernelLoadRunRequest {
                 id: id,
                 source: 0,
@@ -773,23 +814,24 @@ pub mod drtio {
             Packet::SubkernelLoadRunReply {
                 destination: 0,
                 succeeded: false,
-            } => return Err("error on subkernel run request"),
-            _ => return Err("received unexpected aux packet during subkernel run"),
+            } => return Err(Error::SubkernelRunFail(destination)),
+            _ => Err(Error::UnexpectedReply),
         }
     }
 
     pub async fn subkernel_retrieve_exception(
         aux_mutex: &Rc<Mutex<bool>>,
-        routing_table: &drtio_routing::RoutingTable,
+        routing_table: &RoutingTable,
         timer: GlobalTimer,
         destination: u8,
-    ) -> Result<Vec<u8>, &'static str> {
+    ) -> Result<Vec<u8>, Error> {
         let linkno = routing_table.0[destination as usize][0] - 1;
         let mut remote_data: Vec<u8> = Vec::new();
         loop {
             let reply = aux_transact(
                 aux_mutex,
                 linkno,
+                routing_table,
                 &Packet::SubkernelExceptionRequest {
                     destination: destination,
                 },
@@ -803,23 +845,24 @@ pub mod drtio {
                         return Ok(remote_data);
                     }
                 }
-                _ => return Err("received unexpected aux packet during subkernel exception request"),
+                _ => return Err(Error::UnexpectedReply),
             }
         }
     }
 
     pub async fn subkernel_send_message(
         aux_mutex: &Rc<Mutex<bool>>,
-        routing_table: &drtio_routing::RoutingTable,
+        routing_table: &RoutingTable,
         timer: GlobalTimer,
         id: u32,
         destination: u8,
         message: &[u8],
-    ) -> Result<(), &'static str> {
+    ) -> Result<(), Error> {
         let linkno = routing_table.0[destination as usize][0] - 1;
         partition_data(
             linkno,
             aux_mutex,
+            routing_table,
             timer,
             message,
             |slice, status, len| Packet::SubkernelMessage {
@@ -832,7 +875,7 @@ pub mod drtio {
             },
             |reply| match reply {
                 Packet::SubkernelMessageAck { .. } => Ok(()),
-                _ => Err("sending message to subkernel failed, unexpected aux packet"),
+                _ => Err(Error::UnexpectedReply),
             },
         )
         .await
@@ -845,19 +888,19 @@ pub mod drtio {
 
     pub fn startup(
         _aux_mutex: &Rc<Mutex<bool>>,
-        _routing_table: &Rc<RefCell<drtio_routing::RoutingTable>>,
+        _routing_table: &Rc<RefCell<RoutingTable>>,
         _up_destinations: &Rc<RefCell<[bool; drtio_routing::DEST_COUNT]>>,
         _timer: GlobalTimer,
     ) {
     }
 
     #[allow(dead_code)]
-    pub fn reset(_aux_mutex: Rc<Mutex<bool>>, mut _timer: GlobalTimer) {}
+    pub fn reset(_aux_mutex: Rc<Mutex<bool>>, _routing_table: &RoutingTable, mut _timer: GlobalTimer) {}
 }
 
 pub fn startup(
     aux_mutex: &Rc<Mutex<bool>>,
-    routing_table: &Rc<RefCell<drtio_routing::RoutingTable>>,
+    routing_table: &Rc<RefCell<RoutingTable>>,
     up_destinations: &Rc<RefCell<[bool; drtio_routing::DEST_COUNT]>>,
     timer: GlobalTimer,
 ) {
@@ -868,9 +911,9 @@ pub fn startup(
 }
 
 #[allow(dead_code)]
-pub fn reset(aux_mutex: Rc<Mutex<bool>>, timer: GlobalTimer) {
+pub fn reset(aux_mutex: Rc<Mutex<bool>>, routing_table: &RoutingTable, timer: GlobalTimer) {
     unsafe {
         csr::rtio_core::reset_write(1);
     }
-    drtio::reset(aux_mutex, timer)
+    drtio::reset(aux_mutex, routing_table, timer)
 }

src/runtime/src/subkernel.rs

@@ -7,7 +7,7 @@ use libboard_zynq::{time::Milliseconds, timer::GlobalTimer};
 use libcortex_a9::mutex::Mutex;
 use log::{error, warn};
 
-use crate::rtio_mgt::drtio;
+use crate::rtio_mgt::{drtio, drtio::Error as DrtioError};
 
 #[derive(Debug, PartialEq, Clone, Copy)]
 pub enum FinishStatus {
@@ -31,11 +31,11 @@ pub enum Error {
     SubkernelNotFound,
     SubkernelException,
     CommLost,
-    DrtioError(&'static str),
+    DrtioError(DrtioError),
 }
 
-impl From<&'static str> for Error {
-    fn from(value: &'static str) -> Error {
+impl From<DrtioError> for Error {
+    fn from(value: DrtioError) -> Error {
         Error::DrtioError(value)
     }
 }

src/satman/Cargo.toml.tpl

@@ -7,6 +7,7 @@ build = "build.rs"
 [features]
 target_zc706 = ["libboard_zynq/target_zc706", "libsupport_zynq/target_zc706", "libconfig/target_zc706", "libboard_artiq/target_zc706"]
 target_kasli_soc = ["libboard_zynq/target_kasli_soc", "libsupport_zynq/target_kasli_soc", "libconfig/target_kasli_soc", "libboard_artiq/target_kasli_soc"]
+calibrate_wrpll_skew = ["libboard_artiq/calibrate_wrpll_skew"]
 default = ["target_zc706", ]
 
 [build-dependencies]

src/satman/src/main.rs

@@ -40,7 +40,7 @@ use libboard_zynq::error_led::ErrorLED;
 use libboard_zynq::{i2c::I2c, print, println, time::Milliseconds, timer::GlobalTimer};
 use libcortex_a9::{l2c::enable_l2_cache, regs::MPIDR};
 use libregister::RegisterR;
-use libsupport_zynq::ram;
+use libsupport_zynq::{exception_vectors, ram};
 use routing::Router;
 use subkernel::Manager as KernelManager;
 
@@ -50,6 +50,11 @@ mod repeater;
 mod routing;
 mod subkernel;
 
+// linker symbols
+extern "C" {
+    static __exceptions_start: u32;
+}
+
 fn drtiosat_reset(reset: bool) {
     unsafe {
         csr::drtiosat::reset_write(if reset { 1 } else { 0 });
@@ -76,12 +81,6 @@ fn drtiosat_tsc_loaded() -> bool {
     }
 }
 
-fn drtiosat_async_ready() {
-    unsafe {
-        csr::drtiosat::async_messages_ready_write(1);
-    }
-}
-
 #[cfg(has_drtio_routing)]
 macro_rules! forward {
     ($routing_table:expr, $destination:expr, $rank:expr, $repeaters:expr, $packet:expr, $timer:expr) => {{
@@ -240,14 +239,6 @@ fn process_aux_packet(
         #[cfg(not(has_drtio_routing))]
         drtioaux::Packet::RoutingSetRank { rank: _ } => drtioaux::send(0, &drtioaux::Packet::RoutingAck),
 
-        drtioaux::Packet::RoutingRetrievePackets => {
-            let packet = router
-                .get_upstream_packet()
-                .or(Some(drtioaux::Packet::RoutingNoPackets))
-                .unwrap();
-            drtioaux::send(0, &packet)
-        }
-
         drtioaux::Packet::MonitorRequest {
             destination: _destination,
             channel,
@@ -832,9 +823,8 @@ const SI549_SETTINGS: si549::FrequencySetting = si549::FrequencySetting {
         lsdiv: 0,
         fbdiv: 0x04815791F25,
     },
-    #[cfg(has_wrpll)]
     helper: si549::DividerConfig {
-        // 125Mhz*32767/32768
+        // 125MHz*32767/32768
         hsdiv: 0x058,
         lsdiv: 0,
         fbdiv: 0x04814E8F442,
@@ -842,15 +832,14 @@ const SI549_SETTINGS: si549::FrequencySetting = si549::FrequencySetting {
 };
 
 #[cfg(all(has_si549, rtio_frequency = "100.0"))]
-pub const SI549_SETTINGS: si549::FrequencySetting = si549::FrequencySetting {
+const SI549_SETTINGS: si549::FrequencySetting = si549::FrequencySetting {
     main: si549::DividerConfig {
         hsdiv: 0x06C,
         lsdiv: 0,
         fbdiv: 0x046C5F49797,
     },
-    #[cfg(has_wrpll)]
     helper: si549::DividerConfig {
-        // 100Mhz*32767/32768
+        // 100MHz*32767/32768
         hsdiv: 0x06C,
         lsdiv: 0,
         fbdiv: 0x046C5670BBD,
@@ -861,6 +850,9 @@ static mut LOG_BUFFER: [u8; 1 << 17] = [0; 1 << 17];
 
 #[no_mangle]
 pub extern "C" fn main_core0() -> i32 {
+    unsafe {
+        exception_vectors::set_vector_table(&__exceptions_start as *const u32 as u32);
+    }
     enable_l2_cache(0x8);
 
     let mut timer = GlobalTimer::start();
@@ -907,7 +899,7 @@ pub extern "C" fn main_core0() -> i32 {
     #[cfg(has_si5324)]
     si5324::setup(&mut i2c, &SI5324_SETTINGS, si5324::Input::Ckin1, &mut timer).expect("cannot initialize Si5324");
     #[cfg(has_si549)]
-    si549::main_setup(&mut timer, SI549_SETTINGS).expect("cannot initialize main Si549");
+    si549::main_setup(&mut timer, &SI549_SETTINGS).expect("cannot initialize main Si549");
 
     timer.delay_us(100_000);
     info!("Switching SYS clocks...");
@@ -925,8 +917,8 @@ pub extern "C" fn main_core0() -> i32 {
     unsafe {
         csr::gt_drtio::txenable_write(0xffffffffu32 as _);
     }
-    #[cfg(has_wrpll)]
-    si549::wrpll::helper_setup(&mut timer, SI549_SETTINGS).expect("cannot initialize helper Si549");
+    #[cfg(has_si549)]
+    si549::helper_setup(&mut timer, &SI549_SETTINGS).expect("cannot initialize helper Si549");
 
     #[cfg(has_drtio_routing)]
     let mut repeaters = [repeater::Repeater::default(); csr::DRTIOREP.len()];
@@ -1061,8 +1053,8 @@ pub extern "C" fn main_core0() -> i32 {
                 }
             }
 
-            if router.any_upstream_waiting() {
-                drtiosat_async_ready();
+            if let Some(packet) = router.get_upstream_packet() {
+                drtioaux::send(0, &packet).unwrap();
             }
         }
 

src/satman/src/repeater.rs

@@ -119,16 +119,11 @@ impl Repeater {
                 }
             }
             RepeaterState::Up => {
-                self.process_unsolicited_aux();
+                self.process_unsolicited_aux(routing_table, rank, destination, router);
                 if !rep_link_rx_up(self.repno) {
                     info!("[REP#{}] link is down", self.repno);
                     self.state = RepeaterState::Down;
                 }
-                if self.async_messages_ready() {
-                    if let Err(e) = self.handle_async(routing_table, rank, destination, router, timer) {
-                        warn!("[REP#{}] Error handling async messages ({:?})", self.repno, e);
-                    }
-                }
             }
             RepeaterState::Failed => {
                 if !rep_link_rx_up(self.repno) {
@@ -139,9 +134,15 @@ impl Repeater {
         }
     }
 
-    fn process_unsolicited_aux(&self) {
+    fn process_unsolicited_aux(
+        &self,
+        routing_table: &drtio_routing::RoutingTable,
+        rank: u8,
+        destination: u8,
+        router: &mut Router,
+    ) {
         match drtioaux::recv(self.auxno) {
-            Ok(Some(packet)) => warn!("[REP#{}] unsolicited aux packet: {:?}", self.repno, packet),
+            Ok(Some(packet)) => router.route(packet, routing_table, rank, destination),
             Ok(None) => (),
             Err(_) => warn!("[REP#{}] aux packet error", self.repno),
         }
@@ -186,34 +187,6 @@ impl Repeater {
         }
     }
 
-    fn async_messages_ready(&self) -> bool {
-        let async_rdy;
-        unsafe {
-            async_rdy = (csr::DRTIOREP[self.repno as usize].async_messages_ready_read)();
-            (csr::DRTIOREP[self.repno as usize].async_messages_ready_write)(0);
-        }
-        async_rdy == 1
-    }
-
-    fn handle_async(
-        &self,
-        routing_table: &drtio_routing::RoutingTable,
-        rank: u8,
-        self_destination: u8,
-        router: &mut Router,
-        timer: &mut GlobalTimer,
-    ) -> Result<(), drtioaux::Error> {
-        loop {
-            drtioaux::send(self.auxno, &drtioaux::Packet::RoutingRetrievePackets).unwrap();
-            let reply = self.recv_aux_timeout(200, timer)?;
-            match reply {
-                drtioaux::Packet::RoutingNoPackets => break,
-                packet => router.route(packet, routing_table, rank, self_destination),
-            }
-        }
-        Ok(())
-    }
-
     fn recv_aux_timeout(&self, timeout: u32, timer: &mut GlobalTimer) -> Result<drtioaux::Packet, drtioaux::Error> {
         let max_time = timer.get_time() + Milliseconds(timeout.into());
         loop {

src/satman/src/routing.rs

@@ -75,7 +75,6 @@ pub struct Router {
     local_queue: VecDeque<drtioaux::Packet>,
     #[cfg(has_drtio_routing)]
     downstream_queue: VecDeque<(usize, drtioaux::Packet)>,
-    upstream_notified: bool,
 }
 
 impl Router {
@@ -85,7 +84,6 @@ impl Router {
             local_queue: VecDeque::new(),
             #[cfg(has_drtio_routing)]
             downstream_queue: VecDeque::new(),
-            upstream_notified: false,
         }
     }
 
@@ -161,22 +159,8 @@ impl Router {
         }
     }
 
-    pub fn any_upstream_waiting(&mut self) -> bool {
-        let empty = self.upstream_queue.is_empty();
-        if !empty && !self.upstream_notified {
-            self.upstream_notified = true; // so upstream will not get spammed with notifications
-            true
-        } else {
-            false
-        }
-    }
-
     pub fn get_upstream_packet(&mut self) -> Option<drtioaux::Packet> {
-        let packet = self.upstream_queue.pop_front();
-        if packet.is_none() {
-            self.upstream_notified = false;
-        }
-        packet
+        self.upstream_queue.pop_front()
     }
 
     #[cfg(has_drtio_routing)]