From 0d431cb019619487cd56a364060b3fb789bc8c93 Mon Sep 17 00:00:00 2001
From: Robert Jordens <rj@m-labs.hk>
Date: Tue, 8 Mar 2016 17:07:44 +0100
Subject: [PATCH 01/38] pipistrello: make pmod extension header, cleanup

---
 artiq/gateware/targets/pipistrello.py | 62 ++++++++++++++++++---------
 1 file changed, 41 insertions(+), 21 deletions(-)

diff --git a/artiq/gateware/targets/pipistrello.py b/artiq/gateware/targets/pipistrello.py
index 8cd6b22ee..d9dcb3bcc 100755
--- a/artiq/gateware/targets/pipistrello.py
+++ b/artiq/gateware/targets/pipistrello.py
@@ -9,11 +9,10 @@ from fractions import Fraction
 from migen import *
 from migen.genlib.resetsync import AsyncResetSynchronizer
 from migen.genlib.cdc import MultiReg
+from migen.build.generic_platform import *
 
 from misoc.interconnect.csr import *
-from misoc.interconnect import wishbone
 from misoc.cores import gpio
-from misoc.integration.soc_core import mem_decoder
 from misoc.targets.pipistrello import (BaseSoC, soc_pipistrello_args,
                                        soc_pipistrello_argdict)
 from misoc.integration.builder import builder_args, builder_argdict
@@ -24,6 +23,28 @@ from artiq.gateware.rtio.phy import ttl_simple, ttl_serdes_spartan6, dds, spi
 from artiq import __version__ as artiq_version
 
 
+_pmod_spi = [
+    ("pmod_spi", 0,
+     Subsignal("cs_n", Pins("PMOD:0")),
+     Subsignal("mosi", Pins("PMOD:1")),
+     Subsignal("miso", Pins("PMOD:2")),
+     Subsignal("clk", Pins("PMOD:3")),
+     IOStandard("LVTTL")
+     ),
+    ("pmod_extended_spi", 0,
+     Subsignal("cs_n", Pins("PMOD:0")),
+     Subsignal("mosi", Pins("PMOD:1")),
+     Subsignal("miso", Pins("PMOD:2")),
+     Subsignal("clk", Pins("PMOD:3")),
+     Subsignal("int", Pins("PMOD:4")),
+     Subsignal("rst", Pins("PMOD:5")),
+     Subsignal("d0", Pins("PMOD:6")),
+     Subsignal("d1", Pins("PMOD:7")),
+     IOStandard("LVTTL")
+     ),
+]
+
+
 class _RTIOCRG(Module, AutoCSR):
     def __init__(self, platform, clk_freq):
         self._clock_sel = CSRStorage()
@@ -88,7 +109,8 @@ class _RTIOCRG(Module, AutoCSR):
 
         # ISE infers correct period constraints for cd_rtio.clk from
         # the internal clock. The first two TIGs target just the BUFGMUX.
-        platform.add_platform_command("""
+        platform.add_platform_command(
+            """
 NET "sys_clk" TNM_NET = "GRPsys_clk";
 NET "{ext_clk}" TNM_NET = "GRPext_clk";
 TIMESPEC "TSfix_ise1" = FROM "GRPsys_clk" TO "GRPext_clk" TIG;
@@ -97,8 +119,9 @@ TIMESPEC "TSfix_ise2" = FROM "GRPsys_clk" TO "GRPint_clk" TIG;
 NET "{rtio_clk}" TNM_NET = "GRPrtio_clk";
 TIMESPEC "TSfix_ise3" = FROM "GRPrtio_clk" TO "GRPsys_clk" TIG;
 TIMESPEC "TSfix_ise4" = FROM "GRPsys_clk" TO "GRPrtio_clk" TIG;
-""", ext_clk=rtio_external_clk, int_clk=rtio_internal_clk,
-     rtio_clk=self.cd_rtio.clk)
+""",
+            ext_clk=rtio_external_clk, int_clk=rtio_internal_clk,
+            rtio_clk=self.cd_rtio.clk)
 
 
 class NIST_QC1(BaseSoC, AMPSoC):
@@ -112,9 +135,9 @@ class NIST_QC1(BaseSoC, AMPSoC):
     }
     csr_map.update(BaseSoC.csr_map)
     mem_map = {
-        "timer_kernel":  0x10000000, # (shadow @0x90000000)
-        "rtio":          0x20000000, # (shadow @0xa0000000)
-        "mailbox":       0x70000000  # (shadow @0xf0000000)
+        "timer_kernel":  0x10000000,  # (shadow @0x90000000)
+        "rtio":          0x20000000,  # (shadow @0xa0000000)
+        "mailbox":       0x70000000   # (shadow @0xf0000000)
     }
     mem_map.update(BaseSoC.mem_map)
 
@@ -134,6 +157,7 @@ class NIST_QC1(BaseSoC, AMPSoC):
 trce -v 12 -fastpaths -tsi {build_name}.tsi -o {build_name}.twr {build_name}.ncd {build_name}.pcf
 """
         platform.add_extension(nist_qc1.papilio_adapter_io)
+        platform.add_extension(_pmod_spi)
 
         self.submodules.leds = gpio.GPIOOut(platform.request("user_led", 4))
 
@@ -177,13 +201,14 @@ trce -v 12 -fastpaths -tsi {build_name}.tsi -o {build_name}.twr {build_name}.ncd
             self.submodules += phy
             rtio_channels.append(rtio.Channel.from_phy(phy, ofifo_depth=4))
 
-        pmod = self.platform.request("pmod", 0)
+        spi_pins = self.platform.request("pmod_extended_spi", 0)
 
-        for i in range(4, 8):
-            phy = ttl_simple.Inout(pmod.d[i])
+        for i, p in enumerate((spi_pins.int, spi_pins.rst,
+                               spi_pins.d0, spi_pins.d1)):
+            phy = ttl_simple.Inout(p)
             self.submodules += phy
-            rtio_channels.append(rtio.Channel.from_phy(phy, ififo_depth=128,
-                                                       ofifo_depth=128))
+            rtio_channels.append(rtio.Channel.from_phy(phy, ififo_depth=4,
+                                                       ofifo_depth=4))
 
         self.config["RTIO_REGULAR_TTL_COUNT"] = len(rtio_channels)
 
@@ -191,16 +216,11 @@ trce -v 12 -fastpaths -tsi {build_name}.tsi -o {build_name}.twr {build_name}.ncd
         self.submodules += phy
         rtio_channels.append(rtio.Channel.from_phy(phy))
 
-        spi_pins = Module()
-        spi_pins.cs_n = pmod.d[0]
-        spi_pins.mosi = pmod.d[1]
-        spi_pins.miso = pmod.d[2]
-        spi_pins.clk = pmod.d[3]
         phy = spi.SPIMaster(spi_pins)
         self.submodules += phy
         self.config["RTIO_FIRST_SPI_CHANNEL"] = len(rtio_channels)
         rtio_channels.append(rtio.Channel.from_phy(
-            phy, ofifo_depth=128, ififo_depth=128))
+            phy, ofifo_depth=64, ififo_depth=64))
 
         self.config["RTIO_DDS_CHANNEL"] = len(rtio_channels)
         self.config["DDS_CHANNEL_COUNT"] = 8
@@ -222,8 +242,8 @@ trce -v 12 -fastpaths -tsi {build_name}.tsi -o {build_name}.twr {build_name}.ncd
         self.config["RTIO_FINE_TS_WIDTH"] = self.rtio.fine_ts_width
         self.config["DDS_RTIO_CLK_RATIO"] = 8 >> self.rtio.fine_ts_width
         self.submodules.rtio_moninj = rtio.MonInj(rtio_channels)
-        self.submodules.rtio_analyzer = rtio.Analyzer(self.rtio,
-            self.get_native_sdram_if())
+        self.submodules.rtio_analyzer = rtio.Analyzer(
+            self.rtio, self.get_native_sdram_if())
 
 
 def main():

From 2cb58592fff4bab2028081b2432794042efcc66b Mon Sep 17 00:00:00 2001
From: Robert Jordens <rj@m-labs.hk>
Date: Tue, 8 Mar 2016 18:04:34 +0100
Subject: [PATCH 02/38] rtio: add RTIOBusy

---
 artiq/coredevice/__init__.py   |  2 +-
 artiq/coredevice/exceptions.py | 13 +++++++++++++
 artiq/gateware/rtio/core.py    | 27 ++++++++++++++++++++++++---
 artiq/runtime/rtio.c           |  6 ++++++
 artiq/runtime/rtio.h           |  2 ++
 5 files changed, 46 insertions(+), 4 deletions(-)

diff --git a/artiq/coredevice/__init__.py b/artiq/coredevice/__init__.py
index da8c72b1d..4eac74667 100644
--- a/artiq/coredevice/__init__.py
+++ b/artiq/coredevice/__init__.py
@@ -1,6 +1,6 @@
 from artiq.coredevice import exceptions, dds, spi
 from artiq.coredevice.exceptions import (RTIOUnderflow, RTIOSequenceError,
-                                         RTIOCollision, RTIOOverflow,
+                                         RTIOCollision, RTIOOverflow, RTIOBusy,
                                          DDSBatchError, CacheError)
 from artiq.coredevice.dds import (PHASE_MODE_CONTINUOUS, PHASE_MODE_ABSOLUTE,
                                   PHASE_MODE_TRACKING)
diff --git a/artiq/coredevice/exceptions.py b/artiq/coredevice/exceptions.py
index aa00ad784..987f5a127 100644
--- a/artiq/coredevice/exceptions.py
+++ b/artiq/coredevice/exceptions.py
@@ -98,6 +98,19 @@ class RTIOCollision(Exception):
     """
     artiq_builtin = True
 
+class RTIOBusy(Exception):
+    """Raised when an event could not be executed because the given channel
+    was already busy executing a previous event. The two events were not
+    sufficiently spaced on the timeline.
+
+    This exception is raised after the error condition appeared. More
+    specifically it is raised on submitting an event on the same channel after
+    the execution of the faulty event was attempted.
+
+    The offending event is discarded and the RTIO core keeps operating.
+    """
+    artiq_builtin = True
+
 class RTIOOverflow(Exception):
     """Raised when at least one event could not be registered into the RTIO
     input FIFO because it was full (CPU not reading fast enough).
diff --git a/artiq/gateware/rtio/core.py b/artiq/gateware/rtio/core.py
index 8c2d1b237..cdc36598d 100644
--- a/artiq/gateware/rtio/core.py
+++ b/artiq/gateware/rtio/core.py
@@ -104,6 +104,7 @@ class _OutputManager(Module):
         self.underflow = Signal()  # valid 1 cycle after we, pulsed
         self.sequence_error = Signal()
         self.collision = Signal()
+        self.busy = Signal()  # pulsed
 
         # # #
 
@@ -223,6 +224,20 @@ class _OutputManager(Module):
                 dout.timestamp[fine_ts_width:] == counter.value_rtio),
             interface.stb.eq(dout_stb & dout_ack)
         ]
+        busy_sync = PulseSynchronizer("rio", "rsys")
+        busy_ack_sync = PulseSynchronizer("rsys", "rio")
+        self.submodules += busy_sync, busy_ack_sync
+        busy_blind = Signal()
+        self.comb += busy_sync.i.eq(interface.stb & interface.busy & ~busy_blind)
+        self.sync.rio += [
+            If(interface.stb & interface.busy, busy_blind.eq(1)),
+            If(busy_ack_sync.o, busy_blind.eq(0))
+        ]
+        self.comb += [
+            busy_ack_sync.i.eq(busy_sync.o),
+            self.busy.eq(busy_sync.o)
+        ]
+
         if data_width:
             self.comb += interface.data.eq(dout.data)
         if address_width:
@@ -336,10 +351,11 @@ class _KernelCSRs(AutoCSR):
             self.o_address = CSRStorage(address_width)
         self.o_timestamp = CSRStorage(full_ts_width)
         self.o_we = CSR()
-        self.o_status = CSRStatus(4)
+        self.o_status = CSRStatus(5)
         self.o_underflow_reset = CSR()
         self.o_sequence_error_reset = CSR()
         self.o_collision_reset = CSR()
+        self.o_busy_reset = CSR()
 
         if data_width:
             self.i_data = CSRStatus(data_width)
@@ -427,6 +443,7 @@ class RTIO(Module):
             underflow = Signal()
             sequence_error = Signal()
             collision = Signal()
+            busy = Signal()
             self.sync.rsys += [
                 If(selected & self.kcsrs.o_underflow_reset.re,
                    underflow.eq(0)),
@@ -434,14 +451,18 @@ class RTIO(Module):
                    sequence_error.eq(0)),
                 If(selected & self.kcsrs.o_collision_reset.re,
                    collision.eq(0)),
+                If(selected & self.kcsrs.o_busy_reset.re,
+                   busy.eq(0)),
                 If(o_manager.underflow, underflow.eq(1)),
                 If(o_manager.sequence_error, sequence_error.eq(1)),
-                If(o_manager.collision, collision.eq(1))
+                If(o_manager.collision, collision.eq(1)),
+                If(o_manager.busy, busy.eq(1))
             ]
             o_statuses.append(Cat(~o_manager.writable,
                                   underflow,
                                   sequence_error,
-                                  collision))
+                                  collision,
+                                  busy))
 
             if channel.interface.i is not None:
                 i_manager = _InputManager(channel.interface.i, self.counter,
diff --git a/artiq/runtime/rtio.c b/artiq/runtime/rtio.c
index 90dd7fb5e..385011cf5 100644
--- a/artiq/runtime/rtio.c
+++ b/artiq/runtime/rtio.c
@@ -39,6 +39,12 @@ static void rtio_process_exceptional_status(
             "RTIO collision at {0} mu, channel {1}",
             timestamp, channel, 0);
     }
+    if(status & RTIO_O_STATUS_BUSY) {
+        rtio_o_busy_reset_write(1);
+        artiq_raise_from_c("RTIOBusy",
+            "RTIO busy at {0} mu, channel {1}",
+            timestamp, channel, 0);
+    }
 }
 
 
diff --git a/artiq/runtime/rtio.h b/artiq/runtime/rtio.h
index c32dca484..cdaf8a6e1 100644
--- a/artiq/runtime/rtio.h
+++ b/artiq/runtime/rtio.h
@@ -2,11 +2,13 @@
 #define __RTIO_H
 
 #include <stdarg.h>
+#include "generated/csr.h"
 
 #define RTIO_O_STATUS_FULL 1
 #define RTIO_O_STATUS_UNDERFLOW 2
 #define RTIO_O_STATUS_SEQUENCE_ERROR 4
 #define RTIO_O_STATUS_COLLISION 8
+#define RTIO_O_STATUS_BUSY 16
 #define RTIO_I_STATUS_EMPTY 1
 #define RTIO_I_STATUS_OVERFLOW 2
 

From f39208c95af3593e31ca4114d236e64e5d1217fa Mon Sep 17 00:00:00 2001
From: Robert Jordens <rj@m-labs.hk>
Date: Tue, 8 Mar 2016 22:10:47 +0100
Subject: [PATCH 03/38] pipistrello: try with fewer leds/pmod ttl

---
 artiq/gateware/targets/pipistrello.py |  7 +++----
 doc/manual/core_device.rst            | 16 ++++------------
 2 files changed, 7 insertions(+), 16 deletions(-)

diff --git a/artiq/gateware/targets/pipistrello.py b/artiq/gateware/targets/pipistrello.py
index d9dcb3bcc..e75d9804b 100755
--- a/artiq/gateware/targets/pipistrello.py
+++ b/artiq/gateware/targets/pipistrello.py
@@ -196,15 +196,14 @@ trce -v 12 -fastpaths -tsi {build_name}.tsi -o {build_name}.twr {build_name}.ncd
         self.submodules += phy
         rtio_channels.append(rtio.Channel.from_phy(phy, ofifo_depth=4))
 
-        for led_number in range(4):
+        for led_number in range(2):
             phy = ttl_simple.Output(platform.request("user_led", led_number))
             self.submodules += phy
             rtio_channels.append(rtio.Channel.from_phy(phy, ofifo_depth=4))
 
         spi_pins = self.platform.request("pmod_extended_spi", 0)
 
-        for i, p in enumerate((spi_pins.int, spi_pins.rst,
-                               spi_pins.d0, spi_pins.d1)):
+        for i, p in enumerate((spi_pins.int, spi_pins.rst)):
             phy = ttl_simple.Inout(p)
             self.submodules += phy
             rtio_channels.append(rtio.Channel.from_phy(phy, ififo_depth=4,
@@ -220,7 +219,7 @@ trce -v 12 -fastpaths -tsi {build_name}.tsi -o {build_name}.twr {build_name}.ncd
         self.submodules += phy
         self.config["RTIO_FIRST_SPI_CHANNEL"] = len(rtio_channels)
         rtio_channels.append(rtio.Channel.from_phy(
-            phy, ofifo_depth=64, ififo_depth=64))
+            phy, ofifo_depth=256, ififo_depth=256))
 
         self.config["RTIO_DDS_CHANNEL"] = len(rtio_channels)
         self.config["DDS_CHANNEL_COUNT"] = 8
diff --git a/doc/manual/core_device.rst b/doc/manual/core_device.rst
index 8a947dab6..fe3ef2640 100644
--- a/doc/manual/core_device.rst
+++ b/doc/manual/core_device.rst
@@ -128,19 +128,11 @@ When plugged to an adapter, the NIST QC1 hardware can be used. The TTL lines are
 +--------------+------------+--------------+
 | 19           | USER_LED_2 | Output       |
 +--------------+------------+--------------+
-| 20           | USER_LED_3 | Output       |
+| 20           | PMOD_4     | Input+Output |
 +--------------+------------+--------------+
-| 21           | USER_LED_4 | Output       |
+| 21           | PMOD_5     | Input+Output |
 +--------------+------------+--------------+
-| 22           | PMOD_4     | Input+Output |
-+--------------+------------+--------------+
-| 23           | PMOD_5     | Input+Output |
-+--------------+------------+--------------+
-| 24           | PMOD_6     | Input+Output |
-+--------------+------------+--------------+
-| 25           | PMOD_7     | Input+Output |
-+--------------+------------+--------------+
-| 26           | TTL15      | Clock        |
+| 22           | TTL15      | Clock        |
 +--------------+------------+--------------+
 
 The input only limitation on channels 0 and 1 comes from the QC-DAQ adapter. When the adapter is not used (and physically unplugged from the Pipistrello board), the corresponding pins on the Pipistrello can be used as outputs. Do not configure these channels as outputs when the adapter is plugged, as this would cause electrical contention.
@@ -153,5 +145,5 @@ Interface Type 2 (SPI) and 2A (expanded SPI):
 +--------------+--------+--------+--------+--------+
 | RTIO channel | CS_N   | MOSI   | MISO   | CLK    |
 +==============+========+========+========+========+
-| 27           | PMOD_0 | PMOD_1 | PMOD_2 | PMOD_3 |
+| 23           | PMOD_0 | PMOD_1 | PMOD_2 | PMOD_3 |
 +--------------+--------+--------+--------+--------+

From f33baf339f4bfc891ca4e780580a81bfc5086b70 Mon Sep 17 00:00:00 2001
From: Robert Jordens <rj@m-labs.hk>
Date: Tue, 8 Mar 2016 23:34:51 +0100
Subject: [PATCH 04/38] pipistrello: drop ttls on pmod, add leds back in

---
 artiq/gateware/targets/pipistrello.py | 8 +-------
 doc/manual/core_device.rst            | 4 ++--
 2 files changed, 3 insertions(+), 9 deletions(-)

diff --git a/artiq/gateware/targets/pipistrello.py b/artiq/gateware/targets/pipistrello.py
index e75d9804b..620810b60 100755
--- a/artiq/gateware/targets/pipistrello.py
+++ b/artiq/gateware/targets/pipistrello.py
@@ -196,19 +196,13 @@ trce -v 12 -fastpaths -tsi {build_name}.tsi -o {build_name}.twr {build_name}.ncd
         self.submodules += phy
         rtio_channels.append(rtio.Channel.from_phy(phy, ofifo_depth=4))
 
-        for led_number in range(2):
+        for led_number in range(4):
             phy = ttl_simple.Output(platform.request("user_led", led_number))
             self.submodules += phy
             rtio_channels.append(rtio.Channel.from_phy(phy, ofifo_depth=4))
 
         spi_pins = self.platform.request("pmod_extended_spi", 0)
 
-        for i, p in enumerate((spi_pins.int, spi_pins.rst)):
-            phy = ttl_simple.Inout(p)
-            self.submodules += phy
-            rtio_channels.append(rtio.Channel.from_phy(phy, ififo_depth=4,
-                                                       ofifo_depth=4))
-
         self.config["RTIO_REGULAR_TTL_COUNT"] = len(rtio_channels)
 
         phy = ttl_simple.ClockGen(platform.request("ttl", 15))
diff --git a/doc/manual/core_device.rst b/doc/manual/core_device.rst
index fe3ef2640..484f69358 100644
--- a/doc/manual/core_device.rst
+++ b/doc/manual/core_device.rst
@@ -128,9 +128,9 @@ When plugged to an adapter, the NIST QC1 hardware can be used. The TTL lines are
 +--------------+------------+--------------+
 | 19           | USER_LED_2 | Output       |
 +--------------+------------+--------------+
-| 20           | PMOD_4     | Input+Output |
+| 20           | USER_LED_3 | Output       |
 +--------------+------------+--------------+
-| 21           | PMOD_5     | Input+Output |
+| 21           | USER_LED_4 | Output       |
 +--------------+------------+--------------+
 | 22           | TTL15      | Clock        |
 +--------------+------------+--------------+

From 80e1cf5d7823675da9de7156ccff3d7c0f32efc7 Mon Sep 17 00:00:00 2001
From: whitequark <whitequark@whitequark.org>
Date: Wed, 9 Mar 2016 02:34:22 +0000
Subject: [PATCH 05/38] Monkey-patch asyncio create_server (fixes #253).

---
 artiq/tools.py | 108 +++++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 108 insertions(+)

diff --git a/artiq/tools.py b/artiq/tools.py
index ef666cabf..8d2424cdc 100644
--- a/artiq/tools.py
+++ b/artiq/tools.py
@@ -7,6 +7,8 @@ import asyncio
 import time
 import collections
 import os
+import socket
+import itertools
 import atexit
 import string
 
@@ -243,3 +245,109 @@ def get_windows_drives():
             drives.append(letter)
         bitmask >>= 1
     return drives
+
+if sys.version_info[:3] == (3, 5, 1):
+    # See https://github.com/m-labs/artiq/issues/253
+    @asyncio.coroutines.coroutine
+    def create_server(self, protocol_factory, host=None, port=None,
+                      *,
+                      family=socket.AF_UNSPEC,
+                      flags=socket.AI_PASSIVE,
+                      sock=None,
+                      backlog=100,
+                      ssl=None,
+                      reuse_address=None,
+                      reuse_port=None):
+        """Create a TCP server.
+        The host parameter can be a string, in that case the TCP server is bound
+        to host and port.
+        The host parameter can also be a sequence of strings and in that case
+        the TCP server is bound to all hosts of the sequence. If a host
+        appears multiple times (possibly indirectly e.g. when hostnames
+        resolve to the same IP address), the server is only bound once to that
+        host.
+        Return a Server object which can be used to stop the service.
+        This method is a coroutine.
+        """
+        if isinstance(ssl, bool):
+            raise TypeError('ssl argument must be an SSLContext or None')
+        if host is not None or port is not None:
+            if sock is not None:
+                raise ValueError(
+                    'host/port and sock can not be specified at the same time')
+
+            AF_INET6 = getattr(socket, 'AF_INET6', 0)
+            if reuse_address is None:
+                reuse_address = os.name == 'posix' and sys.platform != 'cygwin'
+            sockets = []
+            if host == '':
+                hosts = [None]
+            elif (isinstance(host, str) or
+                  not isinstance(host, collections.Iterable)):
+                hosts = [host]
+            else:
+                hosts = host
+
+            fs = [self._create_server_getaddrinfo(host, port, family=family,
+                                                  flags=flags)
+                  for host in hosts]
+            infos = yield from asyncio.tasks.gather(*fs, loop=self)
+            infos = set(itertools.chain.from_iterable(infos))
+
+            completed = False
+            try:
+                for res in infos:
+                    af, socktype, proto, canonname, sa = res
+                    try:
+                        sock = socket.socket(af, socktype, proto)
+                    except socket.error:
+                        # Assume it's a bad family/type/protocol combination.
+                        if self._debug:
+                            asyncio.log.logger.warning('create_server() failed to create '
+                                           'socket.socket(%r, %r, %r)',
+                                           af, socktype, proto, exc_info=True)
+                        continue
+                    sockets.append(sock)
+                    if reuse_address:
+                        sock.setsockopt(
+                            socket.SOL_SOCKET, socket.SO_REUSEADDR, True)
+                    if reuse_port:
+                        if not hasattr(socket, 'SO_REUSEPORT'):
+                            raise ValueError(
+                                'reuse_port not supported by socket module')
+                        else:
+                            sock.setsockopt(
+                                socket.SOL_SOCKET, socket.SO_REUSEPORT, True)
+                    # Disable IPv4/IPv6 dual stack support (enabled by
+                    # default on Linux) which makes a single socket
+                    # listen on both address families.
+                    if af == AF_INET6 and hasattr(socket, 'IPPROTO_IPV6'):
+                        sock.setsockopt(socket.IPPROTO_IPV6,
+                                        socket.IPV6_V6ONLY,
+                                        True)
+                    try:
+                        sock.bind(sa)
+                    except OSError as err:
+                        raise OSError(err.errno, 'error while attempting '
+                                      'to bind on address %r: %s'
+                                      % (sa, err.strerror.lower()))
+                completed = True
+            finally:
+                if not completed:
+                    for sock in sockets:
+                        sock.close()
+        else:
+            if sock is None:
+                raise ValueError('Neither host/port nor sock were specified')
+            sockets = [sock]
+
+        server = asyncio.base_events.Server(self, sockets)
+        for sock in sockets:
+            sock.listen(backlog)
+            sock.setblocking(False)
+            self._start_serving(protocol_factory, sock, ssl, server)
+        if self._debug:
+            asyncio.log.logger.info("%r is serving", server)
+        return server
+
+    asyncio.base_events.BaseEventLoop.create_server = create_server

From 2e4e251877c966fa5e5da8768f5a372ed793ff27 Mon Sep 17 00:00:00 2001
From: Sebastien Bourdeauducq <sb@m-labs.hk>
Date: Wed, 9 Mar 2016 10:34:58 +0800
Subject: [PATCH 06/38] examples/device_db: remove --no-localhost-bind

---
 examples/master/device_db.pyon | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/examples/master/device_db.pyon b/examples/master/device_db.pyon
index ee475911a..f10b1e6b2 100644
--- a/examples/master/device_db.pyon
+++ b/examples/master/device_db.pyon
@@ -154,25 +154,25 @@
         # that it always resolves to a network-visible IP address (see documentation).
         "host": "::1",
         "port": 4000,
-        "command": "pdq2_controller --no-localhost-bind -p {port} --bind {bind} --simulation --dump qc_q1_0.bin"
+        "command": "pdq2_controller -p {port} --bind {bind} --simulation --dump qc_q1_0.bin"
     },
     "qc_q1_1": {
         "type": "controller",
         "host": "::1",
         "port": 4001,
-        "command": "pdq2_controller --no-localhost-bind -p {port} --bind {bind} --simulation --dump qc_q1_1.bin"
+        "command": "pdq2_controller -p {port} --bind {bind} --simulation --dump qc_q1_1.bin"
     },
     "qc_q1_2": {
         "type": "controller",
         "host": "::1",
         "port": 4002,
-        "command": "pdq2_controller --no-localhost-bind -p {port} --bind {bind} --simulation --dump qc_q1_2.bin"
+        "command": "pdq2_controller -p {port} --bind {bind} --simulation --dump qc_q1_2.bin"
     },
     "qc_q1_3": {
         "type": "controller",
         "host": "::1",
         "port": 4003,
-        "command": "pdq2_controller --no-localhost-bind -p {port} --bind {bind} --simulation --dump qc_q1_3.bin"
+        "command": "pdq2_controller -p {port} --bind {bind} --simulation --dump qc_q1_3.bin"
     },
     "electrodes": {
         "type": "local",

From 9d1903a4e2276b69ba3a8ec058f1a716496f7c7f Mon Sep 17 00:00:00 2001
From: Sebastien Bourdeauducq <sb@m-labs.hk>
Date: Wed, 9 Mar 2016 13:01:34 +0800
Subject: [PATCH 07/38] coredevice/i2c,ttl,spi: consistent device get

---
 artiq/coredevice/i2c.py |  8 ++++----
 artiq/coredevice/spi.py |  4 ++--
 artiq/coredevice/ttl.py | 12 ++++++------
 3 files changed, 12 insertions(+), 12 deletions(-)

diff --git a/artiq/coredevice/i2c.py b/artiq/coredevice/i2c.py
index e9de36161..55148402a 100644
--- a/artiq/coredevice/i2c.py
+++ b/artiq/coredevice/i2c.py
@@ -34,8 +34,8 @@ class PCA9548:
     On the KC705, this chip is used for selecting the I2C buses on the two FMC
     connectors. HPC=1, LPC=2.
     """
-    def __init__(self, dmgr, busno=0, address=0xe8):
-        self.core = dmgr.get("core")
+    def __init__(self, dmgr, busno=0, address=0xe8, core_device="core"):
+        self.core = dmgr.get(core_device)
         self.busno = busno
         self.address = address
 
@@ -77,8 +77,8 @@ class TCA6424A:
 
     On the NIST QC2 hardware, this chip is used for switching the directions
     of TTL buffers."""
-    def __init__(self, dmgr, busno=0, address=0x44):
-        self.core = dmgr.get("core")
+    def __init__(self, dmgr, busno=0, address=0x44, core_device="core"):
+        self.core = dmgr.get(core_device)
         self.busno = busno
         self.address = address
 
diff --git a/artiq/coredevice/spi.py b/artiq/coredevice/spi.py
index 6221c8f53..20095a8ff 100644
--- a/artiq/coredevice/spi.py
+++ b/artiq/coredevice/spi.py
@@ -40,8 +40,8 @@ class SPIMaster:
 
     :param channel: RTIO channel number of the SPI bus to control.
     """
-    def __init__(self, dmgr, channel):
-        self.core = dmgr.get("core")
+    def __init__(self, dmgr, channel, core_device="core"):
+        self.core = dmgr.get(core_device)
         self.ref_period_mu = seconds_to_mu(self.core.coarse_ref_period,
                                            self.core)
         self.channel = channel
diff --git a/artiq/coredevice/ttl.py b/artiq/coredevice/ttl.py
index 9ac12ca48..ef855d6b2 100644
--- a/artiq/coredevice/ttl.py
+++ b/artiq/coredevice/ttl.py
@@ -10,8 +10,8 @@ class TTLOut:
 
     :param channel: channel number
     """
-    def __init__(self, dmgr, channel):
-        self.core = dmgr.get("core")
+    def __init__(self, dmgr, channel, core_device="core"):
+        self.core = dmgr.get(core_device)
         self.channel = channel
 
         # in RTIO cycles
@@ -82,8 +82,8 @@ class TTLInOut:
 
     :param channel: channel number
     """
-    def __init__(self, dmgr, channel):
-        self.core = dmgr.get("core")
+    def __init__(self, dmgr, channel, core_device="core"):
+        self.core = dmgr.get(core_device)
         self.channel = channel
 
         # in RTIO cycles
@@ -232,8 +232,8 @@ class TTLClockGen:
 
     :param channel: channel number
     """
-    def __init__(self, dmgr, channel):
-        self.core = dmgr.get("core")
+    def __init__(self, dmgr, channel, core_device="core"):
+        self.core = dmgr.get(core_device)
         self.channel = channel
 
         # in RTIO cycles

From f0b0b1bac7968af5a6cab9858ce915e29cb4d08d Mon Sep 17 00:00:00 2001
From: Sebastien Bourdeauducq <sb@m-labs.hk>
Date: Wed, 9 Mar 2016 17:12:50 +0800
Subject: [PATCH 08/38] support for multiple DDS buses (untested)

---
 RELEASE_NOTES.rst                     |   6 +-
 artiq/coredevice/analyzer.py          |  63 +++++++++------
 artiq/coredevice/core.py              |   3 +-
 artiq/coredevice/dds.py               |  60 ++++++++------
 artiq/gateware/targets/kc705.py       |  15 ++--
 artiq/gateware/targets/pipistrello.py |   5 +-
 artiq/gui/moninj.py                   |  28 +++++--
 artiq/runtime/analyzer.c              |   2 -
 artiq/runtime/bridge.c                |  30 ++++---
 artiq/runtime/bridge_ctl.c            |  19 +++--
 artiq/runtime/bridge_ctl.h            |  10 +--
 artiq/runtime/dds.c                   |  19 +++--
 artiq/runtime/messages.h              |  14 ++++
 artiq/runtime/moninj.c                |  22 +++--
 artiq/runtime/test_mode.c             | 112 ++++++++++++++++----------
 doc/manual/core_device.rst            |   2 +
 examples/master/device_db.pyon        |  18 ++---
 17 files changed, 265 insertions(+), 163 deletions(-)

diff --git a/RELEASE_NOTES.rst b/RELEASE_NOTES.rst
index 4c905ea5a..d675623d8 100644
--- a/RELEASE_NOTES.rst
+++ b/RELEASE_NOTES.rst
@@ -11,4 +11,8 @@ Release notes
   This requires reflashing the runtime and the flash storage filesystem image
   or erase and rewrite its entries.
 * RTIOCollisionError has been renamed to RTIOCollision
- 
\ No newline at end of file
+* the new API for DDS batches is:
+    with self.core_dds.batch:
+       ...
+  with core_dds a device of type artiq.coredevice.dds.CoreDDS.
+  The dds_bus device should not be used anymore.
diff --git a/artiq/coredevice/analyzer.py b/artiq/coredevice/analyzer.py
index b2b458bff..f526bb2ec 100644
--- a/artiq/coredevice/analyzer.py
+++ b/artiq/coredevice/analyzer.py
@@ -40,16 +40,19 @@ def decode_message(data):
 
 
 DecodedDump = namedtuple(
-    "DecodedDump", "log_channel dds_channel dds_onehot_sel messages")
+    "DecodedDump", "log_channel dds_onehot_sel messages")
 
 
 def decode_dump(data):
-    parts = struct.unpack(">IQbbbb", data[:16])
+    parts = struct.unpack(">IQbbb", data[:15])
     (sent_bytes, total_byte_count,
-     overflow_occured, log_channel, dds_channel, dds_onehot_sel) = parts
+     overflow_occured, log_channel, dds_onehot_sel) = parts
 
-    if sent_bytes + 16 != len(data):
-        raise ValueError("analyzer dump has incorrect length")
+    expected_len = sent_bytes + 15
+    if expected_len != len(data):
+        raise ValueError("analyzer dump has incorrect length "
+                         "(got {}, expected {})".format(
+                            len(data), expected_len))
     if overflow_occured:
         logger.warning("analyzer FIFO overflow occured, "
                        "some messages have been lost")
@@ -57,14 +60,12 @@ def decode_dump(data):
         logger.info("analyzer ring buffer has wrapped %d times",
                     total_byte_count//sent_bytes)
 
-    position = 16
+    position = 15
     messages = []
     for _ in range(sent_bytes//32):
         messages.append(decode_message(data[position:position+32]))
         position += 32
-    return DecodedDump(log_channel,
-                       dds_channel, bool(dds_onehot_sel),
-                       messages)
+    return DecodedDump(log_channel, bool(dds_onehot_sel), messages)
 
 
 def vcd_codes():
@@ -299,21 +300,31 @@ def get_vcd_log_channels(log_channel, messages):
     return vcd_log_channels
 
 
-def get_ref_period(devices):
+def get_single_device_argument(devices, module, cls, argument):
     ref_period = None
     for desc in devices.values():
         if isinstance(desc, dict) and desc["type"] == "local":
-            if (desc["module"] == "artiq.coredevice.core"
-                    and desc["class"] == "Core"):
+            if (desc["module"] == module
+                    and desc["class"] == cls):
                 if ref_period is None:
-                    ref_period = desc["arguments"]["ref_period"]
+                    ref_period = desc["arguments"][argument]
                 else:
-                    return None  # more than one core device found
+                    return None  # more than one device found
     return ref_period
 
 
+def get_ref_period(devices):
+    return get_single_device_argument(devices, "artiq.coredevice.core",
+                                      "Core", "ref_period")
+
+
+def get_dds_sysclk(devices):
+    return get_single_device_argument(devices, "artiq.coredevice.core",
+                                      "CoreDDS", "sysclk")
+
+
 def create_channel_handlers(vcd_manager, devices, ref_period,
-                            dds_channel, dds_onehot_sel):
+                            dds_sysclk, dds_onehot_sel):
     channel_handlers = dict()
     for name, desc in sorted(devices.items(), key=itemgetter(0)):
         if isinstance(desc, dict) and desc["type"] == "local":
@@ -327,19 +338,17 @@ def create_channel_handlers(vcd_manager, devices, ref_period,
                 channel_handlers[channel] = TTLClockGenHandler(vcd_manager, name, ref_period)
             if (desc["module"] == "artiq.coredevice.dds"
                     and desc["class"] in {"AD9858", "AD9914"}):
-                sysclk = desc["arguments"]["sysclk"]
-                dds_channel_ddsbus = desc["arguments"]["channel"]
-                if dds_channel in channel_handlers:
-                    dds_handler = channel_handlers[dds_channel]
+                dds_bus_channel = desc["arguments"]["bus_channel"]
+                dds_channel = desc["arguments"]["channel"]
+                if dds_bus_channel in channel_handlers:
+                    dds_handler = channel_handlers[dds_bus_channel]
                     if dds_handler.dds_type != desc["class"]:
                         raise ValueError("All DDS channels must have the same type")
-                    if dds_handler.sysclk != sysclk:
-                        raise ValueError("All DDS channels must have the same sysclk")
                 else:
                     dds_handler = DDSHandler(vcd_manager, desc["class"],
-                        dds_onehot_sel, sysclk)
-                    channel_handlers[dds_channel] = dds_handler
-                dds_handler.add_dds_channel(name, dds_channel_ddsbus)
+                        dds_sysclk, dds_onehot_sel)
+                    channel_handlers[dds_bus_channel] = dds_handler
+                dds_handler.add_dds_channel(name, dds_channel)
     return channel_handlers
 
 
@@ -355,12 +364,16 @@ def decoded_dump_to_vcd(fileobj, devices, dump):
     else:
         logger.warning("unable to determine core device ref_period")
         ref_period = 1e-9  # guess
+    dds_sysclk = get_dds_sysclk(devices)
+    if dds_sysclk is None:
+        logger.warning("unable to determine DDS sysclk")
+        dds_sysclk = 3e9  # guess
 
     messages = sorted(dump.messages, key=get_message_time)
 
     channel_handlers = create_channel_handlers(
         vcd_manager, devices, ref_period,
-        dump.dds_channel, dump.dds_onehot_sel)
+        dds_sysclk, dump.dds_onehot_sel)
     vcd_log_channels = get_vcd_log_channels(dump.log_channel, messages)
     channel_handlers[dump.log_channel] = LogHandler(vcd_manager, vcd_log_channels)
     slack = vcd_manager.get_channel("rtio_slack", 64)
diff --git a/artiq/coredevice/core.py b/artiq/coredevice/core.py
index 32c2501bd..7aa17fa9b 100644
--- a/artiq/coredevice/core.py
+++ b/artiq/coredevice/core.py
@@ -22,7 +22,7 @@ def _render_diagnostic(diagnostic, colored):
     lines = [shorten_path(path) for path in diagnostic.render(colored=colored)]
     return "\n".join(lines)
 
-colors_supported = (os.name == 'posix')
+colors_supported = os.name == "posix"
 class _DiagnosticEngine(diagnostic.Engine):
     def render_diagnostic(self, diagnostic):
         sys.stderr.write(_render_diagnostic(diagnostic, colored=colors_supported) + "\n")
@@ -49,6 +49,7 @@ def cache_get(key: TStr) -> TList(TInt32):
 def cache_put(key: TStr, value: TList(TInt32)) -> TNone:
     raise NotImplementedError("syscall not simulated")
 
+
 class Core:
     """Core device driver.
 
diff --git a/artiq/coredevice/dds.py b/artiq/coredevice/dds.py
index 99cbc72af..b375292de 100644
--- a/artiq/coredevice/dds.py
+++ b/artiq/coredevice/dds.py
@@ -11,7 +11,12 @@ PHASE_MODE_TRACKING = 2
 
 
 @syscall
-def dds_init(time_mu: TInt64, channel: TInt32) -> TNone:
+def dds_init(time_mu: TInt64, bus_channel: TInt32, channel: TInt32) -> TNone:
+    raise NotImplementedError("syscall not simulated")
+
+@syscall
+def dds_set(time_mu: TInt64, bus_channel: TInt32, channel: TInt32, ftw: TInt32,
+            pow: TInt32, phase_mode: TInt32, amplitude: TInt32) -> TNone:
     raise NotImplementedError("syscall not simulated")
 
 @syscall
@@ -22,35 +27,36 @@ def dds_batch_enter(time_mu: TInt64) -> TNone:
 def dds_batch_exit() -> TNone:
     raise NotImplementedError("syscall not simulated")
 
-@syscall
-def dds_set(time_mu: TInt64, channel: TInt32, ftw: TInt32,
-            pow: TInt32, phase_mode: TInt32, amplitude: TInt32) -> TNone:
-    raise NotImplementedError("syscall not simulated")
-
 
 class _BatchContextManager:
-    def __init__(self, dds_bus):
-        self.dds_bus = dds_bus
+    def __init__(self, core_dds):
+        self.core_dds = core_dds
+        self.core = self.core_dds.core
 
     @kernel
     def __enter__(self):
-        self.dds_bus.batch_enter()
+        self.core_dds.dds_batch_enter()
 
     @kernel
     def __exit__(self, type, value, traceback):
-        self.dds_bus.batch_exit()
+        self.core_dds.dds_batch_exit()
 
 
-class DDSBus:
-    """Core device Direct Digital Synthesis (DDS) bus batching driver.
+class CoreDDS:
+    """Core device Direct Digital Synthesis (DDS) driver.
 
-    Manages batching of DDS commands on a DDS shared bus."""
-    def __init__(self, dmgr):
-        self.core = dmgr.get("core")
+    Gives access to the DDS functionality of the core device.
+
+    :param sysclk: DDS system frequency. The DDS system clock must be a
+        phase-locked multiple of the RTIO clock.
+    """
+    def __init__(self, dmgr, sysclk, core_device="core"):
+        self.core = dmgr.get(core_device)
+        self.sysclk = sysclk
         self.batch = _BatchContextManager(self)
 
     @kernel
-    def batch_enter(self):
+    def dds_batch_enter(self):
         """Starts a DDS command batch. All DDS commands are buffered
         after this call, until ``batch_exit`` is called.
 
@@ -59,26 +65,27 @@ class DDSBus:
         dds_batch_enter(now_mu())
 
     @kernel
-    def batch_exit(self):
+    def dds_batch_exit(self):
         """Ends a DDS command batch. All buffered DDS commands are issued
         on the bus."""
         dds_batch_exit()
 
 
 class _DDSGeneric:
-    """Core device Direct Digital Synthesis (DDS) driver.
+    """Core device Direct Digital Synthesis (DDS) channel driver.
 
     Controls one DDS channel managed directly by the core device's runtime.
 
     This class should not be used directly, instead, use the chip-specific
     drivers such as ``AD9858`` and ``AD9914``.
 
-    :param sysclk: DDS system frequency.
+    :param bus: name of the DDS bus device that this DDS is connected to.
     :param channel: channel number of the DDS device to control.
     """
-    def __init__(self, dmgr, sysclk, channel):
-        self.core = dmgr.get("core")
-        self.sysclk = sysclk
+    def __init__(self, dmgr, sysclk, bus_channel, channel, core_dds_device="core_dds"):
+        self.core_dds = dmgr.get(core_dds_device)
+        self.core = self.core_dds.core
+        self.bus_channel = bus_channel
         self.channel = channel
         self.phase_mode = PHASE_MODE_CONTINUOUS
 
@@ -87,14 +94,14 @@ class _DDSGeneric:
         """Returns the frequency tuning word corresponding to the given
         frequency.
         """
-        return round(int(2, width=64)**32*frequency/self.sysclk)
+        return round(int(2, width=64)**32*frequency/self.core_dds.sysclk)
 
     @portable
     def ftw_to_frequency(self, ftw):
         """Returns the frequency corresponding to the given frequency tuning
         word.
         """
-        return ftw*self.sysclk/int(2, width=64)**32
+        return ftw*self.core_dds.sysclk/int(2, width=64)**32
 
     @portable
     def turns_to_pow(self, turns):
@@ -124,7 +131,7 @@ class _DDSGeneric:
         """Resets and initializes the DDS channel.
 
         The runtime does this for all channels upon core device startup."""
-        dds_init(now_mu(), self.channel)
+        dds_init(now_mu(), self.bus_channel, self.channel)
 
     @kernel
     def set_phase_mode(self, phase_mode):
@@ -163,7 +170,8 @@ class _DDSGeneric:
         """
         if phase_mode == _PHASE_MODE_DEFAULT:
             phase_mode = self.phase_mode
-        dds_set(now_mu(), self.channel, frequency, phase, phase_mode, amplitude)
+        dds_set(now_mu(), self.bus_channel, self.channel,
+                frequency, phase, phase_mode, amplitude)
 
     @kernel
     def set(self, frequency, phase=0.0, phase_mode=_PHASE_MODE_DEFAULT,
diff --git a/artiq/gateware/targets/kc705.py b/artiq/gateware/targets/kc705.py
index 8f766ecce..ae15ec179 100755
--- a/artiq/gateware/targets/kc705.py
+++ b/artiq/gateware/targets/kc705.py
@@ -204,8 +204,9 @@ class NIST_QC1(_NIST_Ions):
         self.submodules += phy
         rtio_channels.append(rtio.Channel.from_phy(phy))
 
-        self.config["RTIO_DDS_CHANNEL"] = len(rtio_channels)
-        self.config["DDS_CHANNEL_COUNT"] = 8
+        self.config["RTIO_FIRST_DDS_CHANNEL"] = len(rtio_channels)
+        self.config["RTIO_DDS_COUNT"] = 1
+        self.config["DDS_CHANNELS_PER_BUS"] = 8
         self.config["DDS_AD9858"] = True
         phy = dds.AD9858(platform.request("dds"), 8)
         self.submodules += phy
@@ -277,8 +278,9 @@ class NIST_CLOCK(_NIST_Ions):
             rtio_channels.append(rtio.Channel.from_phy(
                 phy, ofifo_depth=128, ififo_depth=128))
 
-        self.config["RTIO_DDS_CHANNEL"] = len(rtio_channels)
-        self.config["DDS_CHANNEL_COUNT"] = 11
+        self.config["RTIO_FIRST_DDS_CHANNEL"] = len(rtio_channels)
+        self.config["RTIO_DDS_COUNT"] = 1
+        self.config["DDS_CHANNELS_PER_BUS"] = 11
         self.config["DDS_AD9914"] = True
         self.config["DDS_ONEHOT_SEL"] = True
         phy = dds.AD9914(platform.request("dds"), 11, onehot=True)
@@ -331,8 +333,9 @@ class NIST_QC2(_NIST_Ions):
         self.submodules += phy
         rtio_channels.append(rtio.Channel.from_phy(phy))
 
-        self.config["RTIO_DDS_CHANNEL"] = len(rtio_channels)
-        self.config["DDS_CHANNEL_COUNT"] = 12
+        self.config["RTIO_FIRST_DDS_CHANNEL"] = len(rtio_channels)
+        self.config["RTIO_DDS_COUNT"] = 1
+        self.config["DDS_CHANNELS_PER_BUS"] = 12
         self.config["DDS_AD9914"] = True
         self.config["DDS_ONEHOT_SEL"] = True
         phy = dds.AD9914(platform.request("dds"), 12, onehot=True)
diff --git a/artiq/gateware/targets/pipistrello.py b/artiq/gateware/targets/pipistrello.py
index 620810b60..2b2c7de65 100755
--- a/artiq/gateware/targets/pipistrello.py
+++ b/artiq/gateware/targets/pipistrello.py
@@ -215,8 +215,9 @@ trce -v 12 -fastpaths -tsi {build_name}.tsi -o {build_name}.twr {build_name}.ncd
         rtio_channels.append(rtio.Channel.from_phy(
             phy, ofifo_depth=256, ififo_depth=256))
 
-        self.config["RTIO_DDS_CHANNEL"] = len(rtio_channels)
-        self.config["DDS_CHANNEL_COUNT"] = 8
+        self.config["RTIO_FIRST_DDS_CHANNEL"] = len(rtio_channels)
+        self.config["RTIO_DDS_COUNT"] = 1
+        self.config["DDS_CHANNELS_PER_BUS"] = 8
         self.config["DDS_AD9858"] = True
         dds_pins = platform.request("dds")
         self.comb += dds_pins.p.eq(0)
diff --git a/artiq/gui/moninj.py b/artiq/gui/moninj.py
index 43892876a..818c148f6 100644
--- a/artiq/gui/moninj.py
+++ b/artiq/gui/moninj.py
@@ -118,7 +118,8 @@ class _TTLWidget(QtWidgets.QFrame):
 
 
 class _DDSWidget(QtWidgets.QFrame):
-    def __init__(self, channel, sysclk, title):
+    def __init__(self, bus_channel, channel, sysclk, title):
+        self.bus_channel = bus_channel
         self.channel = channel
         self.sysclk = sysclk
 
@@ -146,13 +147,14 @@ class _DDSWidget(QtWidgets.QFrame):
         self.set_value(0)
 
     def set_value(self, ftw):
-        frequency = ftw*self.sysclk/2**32
+        frequency = ftw*self.sysclk()/2**32
         self._value.setText("<font size=\"6\">{:.7f} MHz</font>"
                             .format(float(frequency)/1e6))
 
 
 class _DeviceManager:
     def __init__(self, send_to_device, init):
+        self.dds_sysclk = 0
         self.send_to_device = send_to_device
         self.ddb = dict()
         self.ttl_cb = lambda: None
@@ -162,6 +164,9 @@ class _DeviceManager:
         for k, v in init.items():
             self[k] = v
 
+    def get_dds_sysclk(self):
+        return self.dds_sysclk
+
     def __setitem__(self, k, v):
         if k in self.ttl_widgets:
             del self[k]
@@ -181,12 +186,15 @@ class _DeviceManager:
                     self.ttl_widgets[k] = _TTLWidget(
                         channel, self.send_to_device, force_out, title)
                     self.ttl_cb()
+                if (v["module"] == "artiq.coredevice.dds"
+                        and v["class"] == "CoreDDS"):
+                    self.dds_sysclk = v["arguments"]["sysclk"]
                 if (v["module"] == "artiq.coredevice.dds"
                         and v["class"] in {"AD9858", "AD9914"}):
+                    bus_channel = v["arguments"]["bus_channel"]
                     channel = v["arguments"]["channel"]
-                    sysclk = v["arguments"]["sysclk"]
                     self.dds_widgets[channel] = _DDSWidget(
-                        channel, sysclk, title)
+                        bus_channel, channel, self.get_dds_sysclk, title)
                     self.dds_cb()
         except KeyError:
             pass
@@ -275,19 +283,23 @@ class MonInj(TaskObject):
                          "is not present yet")
             return
         try:
-            ttl_levels, ttl_oes, ttl_overrides = \
-                struct.unpack(">QQQ", data[:8*3])
+            hlen = 8*3+4
+            (ttl_levels, ttl_oes, ttl_overrides,
+             dds_rtio_first_channel, dds_channels_per_bus) = \
+                struct.unpack(">QQQHH", data[:hlen])
             for w in self.dm.ttl_widgets.values():
                 channel = w.channel
                 w.set_value(ttl_levels & (1 << channel),
                             ttl_oes & (1 << channel),
                             ttl_overrides & (1 << channel))
-            dds_data = data[8*3:]
+            dds_data = data[hlen:]
             ndds = len(dds_data)//4
             ftws = struct.unpack(">" + "I"*ndds, dds_data)
             for w in self.dm.dds_widgets.values():
+                offset = (dds_channels_per_bus*w.bus_channel
+                          + w.channel-dds_rtio_first_channel)
                 try:
-                    ftw = ftws[w.channel]
+                    ftw = ftws[offset]
                 except KeyError:
                     pass
                 else:
diff --git a/artiq/runtime/analyzer.c b/artiq/runtime/analyzer.c
index a1b71b428..23f77c98d 100644
--- a/artiq/runtime/analyzer.c
+++ b/artiq/runtime/analyzer.c
@@ -12,7 +12,6 @@ struct analyzer_header {
     unsigned long long int total_byte_count;
     unsigned char overflow_occured;
     unsigned char log_channel;
-    unsigned char dds_channel;
     unsigned char dds_onehot_sel;
 } __attribute__((packed));
 
@@ -72,7 +71,6 @@ void analyzer_start(void)
 
     analyzer_header.overflow_occured = rtio_analyzer_message_encoder_overflow_read();
     analyzer_header.log_channel = CONFIG_RTIO_LOG_CHANNEL;
-    analyzer_header.dds_channel = CONFIG_RTIO_DDS_CHANNEL;
 #ifdef CONFIG_DDS_ONEHOT_SEL
     analyzer_header.dds_onehot_sel = 1;
 #else
diff --git a/artiq/runtime/bridge.c b/artiq/runtime/bridge.c
index d1157cf65..bfe3e1fcf 100644
--- a/artiq/runtime/bridge.c
+++ b/artiq/runtime/bridge.c
@@ -16,12 +16,12 @@ static void rtio_output_blind(int channel, int addr, int data)
     rtio_o_we_write(1);
 }
 
-static void dds_write(int addr, int data)
+static void dds_write(int bus_channel, int addr, int data)
 {
-    rtio_output_blind(CONFIG_RTIO_DDS_CHANNEL, addr, data);
+    rtio_output_blind(bus_channel, addr, data);
 }
 
-static int dds_read(int addr)
+static int dds_read(int bus_channel, int addr)
 {
     int r;
 
@@ -31,7 +31,7 @@ static int dds_read(int addr)
 #ifdef CONFIG_DDS_AD9914
 #define DDS_READ_FLAG 256
 #endif
-    dds_write(addr | DDS_READ_FLAG, 0);
+    dds_write(bus_channel, addr | DDS_READ_FLAG, 0);
     while(rtio_i_status_read() & RTIO_I_STATUS_EMPTY);
     r = rtio_i_data_read();
     rtio_i_re_write(1);
@@ -75,16 +75,18 @@ void bridge_main(void)
                 struct msg_brg_dds_sel *msg;
 
                 msg = (struct msg_brg_dds_sel *)umsg;
-                dds_write(DDS_GPIO, msg->channel << 1);
+                dds_write(msg->bus_channel, DDS_GPIO, msg->channel << 1);
                 mailbox_acknowledge();
                 break;
             }
             case MESSAGE_TYPE_BRG_DDS_RESET: {
                 unsigned int g;
+                struct msg_brg_dds_reset *msg;
 
-                g = dds_read(DDS_GPIO);
-                dds_write(DDS_GPIO, g | 1);
-                dds_write(DDS_GPIO, g);
+                msg = (struct msg_brg_dds_reset *)umsg;
+                g = dds_read(msg->bus_channel, DDS_GPIO);
+                dds_write(msg->bus_channel, DDS_GPIO, g | 1);
+                dds_write(msg->bus_channel, DDS_GPIO, g);
 
                 mailbox_acknowledge();
                 break;
@@ -95,7 +97,7 @@ void bridge_main(void)
 
                 msg = (struct msg_brg_dds_read_request *)umsg;
                 rmsg.type = MESSAGE_TYPE_BRG_DDS_READ_REPLY;
-                rmsg.data = dds_read(msg->address);
+                rmsg.data = dds_read(msg->bus_channel, msg->address);
                 mailbox_send_and_wait(&rmsg);
                 break;
             }
@@ -103,14 +105,18 @@ void bridge_main(void)
                 struct msg_brg_dds_write *msg;
 
                 msg = (struct msg_brg_dds_write *)umsg;
-                dds_write(msg->address, msg->data);
+                dds_write(msg->bus_channel, msg->address, msg->data);
                 mailbox_acknowledge();
                 break;
             }
-            case MESSAGE_TYPE_BRG_DDS_FUD:
-                dds_write(DDS_FUD, 0);
+            case MESSAGE_TYPE_BRG_DDS_FUD: {
+                struct msg_brg_dds_fud *msg;
+
+                msg = (struct msg_brg_dds_fud *)umsg;
+                dds_write(msg->bus_channel, DDS_FUD, 0);
                 mailbox_acknowledge();
                 break;
+            }
             default:
                 mailbox_acknowledge();
                 break;
diff --git a/artiq/runtime/bridge_ctl.c b/artiq/runtime/bridge_ctl.c
index 11182efb9..1ebc4b0bc 100644
--- a/artiq/runtime/bridge_ctl.c
+++ b/artiq/runtime/bridge_ctl.c
@@ -43,30 +43,33 @@ void brg_ttlo(int n, int value)
     mailbox_send_and_wait(&msg);
 }
 
-void brg_ddssel(int channel)
+void brg_ddssel(int bus_channel, int channel)
 {
     struct msg_brg_dds_sel msg;
 
     msg.type = MESSAGE_TYPE_BRG_DDS_SEL;
+    msg.bus_channel = bus_channel;
     msg.channel = channel;
     mailbox_send_and_wait(&msg);
 }
 
-void brg_ddsreset(void)
+void brg_ddsreset(int bus_channel)
 {
-    struct msg_base msg;
+    struct msg_brg_dds_reset msg;
 
     msg.type = MESSAGE_TYPE_BRG_DDS_RESET;
+    msg.bus_channel = bus_channel;
     mailbox_send_and_wait(&msg);
 }
 
-unsigned int brg_ddsread(unsigned int address)
+unsigned int brg_ddsread(int bus_channel, unsigned int address)
 {
     struct msg_brg_dds_read_request msg;
     struct msg_brg_dds_read_reply *rmsg;
     unsigned int r;
 
     msg.type = MESSAGE_TYPE_BRG_DDS_READ_REQUEST;
+    msg.bus_channel = bus_channel;
     msg.address = address;
     mailbox_send(&msg);
     while(1) {
@@ -82,20 +85,22 @@ unsigned int brg_ddsread(unsigned int address)
     }
 }
 
-void brg_ddswrite(unsigned int address, unsigned int data)
+void brg_ddswrite(int bus_channel, unsigned int address, unsigned int data)
 {
     struct msg_brg_dds_write msg;
 
     msg.type = MESSAGE_TYPE_BRG_DDS_WRITE;
+    msg.bus_channel = bus_channel;
     msg.address = address;
     msg.data = data;
     mailbox_send_and_wait(&msg);
 }
 
-void brg_ddsfud(void)
+void brg_ddsfud(int bus_channel)
 {
-    struct msg_base msg;
+    struct msg_brg_dds_fud msg;
 
     msg.type = MESSAGE_TYPE_BRG_DDS_FUD;
+    msg.bus_channel = bus_channel;
     mailbox_send_and_wait(&msg);
 }
diff --git a/artiq/runtime/bridge_ctl.h b/artiq/runtime/bridge_ctl.h
index e6d2b7306..2929ee8f0 100644
--- a/artiq/runtime/bridge_ctl.h
+++ b/artiq/runtime/bridge_ctl.h
@@ -6,10 +6,10 @@ void brg_start(void);
 void brg_ttloe(int n, int value);
 void brg_ttlo(int n, int value);
 
-void brg_ddssel(int channel);
-void brg_ddsreset(void);
-unsigned int brg_ddsread(unsigned int address);
-void brg_ddswrite(unsigned int address, unsigned int data);
-void brg_ddsfud(void);
+void brg_ddssel(int bus_channel, int channel);
+void brg_ddsreset(int bus_channel);
+unsigned int brg_ddsread(int bus_channel, unsigned int address);
+void brg_ddswrite(int bus_channel, unsigned int address, unsigned int data);
+void brg_ddsfud(int bus_channel);
 
 #endif /* __BRIDGE_CTL_H */
diff --git a/artiq/runtime/dds.c b/artiq/runtime/dds.c
index 3f8e728c1..ea6d650db 100644
--- a/artiq/runtime/dds.c
+++ b/artiq/runtime/dds.c
@@ -26,11 +26,11 @@
 #endif
 
 #define DDS_WRITE(addr, data) do { \
-        rtio_output(now, CONFIG_RTIO_DDS_CHANNEL, addr, data); \
+        rtio_output(now, bus_channel, addr, data); \
         now += DURATION_WRITE; \
     } while(0)
 
-void dds_init(long long int timestamp, int channel)
+void dds_init(long long int timestamp, int bus_channel, int channel)
 {
     long long int now;
 
@@ -84,7 +84,8 @@ void dds_init(long long int timestamp, int channel)
  * to continuous phase mode. */
 static unsigned int continuous_phase_comp[CONFIG_DDS_CHANNEL_COUNT];
 
-static void dds_set_one(long long int now, long long int ref_time, unsigned int channel,
+static void dds_set_one(long long int now, long long int ref_time,
+    unsigned int bus_channel, unsigned int channel,
     unsigned int ftw, unsigned int pow, int phase_mode, unsigned int amplitude)
 {
     unsigned int channel_enc;
@@ -157,6 +158,7 @@ static void dds_set_one(long long int now, long long int ref_time, unsigned int
 }
 
 struct dds_set_params {
+    int bus_channel;
     int channel;
     unsigned int ftw;
     unsigned int pow;
@@ -189,20 +191,22 @@ void dds_batch_exit(void)
     now = batch_ref_time - batch_count*(DURATION_PROGRAM + DURATION_WRITE);
     for(i=0;i<batch_count;i++) {
         dds_set_one(now, batch_ref_time,
-            batch[i].channel, batch[i].ftw, batch[i].pow, batch[i].phase_mode,
+            batch[i].bus_channel, batch[i].channel,
+            batch[i].ftw, batch[i].pow, batch[i].phase_mode,
             batch[i].amplitude);
         now += DURATION_PROGRAM + DURATION_WRITE;
     }
     batch_mode = 0;
 }
 
-void dds_set(long long int timestamp, int channel,
+void dds_set(long long int timestamp, int bus_channel, int channel,
     unsigned int ftw, unsigned int pow, int phase_mode, unsigned int amplitude)
 {
     if(batch_mode) {
         if(batch_count >= DDS_MAX_BATCH)
             artiq_raise_from_c("DDSBatchError", "DDS batch error", 0, 0, 0);
         /* timestamp parameter ignored (determined by batch) */
+        batch[batch_count].bus_channel = bus_channel;
         batch[batch_count].channel = channel;
         batch[batch_count].ftw = ftw;
         batch[batch_count].pow = pow;
@@ -210,7 +214,8 @@ void dds_set(long long int timestamp, int channel,
         batch[batch_count].amplitude = amplitude;
         batch_count++;
     } else {
-        dds_set_one(timestamp - DURATION_PROGRAM, timestamp, channel, ftw, pow, phase_mode,
-                    amplitude);
+        dds_set_one(timestamp - DURATION_PROGRAM, timestamp,
+                    bus_channel, channel,
+                    ftw, pow, phase_mode, amplitude);
     }
 }
diff --git a/artiq/runtime/messages.h b/artiq/runtime/messages.h
index bd2adc912..49df62319 100644
--- a/artiq/runtime/messages.h
+++ b/artiq/runtime/messages.h
@@ -150,23 +150,37 @@ struct msg_brg_ttl_out {
 
 struct msg_brg_dds_sel {
     int type;
+    int bus_channel;
     int channel;
 };
 
+struct msg_brg_dds_reset {
+    int type;
+    int bus_channel;
+};
+
 struct msg_brg_dds_read_request {
     int type;
+    int bus_channel;
     unsigned int address;
 };
 
 struct msg_brg_dds_read_reply {
     int type;
+    int bus_channel;
     unsigned int data;
 };
 
 struct msg_brg_dds_write {
     int type;
+    int bus_channel;
     unsigned int address;
     unsigned int data;
 };
 
+struct msg_brg_dds_fud {
+    int type;
+    int bus_channel;
+};
+
 #endif /* __MESSAGES_H */
diff --git a/artiq/runtime/moninj.c b/artiq/runtime/moninj.c
index aea5bfd69..4a7c14205 100644
--- a/artiq/runtime/moninj.c
+++ b/artiq/runtime/moninj.c
@@ -36,13 +36,15 @@ struct monitor_reply {
     long long int ttl_levels;
     long long int ttl_oes;
     long long int ttl_overrides;
-    unsigned int dds_ftws[CONFIG_DDS_CHANNEL_COUNT];
-};
+    unsigned short int dds_rtio_first_channel;
+    unsigned short int dds_channels_per_bus;
+    unsigned int dds_ftws[CONFIG_RTIO_DDS_COUNT*CONFIG_DDS_CHANNELS_PER_BUS];
+} __attribute__((packed));
 
 static void moninj_monitor(const ip_addr_t *addr, u16_t port)
 {
     struct monitor_reply reply;
-    int i;
+    int i, j;
     struct pbuf *reply_p;
 
     reply.ttl_levels = 0;
@@ -64,11 +66,15 @@ static void moninj_monitor(const ip_addr_t *addr, u16_t port)
             reply.ttl_overrides |= 1LL << i;
     }
 
-    rtio_moninj_mon_chan_sel_write(CONFIG_RTIO_DDS_CHANNEL);
-    for(i=0;i<CONFIG_DDS_CHANNEL_COUNT;i++) {
-        rtio_moninj_mon_probe_sel_write(i);
-        rtio_moninj_mon_value_update_write(1);
-        reply.dds_ftws[i] = rtio_moninj_mon_value_read();
+    reply.dds_rtio_first_channel = CONFIG_RTIO_FIRST_DDS_CHANNEL;
+    reply.dds_channels_per_bus = CONFIG_DDS_CHANNELS_PER_BUS;
+    for(j=0;j<CONFIG_RTIO_DDS_COUNT;j++) {
+        rtio_moninj_mon_chan_sel_write(CONFIG_RTIO_FIRST_DDS_CHANNEL+j);
+        for(i=0;i<CONFIG_DDS_CHANNELS_PER_BUS;i++) {
+            rtio_moninj_mon_probe_sel_write(i);
+            rtio_moninj_mon_value_update_write(1);
+            reply.dds_ftws[CONFIG_DDS_CHANNELS_PER_BUS*j+i] = rtio_moninj_mon_value_read();
+        }
     }
 
     reply_p = pbuf_alloc(PBUF_TRANSPORT, sizeof(struct monitor_reply), PBUF_RAM);
diff --git a/artiq/runtime/test_mode.c b/artiq/runtime/test_mode.c
index 9789ad11c..d3b8f627f 100644
--- a/artiq/runtime/test_mode.c
+++ b/artiq/runtime/test_mode.c
@@ -104,6 +104,28 @@ static void ttlo(char *n, char *value)
     brg_ttlo(n2, value2);
 }
 
+static int bus_channel = CONFIG_RTIO_FIRST_DDS_CHANNEL;
+
+static void ddsbus(char *n)
+{
+    char *c;
+    unsigned int n2;
+
+    if(*n == 0) {
+        printf("ddsbus <n>\n");
+        return;
+    }
+
+    n2 = strtoul(n, &c, 0);
+    if(*c != 0) {
+        printf("incorrect bus channel\n");
+        return;
+    }
+
+    bus_channel = n2;
+}
+
+
 static void ddssel(char *n)
 {
     char *c;
@@ -123,7 +145,7 @@ static void ddssel(char *n)
 #ifdef CONFIG_DDS_ONEHOT_SEL
     n2 = 1 << n2;
 #endif
-    brg_ddssel(n2);
+    brg_ddssel(bus_channel, n2);
 }
 
 static void ddsw(char *addr, char *value)
@@ -147,7 +169,7 @@ static void ddsw(char *addr, char *value)
         return;
     }
 
-    brg_ddswrite(addr2, value2);
+    brg_ddswrite(bus_channel, addr2, value2);
 }
 
 static void ddsr(char *addr)
@@ -167,16 +189,16 @@ static void ddsr(char *addr)
     }
 
 #ifdef CONFIG_DDS_AD9858
-    printf("0x%02x\n", brg_ddsread(addr2));
+    printf("0x%02x\n", brg_ddsread(bus_channel, addr2));
 #endif
 #ifdef CONFIG_DDS_AD9914
-    printf("0x%04x\n", brg_ddsread(addr2));
+    printf("0x%04x\n", brg_ddsread(bus_channel, addr2));
 #endif
 }
 
 static void ddsfud(void)
 {
-    brg_ddsfud();
+    brg_ddsfud(bus_channel);
 }
 
 static void ddsftw(char *n, char *ftw)
@@ -203,36 +225,36 @@ static void ddsftw(char *n, char *ftw)
 #ifdef CONFIG_DDS_ONEHOT_SEL
     n2 = 1 << n2;
 #endif
-    brg_ddssel(n2);
+    brg_ddssel(bus_channel, n2);
 
 #ifdef CONFIG_DDS_AD9858
-    brg_ddswrite(DDS_FTW0, ftw2 & 0xff);
-    brg_ddswrite(DDS_FTW1, (ftw2 >> 8) & 0xff);
-    brg_ddswrite(DDS_FTW2, (ftw2 >> 16) & 0xff);
-    brg_ddswrite(DDS_FTW3, (ftw2 >> 24) & 0xff);
+    brg_ddswrite(bus_channel, DDS_FTW0, ftw2 & 0xff);
+    brg_ddswrite(bus_channel, DDS_FTW1, (ftw2 >> 8) & 0xff);
+    brg_ddswrite(bus_channel, DDS_FTW2, (ftw2 >> 16) & 0xff);
+    brg_ddswrite(bus_channel, DDS_FTW3, (ftw2 >> 24) & 0xff);
 #endif
 #ifdef CONFIG_DDS_AD9914
-    brg_ddswrite(DDS_FTWL, ftw2 & 0xffff);
-    brg_ddswrite(DDS_FTWH, (ftw2 >> 16) & 0xffff);
+    brg_ddswrite(bus_channel, DDS_FTWL, ftw2 & 0xffff);
+    brg_ddswrite(bus_channel, DDS_FTWH, (ftw2 >> 16) & 0xffff);
 #endif
 
-    brg_ddsfud();
+    brg_ddsfud(bus_channel);
 }
 
 static void ddsreset(void)
 {
-    brg_ddsreset();
+    brg_ddsreset(bus_channel);
 }
 
 #ifdef CONFIG_DDS_AD9858
 static void ddsinit(void)
 {
-    brg_ddsreset();
-    brg_ddswrite(DDS_CFR0, 0x78);
-    brg_ddswrite(DDS_CFR1, 0x00);
-    brg_ddswrite(DDS_CFR2, 0x00);
-    brg_ddswrite(DDS_CFR3, 0x00);
-    brg_ddsfud();
+    brg_ddsreset(bus_channel);
+    brg_ddswrite(bus_channel, DDS_CFR0, 0x78);
+    brg_ddswrite(bus_channel, DDS_CFR1, 0x00);
+    brg_ddswrite(bus_channel, DDS_CFR2, 0x00);
+    brg_ddswrite(bus_channel, DDS_CFR3, 0x00);
+    brg_ddsfud(bus_channel);
 }
 #endif
 
@@ -241,17 +263,17 @@ static void ddsinit(void)
 {
     long long int t;
 
-    brg_ddsreset();
-    brg_ddswrite(DDS_CFR1H, 0x0000); /* Enable cosine output */
-    brg_ddswrite(DDS_CFR2L, 0x8900); /* Enable matched latency */
-    brg_ddswrite(DDS_CFR2H, 0x0080); /* Enable profile mode */
-    brg_ddswrite(DDS_ASF, 0x0fff); /* Set amplitude to maximum */
-    brg_ddswrite(DDS_CFR4H, 0x0105); /* Enable DAC calibration */
-    brg_ddswrite(DDS_FUD, 0);
+    brg_ddsreset(bus_channel);
+    brg_ddswrite(bus_channel, DDS_CFR1H, 0x0000); /* Enable cosine output */
+    brg_ddswrite(bus_channel, DDS_CFR2L, 0x8900); /* Enable matched latency */
+    brg_ddswrite(bus_channel, DDS_CFR2H, 0x0080); /* Enable profile mode */
+    brg_ddswrite(bus_channel, DDS_ASF, 0x0fff); /* Set amplitude to maximum */
+    brg_ddswrite(bus_channel, DDS_CFR4H, 0x0105); /* Enable DAC calibration */
+    brg_ddswrite(bus_channel, DDS_FUD, 0);
     t = clock_get_ms();
     while(clock_get_ms() < t + 2);
-    brg_ddswrite(DDS_CFR4H, 0x0005); /* Disable DAC calibration */
-    brg_ddsfud();
+    brg_ddswrite(bus_channel, DDS_CFR4H, 0x0005); /* Disable DAC calibration */
+    brg_ddsfud(bus_channel);
 }
 #endif
 
@@ -265,34 +287,34 @@ static void do_ddstest_one(unsigned int i)
     unsigned int f, g, j;
 
 #ifdef CONFIG_DDS_ONEHOT_SEL
-    brg_ddssel(1 << i);
+    brg_ddssel(bus_channel, 1 << i);
 #else
-    brg_ddssel(i);
+    brg_ddssel(bus_channel, i);
 #endif
     ddsinit();
 
     for(j=0; j<12; j++) {
         f = v[j];
 #ifdef CONFIG_DDS_AD9858
-        brg_ddswrite(DDS_FTW0, f & 0xff);
-        brg_ddswrite(DDS_FTW1, (f >> 8) & 0xff);
-        brg_ddswrite(DDS_FTW2, (f >> 16) & 0xff);
-        brg_ddswrite(DDS_FTW3, (f >> 24) & 0xff);
+        brg_ddswrite(bus_channel, DDS_FTW0, f & 0xff);
+        brg_ddswrite(bus_channel, DDS_FTW1, (f >> 8) & 0xff);
+        brg_ddswrite(bus_channel, DDS_FTW2, (f >> 16) & 0xff);
+        brg_ddswrite(bus_channel, DDS_FTW3, (f >> 24) & 0xff);
 #endif
 #ifdef CONFIG_DDS_AD9914
-        brg_ddswrite(DDS_FTWL, f & 0xffff);
-        brg_ddswrite(DDS_FTWH, (f >> 16) & 0xffff);
+        brg_ddswrite(bus_channel, DDS_FTWL, f & 0xffff);
+        brg_ddswrite(bus_channel, DDS_FTWH, (f >> 16) & 0xffff);
 #endif
         brg_ddsfud();
 #ifdef CONFIG_DDS_AD9858
-        g = brg_ddsread(DDS_FTW0);
-        g |= brg_ddsread(DDS_FTW1) << 8;
-        g |= brg_ddsread(DDS_FTW2) << 16;
-        g |= brg_ddsread(DDS_FTW3) << 24;
+        g = brg_ddsread(bus_channel, DDS_FTW0);
+        g |= brg_ddsread(bus_channel, DDS_FTW1) << 8;
+        g |= brg_ddsread(bus_channel, DDS_FTW2) << 16;
+        g |= brg_ddsread(bus_channel, DDS_FTW3) << 24;
 #endif
 #ifdef CONFIG_DDS_AD9914
-        g = brg_ddsread(DDS_FTWL);
-        g |= brg_ddsread(DDS_FTWH) << 16;
+        g = brg_ddsread(bus_channel, DDS_FTWL);
+        g |= brg_ddsread(bus_channel, DDS_FTWH) << 16;
 #endif
         if(g != f)
             printf("readback fail on DDS %d, 0x%08x != 0x%08x\n", i, g, f);
@@ -330,7 +352,7 @@ static void ddstest(char *n, char *channel)
             do_ddstest_one(channel2);
     } else {
         for(i=0;i<n2;i++)
-            for(j=0;j<CONFIG_DDS_CHANNEL_COUNT;j++)
+            for(j=0;j<CONFIG_DDS_CHANNELS_PER_BUS;j++)
                 do_ddstest_one(j);
     }
 }
@@ -542,6 +564,7 @@ static void help(void)
     puts("clksrc <n>      - select RTIO clock source");
     puts("ttloe <n> <v>   - set TTL output enable");
     puts("ttlo <n> <v>    - set TTL output value");
+    puts("ddsbus <n>      - select the DDS bus RTIO channel");
     puts("ddssel <n>      - select a DDS");
     puts("ddsinit         - reset, config, FUD DDS");
     puts("ddsreset        - reset DDS");
@@ -624,6 +647,7 @@ static void do_command(char *c)
     else if(strcmp(token, "ttloe") == 0) ttloe(get_token(&c), get_token(&c));
     else if(strcmp(token, "ttlo") == 0) ttlo(get_token(&c), get_token(&c));
 
+    else if(strcmp(token, "ddsbus") == 0) ddsbus(get_token(&c));
     else if(strcmp(token, "ddssel") == 0) ddssel(get_token(&c));
     else if(strcmp(token, "ddsw") == 0) ddsw(get_token(&c), get_token(&c));
     else if(strcmp(token, "ddsr") == 0) ddsr(get_token(&c));
diff --git a/doc/manual/core_device.rst b/doc/manual/core_device.rst
index 484f69358..9489277a7 100644
--- a/doc/manual/core_device.rst
+++ b/doc/manual/core_device.rst
@@ -89,6 +89,8 @@ The board has RTIO SPI buses mapped as follows:
 | 25           | SPI2_CS_N   | SPI2_MOSI   | SPI2_MISO | SPI2_CLK   |
 +--------------+-------------+-------------+-----------+------------+
 
+The DDS bus is on channel 26.
+
 
 NIST QC2
 ++++++++
diff --git a/examples/master/device_db.pyon b/examples/master/device_db.pyon
index f10b1e6b2..05f9bc7ca 100644
--- a/examples/master/device_db.pyon
+++ b/examples/master/device_db.pyon
@@ -14,6 +14,12 @@
         "class": "Core",
         "arguments": {"ref_period": 1e-9}
     },
+    "core_dds": {
+        "type": "local",
+        "module": "artiq.coredevice.dds",
+        "class": "CoreDDS",
+        "arguments": {"sysclk": 3e9}
+    },
 
     "i2c_switch": {
         "type": "local",
@@ -121,30 +127,24 @@
         "arguments": {"spi_device": "spi0", "ldac_device": "ttl0"}
     },
 
-    "dds_bus": {
-        "type": "local",
-        "module": "artiq.coredevice.dds",
-        "class": "DDSBus",
-        "arguments": {}
-    },
     "dds0": {
         "type": "local",
         "module": "artiq.coredevice.dds",
         "class": "AD9914",
-        "arguments": {"sysclk": 3e9, "channel": 0},
+        "arguments": {"bus_channel": 26, "channel": 0},
         "comment": "Comments work in DDS panel as well"
     },
     "dds1": {
         "type": "local",
         "module": "artiq.coredevice.dds",
         "class": "AD9914",
-        "arguments": {"sysclk": 3e9, "channel": 1}
+        "arguments": {"bus_channel": 26, "channel": 1}
     },
     "dds2": {
         "type": "local",
         "module": "artiq.coredevice.dds",
         "class": "AD9914",
-        "arguments": {"sysclk": 3e9, "channel": 2}
+        "arguments": {"bus_channel": 26, "channel": 2}
     },
 
     "qc_q1_0": {

From de37487a5c8f1d69609a906d2e505a71308ed709 Mon Sep 17 00:00:00 2001
From: Sebastien Bourdeauducq <sb@m-labs.hk>
Date: Wed, 9 Mar 2016 18:27:51 +0800
Subject: [PATCH 09/38] runtime: fix dds declarations

---
 artiq/runtime/dds.c | 2 +-
 artiq/runtime/dds.h | 4 ++--
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/artiq/runtime/dds.c b/artiq/runtime/dds.c
index ea6d650db..18cb11cec 100644
--- a/artiq/runtime/dds.c
+++ b/artiq/runtime/dds.c
@@ -85,7 +85,7 @@ void dds_init(long long int timestamp, int bus_channel, int channel)
 static unsigned int continuous_phase_comp[CONFIG_DDS_CHANNEL_COUNT];
 
 static void dds_set_one(long long int now, long long int ref_time,
-    unsigned int bus_channel, unsigned int channel,
+    int bus_channel, int channel,
     unsigned int ftw, unsigned int pow, int phase_mode, unsigned int amplitude)
 {
     unsigned int channel_enc;
diff --git a/artiq/runtime/dds.h b/artiq/runtime/dds.h
index 6d4e5d972..10fa3fffc 100644
--- a/artiq/runtime/dds.h
+++ b/artiq/runtime/dds.h
@@ -54,10 +54,10 @@ enum {
     PHASE_MODE_TRACKING = 2
 };
 
-void dds_init(long long int timestamp, int channel);
+void dds_init(long long int timestamp, int bus_channel, int channel);
 void dds_batch_enter(long long int timestamp);
 void dds_batch_exit(void);
-void dds_set(long long int timestamp, int channel,
+void dds_set(long long int timestamp, int bus_channel, int channel,
     unsigned int ftw, unsigned int pow, int phase_mode, unsigned int amplitude);
 
 #endif /* __DDS_H */

From 861c4a9ae523927d51de498c8787ebd1ae60acf4 Mon Sep 17 00:00:00 2001
From: Sebastien Bourdeauducq <sb@m-labs.hk>
Date: Wed, 9 Mar 2016 19:03:05 +0800
Subject: [PATCH 10/38] fix more multi-DDS-bus problems

---
 artiq/runtime/dds.c       | 13 +++++++++----
 artiq/runtime/test_mode.c |  2 +-
 2 files changed, 10 insertions(+), 5 deletions(-)

diff --git a/artiq/runtime/dds.c b/artiq/runtime/dds.c
index 18cb11cec..b9b07bde7 100644
--- a/artiq/runtime/dds.c
+++ b/artiq/runtime/dds.c
@@ -82,7 +82,7 @@ void dds_init(long long int timestamp, int bus_channel, int channel)
 
 /* Compensation to keep phase continuity when switching from absolute or tracking
  * to continuous phase mode. */
-static unsigned int continuous_phase_comp[CONFIG_DDS_CHANNEL_COUNT];
+static unsigned int continuous_phase_comp[CONFIG_RTIO_DDS_COUNT][CONFIG_DDS_CHANNELS_PER_BUS];
 
 static void dds_set_one(long long int now, long long int ref_time,
     int bus_channel, int channel,
@@ -90,10 +90,15 @@ static void dds_set_one(long long int now, long long int ref_time,
 {
     unsigned int channel_enc;
 
-    if(channel >= CONFIG_DDS_CHANNEL_COUNT) {
+    if((channel < 0) || (channel >= CONFIG_DDS_CHANNELS_PER_BUS)) {
         core_log("Attempted to set invalid DDS channel\n");
         return;
     }
+    if((bus_channel < CONFIG_RTIO_FIRST_DDS_CHANNEL)
+       || (bus_channel >= (CONFIG_RTIO_FIRST_DDS_CHANNEL+CONFIG_RTIO_DDS_COUNT))) {
+        core_log("Attempted to use invalid DDS bus\n");
+        return;
+    }
 #ifdef CONFIG_DDS_ONEHOT_SEL
     channel_enc = 1 << channel;
 #else
@@ -125,7 +130,7 @@ static void dds_set_one(long long int now, long long int ref_time,
         /* Disable autoclear phase accumulator and enables OSK. */
         DDS_WRITE(DDS_CFR1L, 0x0108);
 #endif
-        pow += continuous_phase_comp[channel];
+        pow += continuous_phase_comp[bus_channel-CONFIG_RTIO_FIRST_DDS_CHANNEL][channel];
     } else {
         long long int fud_time;
 
@@ -141,7 +146,7 @@ static void dds_set_one(long long int now, long long int ref_time,
         pow -= (ref_time - fud_time)*CONFIG_DDS_RTIO_CLK_RATIO*ftw >> (32-DDS_POW_WIDTH);
         if(phase_mode == PHASE_MODE_TRACKING)
             pow += ref_time*CONFIG_DDS_RTIO_CLK_RATIO*ftw >> (32-DDS_POW_WIDTH);
-        continuous_phase_comp[channel] = pow;
+        continuous_phase_comp[bus_channel-CONFIG_RTIO_FIRST_DDS_CHANNEL][channel] = pow;
     }
 
 #ifdef CONFIG_DDS_AD9858
diff --git a/artiq/runtime/test_mode.c b/artiq/runtime/test_mode.c
index d3b8f627f..3fcb8756b 100644
--- a/artiq/runtime/test_mode.c
+++ b/artiq/runtime/test_mode.c
@@ -305,7 +305,7 @@ static void do_ddstest_one(unsigned int i)
         brg_ddswrite(bus_channel, DDS_FTWL, f & 0xffff);
         brg_ddswrite(bus_channel, DDS_FTWH, (f >> 16) & 0xffff);
 #endif
-        brg_ddsfud();
+        brg_ddsfud(bus_channel);
 #ifdef CONFIG_DDS_AD9858
         g = brg_ddsread(bus_channel, DDS_FTW0);
         g |= brg_ddsread(bus_channel, DDS_FTW1) << 8;

From b0de9ee90a5c54c80910dc314dfa8b14280448b2 Mon Sep 17 00:00:00 2001
From: Robert Jordens <rj@m-labs.hk>
Date: Wed, 9 Mar 2016 12:27:45 +0100
Subject: [PATCH 11/38] coredevice: add RTIOBusy to __all__

---
 artiq/coredevice/__init__.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/artiq/coredevice/__init__.py b/artiq/coredevice/__init__.py
index 4eac74667..9ea9f5c33 100644
--- a/artiq/coredevice/__init__.py
+++ b/artiq/coredevice/__init__.py
@@ -7,6 +7,6 @@ from artiq.coredevice.dds import (PHASE_MODE_CONTINUOUS, PHASE_MODE_ABSOLUTE,
 
 __all__ = []
 __all__ += ["RTIOUnderflow", "RTIOSequenceError", "RTIOCollision",
-            "RTIOOverflow", "DDSBatchError", "CacheError"]
+            "RTIOOverflow", "RTIOBusy", "DDSBatchError", "CacheError"]
 __all__ += ["PHASE_MODE_CONTINUOUS", "PHASE_MODE_ABSOLUTE",
             "PHASE_MODE_TRACKING"]

From 522ec60f6ee630f2b357a6cb53640b2161f3730c Mon Sep 17 00:00:00 2001
From: Robert Jordens <rj@m-labs.hk>
Date: Wed, 9 Mar 2016 12:28:07 +0100
Subject: [PATCH 12/38] hardware_testbench: don't allow unused *args

---
 artiq/test/hardware_testbench.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/artiq/test/hardware_testbench.py b/artiq/test/hardware_testbench.py
index b8c1de6f8..1eaa6530d 100644
--- a/artiq/test/hardware_testbench.py
+++ b/artiq/test/hardware_testbench.py
@@ -109,7 +109,7 @@ class ExperimentCase(unittest.TestCase):
             # skip if ddb does not match requirements
             raise unittest.SkipTest(*e.args)
 
-    def execute(self, cls, *args, **kwargs):
+    def execute(self, cls, **kwargs):
         expid = {
             "file": sys.modules[cls.__module__].__file__,
             "class_name": cls.__name__,

From 0bd9add95e4213a227e810922ba512e95d6c5d3a Mon Sep 17 00:00:00 2001
From: Robert Jordens <rj@m-labs.hk>
Date: Wed, 9 Mar 2016 12:32:31 +0100
Subject: [PATCH 13/38] spi: fix frequency_to_div()

---
 artiq/coredevice/spi.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/artiq/coredevice/spi.py b/artiq/coredevice/spi.py
index 20095a8ff..80909e527 100644
--- a/artiq/coredevice/spi.py
+++ b/artiq/coredevice/spi.py
@@ -1,5 +1,5 @@
 from artiq.language.core import (kernel, portable, seconds_to_mu, now_mu,
-                                 delay_mu, int)
+                                 delay_mu, int, mu_to_seconds)
 from artiq.language.units import MHz
 from artiq.coredevice.rtio import rtio_output, rtio_input_data
 
@@ -58,7 +58,7 @@ class SPIMaster:
 
     @portable
     def frequency_to_div(self, f):
-        return int(1/(f*self.ref_period)) + 1
+        return int(1/(f*mu_to_seconds(self.ref_period_mu))) + 1
 
     @kernel
     def set_config(self, flags=0, write_freq=20*MHz, read_freq=20*MHz):

From db5231216d2ba653103e634654e2668c1df7e19f Mon Sep 17 00:00:00 2001
From: Robert Jordens <rj@m-labs.hk>
Date: Wed, 9 Mar 2016 13:42:15 +0100
Subject: [PATCH 14/38] rtio: remove unused include

---
 artiq/runtime/rtio.h | 1 -
 1 file changed, 1 deletion(-)

diff --git a/artiq/runtime/rtio.h b/artiq/runtime/rtio.h
index cdaf8a6e1..aad6460ba 100644
--- a/artiq/runtime/rtio.h
+++ b/artiq/runtime/rtio.h
@@ -2,7 +2,6 @@
 #define __RTIO_H
 
 #include <stdarg.h>
-#include "generated/csr.h"
 
 #define RTIO_O_STATUS_FULL 1
 #define RTIO_O_STATUS_UNDERFLOW 2

From bf188d05bf3d2daf28a7c6551aa1624edf2d1d82 Mon Sep 17 00:00:00 2001
From: Robert Jordens <rj@m-labs.hk>
Date: Wed, 9 Mar 2016 13:42:32 +0100
Subject: [PATCH 15/38] rtio: cleanup RTIOBusy message

---
 artiq/runtime/rtio.c | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/artiq/runtime/rtio.c b/artiq/runtime/rtio.c
index 385011cf5..92f02f651 100644
--- a/artiq/runtime/rtio.c
+++ b/artiq/runtime/rtio.c
@@ -42,8 +42,8 @@ static void rtio_process_exceptional_status(
     if(status & RTIO_O_STATUS_BUSY) {
         rtio_o_busy_reset_write(1);
         artiq_raise_from_c("RTIOBusy",
-            "RTIO busy at {0} mu, channel {1}",
-            timestamp, channel, 0);
+            "RTIO busy on channel {0}",
+            channel, 0, 0);
     }
 }
 

From 58e0e670fc7d61cdaa6fea9d6434d2d047e441a7 Mon Sep 17 00:00:00 2001
From: Robert Jordens <rj@m-labs.hk>
Date: Wed, 9 Mar 2016 15:39:16 +0100
Subject: [PATCH 16/38] tests: test spi business

---
 artiq/test/coredevice/test_spi.py | 24 ++++++++++++++++++++++++
 1 file changed, 24 insertions(+)
 create mode 100644 artiq/test/coredevice/test_spi.py

diff --git a/artiq/test/coredevice/test_spi.py b/artiq/test/coredevice/test_spi.py
new file mode 100644
index 000000000..ac1838d5d
--- /dev/null
+++ b/artiq/test/coredevice/test_spi.py
@@ -0,0 +1,24 @@
+from artiq.experiment import *
+from artiq.test.hardware_testbench import ExperimentCase
+
+
+class Busy(EnvExperiment):
+    def build(self):
+        self.setattr_device("core")
+        self.setattr_device("spi0")
+        self.setattr_device("led")
+
+    @kernel
+    def run(self):
+        self.spi0.set_config_mu()
+        delay(-8*ns)
+        self.spi0.set_config_mu()  # causes the error
+        self.led.on()
+        self.led.sync()            # registers the error
+        self.core.break_realtime()
+        self.spi0.set_config_mu()  # raises the error
+
+
+class SPITest(ExperimentCase):
+    def test_busy(self):
+        self.execute(Busy)

From 1c706fae49bb5d046d167358709df9f2f09b360b Mon Sep 17 00:00:00 2001
From: Sebastien Bourdeauducq <sb@m-labs.hk>
Date: Wed, 9 Mar 2016 23:27:41 +0800
Subject: [PATCH 17/38] examples: dds_bus -> core_dds

---
 .../master/repository/coredevice_examples/photon_histogram.py | 4 ++--
 .../master/repository/coredevice_examples/simple/dds_test.py  | 4 ++--
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/examples/master/repository/coredevice_examples/photon_histogram.py b/examples/master/repository/coredevice_examples/photon_histogram.py
index 307b3a52e..5a919aa4a 100644
--- a/examples/master/repository/coredevice_examples/photon_histogram.py
+++ b/examples/master/repository/coredevice_examples/photon_histogram.py
@@ -6,7 +6,7 @@ class PhotonHistogram(EnvExperiment):
 
     def build(self):
         self.setattr_device("core")
-        self.setattr_device("dds_bus")
+        self.setattr_device("core_dds")
         self.setattr_device("bd_dds")
         self.setattr_device("bd_sw")
         self.setattr_device("bdd_dds")
@@ -22,7 +22,7 @@ class PhotonHistogram(EnvExperiment):
 
     @kernel
     def program_cooling(self):
-        with self.dds_bus.batch:
+        with self.core_dds.batch:
             self.bd_dds.set(200*MHz)
             self.bdd_dds.set(300*MHz)
 
diff --git a/examples/master/repository/coredevice_examples/simple/dds_test.py b/examples/master/repository/coredevice_examples/simple/dds_test.py
index a0b60bf73..5e262211a 100644
--- a/examples/master/repository/coredevice_examples/simple/dds_test.py
+++ b/examples/master/repository/coredevice_examples/simple/dds_test.py
@@ -6,7 +6,7 @@ class DDSTest(EnvExperiment):
 
     def build(self):
         self.setattr_device("core")
-        self.setattr_device("dds_bus")
+        self.setattr_device("core_dds")
         self.setattr_device("dds0")
         self.setattr_device("dds1")
         self.setattr_device("dds2")
@@ -17,7 +17,7 @@ class DDSTest(EnvExperiment):
 
     @kernel
     def run(self):
-        with self.dds_bus.batch:
+        with self.core_dds.batch:
             self.dds1.set(120*MHz)
             self.dds2.set(200*MHz)
         delay(1*us)

From 03b53c3af999740e134cf4fe4b24d07832b623cc Mon Sep 17 00:00:00 2001
From: Sebastien Bourdeauducq <sb@m-labs.hk>
Date: Wed, 9 Mar 2016 23:37:04 +0800
Subject: [PATCH 18/38] rtio: disable replace on rt2wb channels

---
 artiq/gateware/rtio/core.py         | 24 ++++++++++++++----------
 artiq/gateware/rtio/phy/wishbone.py |  3 ++-
 artiq/gateware/rtio/rtlink.py       |  4 +++-
 3 files changed, 19 insertions(+), 12 deletions(-)

diff --git a/artiq/gateware/rtio/core.py b/artiq/gateware/rtio/core.py
index 8c2d1b237..d1095137b 100644
--- a/artiq/gateware/rtio/core.py
+++ b/artiq/gateware/rtio/core.py
@@ -129,24 +129,28 @@ class _OutputManager(Module):
         collision = Signal()
         any_error = Signal()
         nop = Signal()
-        self.sync.rsys += [
+        if interface.enable_replace:
             # Note: replace may be asserted at the same time as collision
             # when addresses are different. In that case, it is a collision.
-            replace.eq(self.ev.timestamp == buf.timestamp),
+            self.sync.rsys += replace.eq(self.ev.timestamp == buf.timestamp)
+        self.sync.rsys += \
             # Detect sequence errors on coarse timestamps only
             # so that they are mutually exclusive with collision errors.
             sequence_error.eq(self.ev.timestamp[fine_ts_width:]
                               < buf.timestamp[fine_ts_width:])
-        ]
-        if hasattr(self.ev, "a"):
-            different_addresses = self.ev.a != buf.a
+        if interface.enable_replace:
+            if hasattr(self.ev, "a"):
+                different_addresses = self.ev.a != buf.a
+            else:
+                different_addresses = 0
+            if fine_ts_width:
+                self.sync.rsys += collision.eq(
+                    (self.ev.timestamp[fine_ts_width:] == buf.timestamp[fine_ts_width:])
+                    & ((self.ev.timestamp[:fine_ts_width] != buf.timestamp[:fine_ts_width])
+                       |different_addresses))
         else:
-            different_addresses = 0
-        if fine_ts_width:
             self.sync.rsys += collision.eq(
-                (self.ev.timestamp[fine_ts_width:] == buf.timestamp[fine_ts_width:])
-                & ((self.ev.timestamp[:fine_ts_width] != buf.timestamp[:fine_ts_width])
-                   |different_addresses))
+                self.ev.timestamp[fine_ts_width:] == buf.timestamp[fine_ts_width:])
         self.comb += any_error.eq(sequence_error | collision)
         if interface.suppress_nop:
             # disable NOP at reset: do not suppress a first write with all 0s
diff --git a/artiq/gateware/rtio/phy/wishbone.py b/artiq/gateware/rtio/phy/wishbone.py
index 1b58525f6..2e9d6bc4a 100644
--- a/artiq/gateware/rtio/phy/wishbone.py
+++ b/artiq/gateware/rtio/phy/wishbone.py
@@ -13,7 +13,8 @@ class RT2WB(Module):
             rtlink.OInterface(
                 len(wb.dat_w),
                 address_width + 1,
-                suppress_nop=False),
+                suppress_nop=False,
+                enable_replace=False),
             rtlink.IInterface(
                 len(wb.dat_r),
                 timestamped=False)
diff --git a/artiq/gateware/rtio/rtlink.py b/artiq/gateware/rtio/rtlink.py
index c9d281ceb..b8bda7aac 100644
--- a/artiq/gateware/rtio/rtlink.py
+++ b/artiq/gateware/rtio/rtlink.py
@@ -3,7 +3,8 @@ from migen import *
 
 class OInterface:
     def __init__(self, data_width, address_width=0,
-                 fine_ts_width=0, suppress_nop=True):
+                 fine_ts_width=0, suppress_nop=True,
+                 enable_replace=True):
         self.stb = Signal()
         self.busy = Signal()
 
@@ -15,6 +16,7 @@ class OInterface:
             self.fine_ts = Signal(fine_ts_width)
 
         self.suppress_nop = suppress_nop
+        self.enable_replace = enable_replace
 
     @classmethod
     def like(cls, other):

From 3f8e431de6f3dbc4d0f3203d3b2a4a11644c5325 Mon Sep 17 00:00:00 2001
From: Robert Jordens <rj@m-labs.hk>
Date: Wed, 9 Mar 2016 17:10:21 +0100
Subject: [PATCH 19/38] rtio/core: fix syntax

---
 artiq/gateware/rtio/core.py | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/artiq/gateware/rtio/core.py b/artiq/gateware/rtio/core.py
index d1095137b..a1523f7ab 100644
--- a/artiq/gateware/rtio/core.py
+++ b/artiq/gateware/rtio/core.py
@@ -133,11 +133,10 @@ class _OutputManager(Module):
             # Note: replace may be asserted at the same time as collision
             # when addresses are different. In that case, it is a collision.
             self.sync.rsys += replace.eq(self.ev.timestamp == buf.timestamp)
-        self.sync.rsys += \
             # Detect sequence errors on coarse timestamps only
             # so that they are mutually exclusive with collision errors.
-            sequence_error.eq(self.ev.timestamp[fine_ts_width:]
-                              < buf.timestamp[fine_ts_width:])
+        self.sync.rsys += sequence_error.eq(self.ev.timestamp[fine_ts_width:] <
+                                            buf.timestamp[fine_ts_width:])
         if interface.enable_replace:
             if hasattr(self.ev, "a"):
                 different_addresses = self.ev.a != buf.a

From b50e3fabb03b54f98910e21e92cd0327b39af493 Mon Sep 17 00:00:00 2001
From: Robert Jordens <rj@m-labs.hk>
Date: Wed, 9 Mar 2016 14:34:10 +0100
Subject: [PATCH 20/38] hardware_testbench: also print artiq_core_exeption

---
 artiq/test/hardware_testbench.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/artiq/test/hardware_testbench.py b/artiq/test/hardware_testbench.py
index 1eaa6530d..a8726fce7 100644
--- a/artiq/test/hardware_testbench.py
+++ b/artiq/test/hardware_testbench.py
@@ -124,3 +124,6 @@ class ExperimentCase(unittest.TestCase):
         except CompileError as error:
             # Reduce amount of text on terminal.
             raise error from None
+        except Exception as exn:
+            exn.args = exn.args[0] + "\n" + str(exn.artiq_core_exception),
+            raise exn

From b32217cc84c51734d66d5b3d7e2540f5f09baa3e Mon Sep 17 00:00:00 2001
From: Robert Jordens <rj@m-labs.hk>
Date: Wed, 9 Mar 2016 17:23:02 +0100
Subject: [PATCH 21/38] coredevice: fix _DDSGeneric __init__ args

---
 artiq/coredevice/dds.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/artiq/coredevice/dds.py b/artiq/coredevice/dds.py
index b375292de..622292476 100644
--- a/artiq/coredevice/dds.py
+++ b/artiq/coredevice/dds.py
@@ -82,7 +82,7 @@ class _DDSGeneric:
     :param bus: name of the DDS bus device that this DDS is connected to.
     :param channel: channel number of the DDS device to control.
     """
-    def __init__(self, dmgr, sysclk, bus_channel, channel, core_dds_device="core_dds"):
+    def __init__(self, dmgr, bus_channel, channel, core_dds_device="core_dds"):
         self.core_dds = dmgr.get(core_dds_device)
         self.core = self.core_dds.core
         self.bus_channel = bus_channel

From 9a661bd27323e57ee88d6f563b5e2cc3429b7737 Mon Sep 17 00:00:00 2001
From: Robert Jordens <rj@m-labs.hk>
Date: Wed, 9 Mar 2016 17:22:15 +0100
Subject: [PATCH 22/38] hardware_testbench: clean up artiq_core_exeption
 printing

---
 artiq/test/hardware_testbench.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/artiq/test/hardware_testbench.py b/artiq/test/hardware_testbench.py
index a8726fce7..3cb0d3700 100644
--- a/artiq/test/hardware_testbench.py
+++ b/artiq/test/hardware_testbench.py
@@ -125,5 +125,7 @@ class ExperimentCase(unittest.TestCase):
             # Reduce amount of text on terminal.
             raise error from None
         except Exception as exn:
-            exn.args = exn.args[0] + "\n" + str(exn.artiq_core_exception),
+            if hasattr(exn, "artiq_core_exception"):
+                exn.args = "{}\n{}".format(exn.args[0],
+                                           exn.artiq_core_exception),
             raise exn

From 8f6653ef72bb6a3839efef8c681e1be9191af1da Mon Sep 17 00:00:00 2001
From: Robert Jordens <rj@m-labs.hk>
Date: Wed, 9 Mar 2016 17:58:42 +0100
Subject: [PATCH 23/38] test_spi: simplify test, add collision vs busy test

---
 artiq/test/coredevice/test_spi.py | 23 +++++++++++++++++++++--
 1 file changed, 21 insertions(+), 2 deletions(-)

diff --git a/artiq/test/coredevice/test_spi.py b/artiq/test/coredevice/test_spi.py
index ac1838d5d..436f720d9 100644
--- a/artiq/test/coredevice/test_spi.py
+++ b/artiq/test/coredevice/test_spi.py
@@ -2,6 +2,19 @@ from artiq.experiment import *
 from artiq.test.hardware_testbench import ExperimentCase
 
 
+class Collision(EnvExperiment):
+    def build(self):
+        self.setattr_device("core")
+        self.setattr_device("spi0")
+
+    @kernel
+    def run(self):
+        t = now_mu()
+        self.spi0.set_config_mu()
+        at_mu(t)
+        self.spi0.set_config_mu()
+
+
 class Busy(EnvExperiment):
     def build(self):
         self.setattr_device("core")
@@ -10,8 +23,9 @@ class Busy(EnvExperiment):
 
     @kernel
     def run(self):
+        t = now_mu()
         self.spi0.set_config_mu()
-        delay(-8*ns)
+        at_mu(t + self.spi0.ref_period_mu)
         self.spi0.set_config_mu()  # causes the error
         self.led.on()
         self.led.sync()            # registers the error
@@ -20,5 +34,10 @@ class Busy(EnvExperiment):
 
 
 class SPITest(ExperimentCase):
+    def test_collision(self):
+        with self.assertRaises(RTIOCollision):
+            self.execute(Collision)
+
     def test_busy(self):
-        self.execute(Busy)
+        with self.assertRaises(RTIOBusy):
+            self.execute(Busy)

From 10a09122eacce58c5c9add1af261c2d450f5bcc2 Mon Sep 17 00:00:00 2001
From: Robert Jordens <rj@m-labs.hk>
Date: Wed, 9 Mar 2016 18:23:27 +0100
Subject: [PATCH 24/38] test_spi: break_realtime

---
 artiq/test/coredevice/test_spi.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/artiq/test/coredevice/test_spi.py b/artiq/test/coredevice/test_spi.py
index 436f720d9..53b1a3ec0 100644
--- a/artiq/test/coredevice/test_spi.py
+++ b/artiq/test/coredevice/test_spi.py
@@ -9,6 +9,7 @@ class Collision(EnvExperiment):
 
     @kernel
     def run(self):
+        self.core.break_realtime()
         t = now_mu()
         self.spi0.set_config_mu()
         at_mu(t)
@@ -23,6 +24,7 @@ class Busy(EnvExperiment):
 
     @kernel
     def run(self):
+        self.core.break_realtime()
         t = now_mu()
         self.spi0.set_config_mu()
         at_mu(t + self.spi0.ref_period_mu)

From 107e5cfbd4fa0e7fe16ef49b4b0b4e7b46c91454 Mon Sep 17 00:00:00 2001
From: Robert Jordens <rj@m-labs.hk>
Date: Wed, 9 Mar 2016 19:04:06 +0100
Subject: [PATCH 25/38] gateware/rtio: factor _BlindTransfer

---
 artiq/gateware/rtio/core.py | 54 ++++++++++++++++++++-----------------
 1 file changed, 30 insertions(+), 24 deletions(-)

diff --git a/artiq/gateware/rtio/core.py b/artiq/gateware/rtio/core.py
index 0ceccab4a..2fb9ee02b 100644
--- a/artiq/gateware/rtio/core.py
+++ b/artiq/gateware/rtio/core.py
@@ -54,6 +54,26 @@ class _RTIOCounter(Module):
         self.comb += gt.i.eq(self.value_rtio), self.value_sys.eq(gt.o)
 
 
+class _BlindTransfer(Module):
+    def __init__(self):
+        self.i = Signal()
+        self.o = Signal()
+
+        ps = PulseSynchronizer("rio", "rsys")
+        ps_ack = PulseSynchronizer("rsys", "rio")
+        self.submodules += ps, ps_ack
+        blind = Signal()
+        self.sync.rio += [
+            If(self.i, blind.eq(1)),
+            If(ps_ack.o, blind.eq(0))
+        ]
+        self.comb += [
+            ps.i.eq(self.i & ~blind),
+            ps_ack.i.eq(ps.o),
+            self.o.eq(ps.o)
+        ]
+
+
 # CHOOSING A GUARD TIME
 #
 # The buffer must be transferred to the FIFO soon enough to account for:
@@ -221,24 +241,17 @@ class _OutputManager(Module):
         self.comb += fifo.re.eq(fifo.readable & (~dout_stb | dout_ack))
 
         # FIFO read through buffer
-        # TODO: report error on stb & busy
         self.comb += [
             dout_ack.eq(
                 dout.timestamp[fine_ts_width:] == counter.value_rtio),
             interface.stb.eq(dout_stb & dout_ack)
         ]
-        busy_sync = PulseSynchronizer("rio", "rsys")
-        busy_ack_sync = PulseSynchronizer("rsys", "rio")
-        self.submodules += busy_sync, busy_ack_sync
-        busy_blind = Signal()
-        self.comb += busy_sync.i.eq(interface.stb & interface.busy & ~busy_blind)
-        self.sync.rio += [
-            If(interface.stb & interface.busy, busy_blind.eq(1)),
-            If(busy_ack_sync.o, busy_blind.eq(0))
-        ]
+
+        busy_transfer = _BlindTransfer()
+        self.submodules += busy_transfer
         self.comb += [
-            busy_ack_sync.i.eq(busy_sync.o),
-            self.busy.eq(busy_sync.o)
+            busy_transfer.i.eq(interface.stb & interface.busy),
+            self.busy.eq(busy_transfer.o),
         ]
 
         if data_width:
@@ -263,7 +276,7 @@ class _InputManager(Module):
 
         self.readable = Signal()
         self.re = Signal()
-        
+
         self.overflow = Signal()  # pulsed
 
         # # #
@@ -296,18 +309,11 @@ class _InputManager(Module):
             fifo.re.eq(self.re)
         ]
 
-        overflow_sync = PulseSynchronizer("rio", "rsys")
-        overflow_ack_sync = PulseSynchronizer("rsys", "rio")
-        self.submodules += overflow_sync, overflow_ack_sync
-        overflow_blind = Signal()
-        self.comb += overflow_sync.i.eq(fifo.we & ~fifo.writable & ~overflow_blind)
-        self.sync.rio += [
-            If(fifo.we & ~fifo.writable, overflow_blind.eq(1)),
-            If(overflow_ack_sync.o, overflow_blind.eq(0))
-        ]
+        overflow_transfer = _BlindTransfer()
+        self.submodules += overflow_transfer
         self.comb += [
-            overflow_ack_sync.i.eq(overflow_sync.o),
-            self.overflow.eq(overflow_sync.o)
+            overflow_transfer.i.eq(fifo.we & ~fifo.writable),
+            self.overflow.eq(overflow_transfer.o),
         ]
 
 

From 9edaf16735552c5e5618f7c2e3633323f96d018b Mon Sep 17 00:00:00 2001
From: Robert Jordens <rj@m-labs.hk>
Date: Wed, 9 Mar 2016 22:11:32 +0100
Subject: [PATCH 26/38] exceptions: clarify RTIOBusy

---
 artiq/coredevice/exceptions.py | 9 ++++-----
 1 file changed, 4 insertions(+), 5 deletions(-)

diff --git a/artiq/coredevice/exceptions.py b/artiq/coredevice/exceptions.py
index 987f5a127..6317edfe6 100644
--- a/artiq/coredevice/exceptions.py
+++ b/artiq/coredevice/exceptions.py
@@ -99,15 +99,14 @@ class RTIOCollision(Exception):
     artiq_builtin = True
 
 class RTIOBusy(Exception):
-    """Raised when an event could not be executed because the given channel
-    was already busy executing a previous event. The two events were not
-    sufficiently spaced on the timeline.
+    """Raised when at least one output event could not be executed because
+    the given channel was already busy executing a previous event.
 
-    This exception is raised after the error condition appeared. More
+    This exception is raised late: after the error condition occurred. More
     specifically it is raised on submitting an event on the same channel after
     the execution of the faulty event was attempted.
 
-    The offending event is discarded and the RTIO core keeps operating.
+    The offending event was discarded.
     """
     artiq_builtin = True
 

From 2e39802a611701d6adb1ce3929bfa316e1704690 Mon Sep 17 00:00:00 2001
From: Sebastien Bourdeauducq <sb@m-labs.hk>
Date: Thu, 10 Mar 2016 09:44:05 +0800
Subject: [PATCH 27/38] rtio/wishbone: make replace configurable

---
 artiq/gateware/rtio/phy/wishbone.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/artiq/gateware/rtio/phy/wishbone.py b/artiq/gateware/rtio/phy/wishbone.py
index 2e9d6bc4a..bff70ef84 100644
--- a/artiq/gateware/rtio/phy/wishbone.py
+++ b/artiq/gateware/rtio/phy/wishbone.py
@@ -5,7 +5,7 @@ from artiq.gateware.rtio import rtlink
 
 
 class RT2WB(Module):
-    def __init__(self, address_width, wb=None):
+    def __init__(self, address_width, wb=None, rtio_enable_replace=False):
         if wb is None:
             wb = wishbone.Interface()
         self.wb = wb
@@ -14,7 +14,7 @@ class RT2WB(Module):
                 len(wb.dat_w),
                 address_width + 1,
                 suppress_nop=False,
-                enable_replace=False),
+                enable_replace=rtio_enable_replace),
             rtlink.IInterface(
                 len(wb.dat_r),
                 timestamped=False)

From 542a3753053f418e7adb3700efb257228e058712 Mon Sep 17 00:00:00 2001
From: Sebastien Bourdeauducq <sb@m-labs.hk>
Date: Thu, 10 Mar 2016 09:47:29 +0800
Subject: [PATCH 28/38] rtio: remove NOP suppression capability

Back when RTIO was driving TTLs, this functionality made it simpler to use by removing some irrelevant underflows.

The same technique is not applicable to DDS and SPI, so the user will have to deal with such underflows.

This patch makes the behavior of RTIO more consistent and the code simpler.
---
 artiq/gateware/rtio/core.py           | 29 +++++----------------------
 artiq/gateware/rtio/phy/ttl_simple.py |  3 +--
 artiq/gateware/rtio/phy/wishbone.py   |  1 -
 artiq/gateware/rtio/rtlink.py         |  6 ++----
 4 files changed, 8 insertions(+), 31 deletions(-)

diff --git a/artiq/gateware/rtio/core.py b/artiq/gateware/rtio/core.py
index d1095137b..ed3f5138a 100644
--- a/artiq/gateware/rtio/core.py
+++ b/artiq/gateware/rtio/core.py
@@ -95,7 +95,7 @@ class _OutputManager(Module):
             ev_layout.append(("address", address_width))
         ev_layout.append(("timestamp", counter.width + fine_ts_width))
         # ev must be valid 1 cycle before we to account for the latency in
-        # generating replace, sequence_error and nop
+        # generating replace, sequence_error and collision
         self.ev = Record(ev_layout)
 
         self.writable = Signal()
@@ -128,7 +128,6 @@ class _OutputManager(Module):
         sequence_error = Signal()
         collision = Signal()
         any_error = Signal()
-        nop = Signal()
         if interface.enable_replace:
             # Note: replace may be asserted at the same time as collision
             # when addresses are different. In that case, it is a collision.
@@ -151,25 +150,8 @@ class _OutputManager(Module):
         else:
             self.sync.rsys += collision.eq(
                 self.ev.timestamp[fine_ts_width:] == buf.timestamp[fine_ts_width:])
-        self.comb += any_error.eq(sequence_error | collision)
-        if interface.suppress_nop:
-            # disable NOP at reset: do not suppress a first write with all 0s
-            nop_en = Signal(reset=0)
-            addresses_equal = [getattr(self.ev, a) == getattr(buf, a)
-                               for a in ("data", "address")
-                               if hasattr(self.ev, a)]
-            if addresses_equal:
-                self.sync.rsys += nop.eq(
-                    nop_en & reduce(and_, addresses_equal))
-            else:
-                self.comb.eq(nop.eq(0))
-            self.sync.rsys += [
-                # buf now contains valid data. enable NOP.
-                If(self.we & ~any_error, nop_en.eq(1)),
-                # underflows cancel the write. allow it to be retried.
-                If(self.underflow, nop_en.eq(0))
-            ]
         self.comb += [
+            any_error.eq(sequence_error | collision),
             self.sequence_error.eq(self.we & sequence_error),
             self.collision.eq(self.we & collision)
         ]
@@ -190,7 +172,7 @@ class _OutputManager(Module):
                         fifo.we.eq(1)
                     )
                 ),
-                If(self.we & ~replace & ~nop & ~any_error,
+                If(self.we & ~replace & ~any_error,
                    fifo.we.eq(1)
                 )
             )
@@ -199,7 +181,7 @@ class _OutputManager(Module):
         # Must come after read to handle concurrent read+write properly
         self.sync.rsys += [
             buf_just_written.eq(0),
-            If(self.we & ~nop & ~any_error,
+            If(self.we & ~any_error,
                 buf_just_written.eq(1),
                 buf_pending.eq(1),
                 buf.eq(self.ev)
@@ -321,8 +303,7 @@ class Channel:
 class LogChannel:
     """A degenerate channel used to log messages into the analyzer."""
     def __init__(self):
-        self.interface = rtlink.Interface(
-            rtlink.OInterface(32, suppress_nop=False))
+        self.interface = rtlink.Interface(rtlink.OInterface(32))
         self.probes = []
         self.overrides = []
 
diff --git a/artiq/gateware/rtio/phy/ttl_simple.py b/artiq/gateware/rtio/phy/ttl_simple.py
index 1cc55ac48..a640cde5b 100644
--- a/artiq/gateware/rtio/phy/ttl_simple.py
+++ b/artiq/gateware/rtio/phy/ttl_simple.py
@@ -79,8 +79,7 @@ class Inout(Module):
 
 class ClockGen(Module):
     def __init__(self, pad, ftw_width=24):
-        self.rtlink = rtlink.Interface(
-            rtlink.OInterface(ftw_width, suppress_nop=False))
+        self.rtlink = rtlink.Interface(rtlink.OInterface(ftw_width))
 
         # # #
 
diff --git a/artiq/gateware/rtio/phy/wishbone.py b/artiq/gateware/rtio/phy/wishbone.py
index bff70ef84..d8fa752c2 100644
--- a/artiq/gateware/rtio/phy/wishbone.py
+++ b/artiq/gateware/rtio/phy/wishbone.py
@@ -13,7 +13,6 @@ class RT2WB(Module):
             rtlink.OInterface(
                 len(wb.dat_w),
                 address_width + 1,
-                suppress_nop=False,
                 enable_replace=rtio_enable_replace),
             rtlink.IInterface(
                 len(wb.dat_r),
diff --git a/artiq/gateware/rtio/rtlink.py b/artiq/gateware/rtio/rtlink.py
index b8bda7aac..d52e38ad3 100644
--- a/artiq/gateware/rtio/rtlink.py
+++ b/artiq/gateware/rtio/rtlink.py
@@ -3,8 +3,7 @@ from migen import *
 
 class OInterface:
     def __init__(self, data_width, address_width=0,
-                 fine_ts_width=0, suppress_nop=True,
-                 enable_replace=True):
+                 fine_ts_width=0, enable_replace=True):
         self.stb = Signal()
         self.busy = Signal()
 
@@ -15,7 +14,6 @@ class OInterface:
         if fine_ts_width:
             self.fine_ts = Signal(fine_ts_width)
 
-        self.suppress_nop = suppress_nop
         self.enable_replace = enable_replace
 
     @classmethod
@@ -23,7 +21,7 @@ class OInterface:
         return cls(get_data_width(other),
                    get_address_width(other),
                    get_fine_ts_width(other),
-                   other.suppress_nop)
+                   other.enable_replace)
 
 
 class IInterface:

From c5552f90091d90a1703405e5e054cde151a17399 Mon Sep 17 00:00:00 2001
From: Sebastien Bourdeauducq <sb@m-labs.hk>
Date: Thu, 10 Mar 2016 10:20:20 +0800
Subject: [PATCH 29/38] gui/moninj: make DDS widgets look less like buttons

---
 artiq/gui/moninj.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/artiq/gui/moninj.py b/artiq/gui/moninj.py
index 818c148f6..c9953d62a 100644
--- a/artiq/gui/moninj.py
+++ b/artiq/gui/moninj.py
@@ -125,7 +125,7 @@ class _DDSWidget(QtWidgets.QFrame):
 
         QtWidgets.QFrame.__init__(self)
 
-        self.setFrameShape(QtWidgets.QFrame.Panel)
+        self.setFrameShape(QtWidgets.QFrame.Box)
         self.setFrameShadow(QtWidgets.QFrame.Raised)
 
         grid = QtWidgets.QGridLayout()

From 7f501820de2d1f7578f6519d899ddbcc0b8c6089 Mon Sep 17 00:00:00 2001
From: Sebastien Bourdeauducq <sb@m-labs.hk>
Date: Thu, 10 Mar 2016 10:34:37 +0800
Subject: [PATCH 30/38] gui: delete log/applet docks instead of hiding them

---
 artiq/gui/applets.py | 12 ++++++++----
 artiq/gui/log.py     |  3 +++
 2 files changed, 11 insertions(+), 4 deletions(-)

diff --git a/artiq/gui/applets.py b/artiq/gui/applets.py
index b25ac1829..f9c93d9f5 100644
--- a/artiq/gui/applets.py
+++ b/artiq/gui/applets.py
@@ -85,7 +85,7 @@ class AppletIPCServer(AsyncioParentComm):
         await asyncio.wait([self.server_task])
 
 
-class AppletDock(QDockWidgetCloseDetect):
+class _AppletDock(QDockWidgetCloseDetect):
     def __init__(self, datasets_sub, uid, name, command):
         QDockWidgetCloseDetect.__init__(self, "Applet: " + name)
         self.setObjectName("applet" + str(uid))
@@ -137,7 +137,7 @@ class AppletDock(QDockWidgetCloseDetect):
     def fix_initial_size(self):
         self.embed_window.resize(self.embed_widget.size())
 
-    async def terminate(self):
+    async def terminate(self, delete_self=True):
         if self.starting_stopping:
             return
         self.starting_stopping = True
@@ -163,8 +163,12 @@ class AppletDock(QDockWidgetCloseDetect):
 
         self.starting_stopping = False
 
+        if delete_self:
+            self.setParent(None)
+            self.deleteLater()
+
     async def restart(self):
-        await self.terminate()
+        await self.terminate(False)
         await self.start()
 
 
@@ -235,7 +239,7 @@ class AppletsDock(QtWidgets.QDockWidget):
         self.table.cellChanged.connect(self.cell_changed)
 
     def create(self, uid, name, command):
-        dock = AppletDock(self.datasets_sub, uid, name, command)
+        dock = _AppletDock(self.datasets_sub, uid, name, command)
         self.main_window.addDockWidget(QtCore.Qt.RightDockWidgetArea, dock)
         dock.setFloating(True)
         asyncio.ensure_future(dock.start())
diff --git a/artiq/gui/log.py b/artiq/gui/log.py
index 3a33a9d3d..5b5dbbd39 100644
--- a/artiq/gui/log.py
+++ b/artiq/gui/log.py
@@ -303,6 +303,9 @@ class LogDockManager:
         return dock
 
     def on_dock_closed(self, name):
+        dock = self.docks[name]
+        dock.setParent(None)
+        dock.deleteLater()
         del self.docks[name]
         self.update_closable()
 

From 1739e0f2f872eeb109d2d8a83e55df5ffe11ff85 Mon Sep 17 00:00:00 2001
From: Sebastien Bourdeauducq <sb@m-labs.hk>
Date: Thu, 10 Mar 2016 10:45:16 +0800
Subject: [PATCH 31/38] coredevice: put cache into separate file/device

---
 artiq/coredevice/cache.py             | 45 +++++++++++++++++++++++++++
 artiq/coredevice/core.py              | 36 ---------------------
 artiq/test/coredevice/test_cache.py   |  7 +++--
 doc/manual/core_drivers_reference.rst |  6 ++++
 examples/master/device_db.pyon        |  6 ++++
 5 files changed, 61 insertions(+), 39 deletions(-)
 create mode 100644 artiq/coredevice/cache.py

diff --git a/artiq/coredevice/cache.py b/artiq/coredevice/cache.py
new file mode 100644
index 000000000..02d07d5d5
--- /dev/null
+++ b/artiq/coredevice/cache.py
@@ -0,0 +1,45 @@
+from artiq.language.core import *
+from artiq.language.types import *
+
+
+@syscall
+def cache_get(key: TStr) -> TList(TInt32):
+    raise NotImplementedError("syscall not simulated")
+
+@syscall
+def cache_put(key: TStr, value: TList(TInt32)) -> TNone:
+    raise NotImplementedError("syscall not simulated")
+
+
+class CoreCache:
+    """Core device cache access"""
+    def __init__(self, dmgr, core_device="core"):
+        self.core = dmgr.get(core_device)
+
+    @kernel
+    def get(self, key):
+        """Extract a value from the core device cache.
+        After a value is extracted, it cannot be replaced with another value using
+        :meth:`put` until all kernel functions finish executing; attempting
+        to replace it will result in a :class:`artiq.coredevice.exceptions.CacheError`.
+
+        If the cache does not contain any value associated with ``key``, an empty list
+        is returned.
+
+        The value is not copied, so mutating it will change what's stored in the cache.
+
+        :param str key: cache key
+        :return: a list of 32-bit integers
+        """
+        return cache_get(key)
+
+    @kernel
+    def put(self, key, value):
+        """Put a value into the core device cache. The value will persist until reboot.
+
+        To remove a value from the cache, call :meth:`put` with an empty list.
+
+        :param str key: cache key
+        :param list value: a list of 32-bit integers
+        """
+        cache_put(key, value)
diff --git a/artiq/coredevice/core.py b/artiq/coredevice/core.py
index 7aa17fa9b..a836f5093 100644
--- a/artiq/coredevice/core.py
+++ b/artiq/coredevice/core.py
@@ -41,14 +41,6 @@ class CompileError(Exception):
 def rtio_get_counter() -> TInt64:
     raise NotImplementedError("syscall not simulated")
 
-@syscall
-def cache_get(key: TStr) -> TList(TInt32):
-    raise NotImplementedError("syscall not simulated")
-
-@syscall
-def cache_put(key: TStr, value: TList(TInt32)) -> TNone:
-    raise NotImplementedError("syscall not simulated")
-
 
 class Core:
     """Core device driver.
@@ -127,31 +119,3 @@ class Core:
         min_now = rtio_get_counter() + 125000
         if now_mu() < min_now:
             at_mu(min_now)
-
-    @kernel
-    def get_cache(self, key):
-        """Extract a value from the core device cache.
-        After a value is extracted, it cannot be replaced with another value using
-        :meth:`put_cache` until all kernel functions finish executing; attempting
-        to replace it will result in a :class:`artiq.coredevice.exceptions.CacheError`.
-
-        If the cache does not contain any value associated with ``key``, an empty list
-        is returned.
-
-        The value is not copied, so mutating it will change what's stored in the cache.
-
-        :param str key: cache key
-        :return: a list of 32-bit integers
-        """
-        return cache_get(key)
-
-    @kernel
-    def put_cache(self, key, value):
-        """Put a value into the core device cache. The value will persist until reboot.
-
-        To remove a value from the cache, call :meth:`put_cache` with an empty list.
-
-        :param str key: cache key
-        :param list value: a list of 32-bit integers
-        """
-        cache_put(key, value)
diff --git a/artiq/test/coredevice/test_cache.py b/artiq/test/coredevice/test_cache.py
index a81950e00..717f68b34 100644
--- a/artiq/test/coredevice/test_cache.py
+++ b/artiq/test/coredevice/test_cache.py
@@ -5,21 +5,22 @@ from artiq.test.hardware_testbench import ExperimentCase
 class _Cache(EnvExperiment):
     def build(self):
         self.setattr_device("core")
-        self.print = lambda x: print(x)
+        self.setattr_device("core_cache")
 
     @kernel
     def get(self, key):
-        return self.core.get_cache(key)
+        return self.core_cache.get(key)
 
     @kernel
     def put(self, key, value):
-        self.core.put_cache(key, value)
+        self.core_cache.put(key, value)
 
     @kernel
     def get_put(self, key, value):
         self.get(key)
         self.put(key, value)
 
+
 class CacheTest(ExperimentCase):
     def test_get_empty(self):
         exp = self.create(_Cache)
diff --git a/doc/manual/core_drivers_reference.rst b/doc/manual/core_drivers_reference.rst
index e174930e4..9895b04f7 100644
--- a/doc/manual/core_drivers_reference.rst
+++ b/doc/manual/core_drivers_reference.rst
@@ -42,6 +42,12 @@ These drivers are for the core device and the peripherals closely integrated int
 .. automodule:: artiq.coredevice.i2c
     :members:
 
+:mod:`artiq.coredevice.cache` module
+-----------------------------------------
+
+.. automodule:: artiq.coredevice.cache
+    :members:
+
 :mod:`artiq.coredevice.exceptions` module
 -----------------------------------------
 
diff --git a/examples/master/device_db.pyon b/examples/master/device_db.pyon
index 05f9bc7ca..ebc75b1ff 100644
--- a/examples/master/device_db.pyon
+++ b/examples/master/device_db.pyon
@@ -1,5 +1,6 @@
 # This is an example device database that needs to be adapted to your setup.
 # The RTIO channel numbers here are for NIST CLOCK on KC705.
+# The list of devices here is not exhaustive.
 
 {
     "comm": {
@@ -14,6 +15,11 @@
         "class": "Core",
         "arguments": {"ref_period": 1e-9}
     },
+    "core_cache": {
+        "type": "local",
+        "module": "artiq.coredevice.cache",
+        "class": "CoreCache"
+    },
     "core_dds": {
         "type": "local",
         "module": "artiq.coredevice.dds",

From ea523c765b66a13791fa86ddc293033da5b6cfd8 Mon Sep 17 00:00:00 2001
From: Sebastien Bourdeauducq <sb@m-labs.hk>
Date: Thu, 10 Mar 2016 11:33:21 +0800
Subject: [PATCH 32/38] frontend/coreanalyzer: do not attempt to print obsolete
 decoded_dump attribute. Closes #324

---
 artiq/frontend/artiq_coreanalyzer.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/artiq/frontend/artiq_coreanalyzer.py b/artiq/frontend/artiq_coreanalyzer.py
index 491b276f1..b715eebc0 100755
--- a/artiq/frontend/artiq_coreanalyzer.py
+++ b/artiq/frontend/artiq_coreanalyzer.py
@@ -48,7 +48,6 @@ def main():
         decoded_dump = decode_dump(dump)
         if args.print_decoded:
             print("Log channel:", decoded_dump.log_channel)
-            print("DDS channel:", decoded_dump.dds_channel)
             print("DDS one-hot:", decoded_dump.dds_onehot_sel)
             for message in decoded_dump.messages:
                 print(message)

From de718fc8192bb2fa95072d86c6d20829ddc9c571 Mon Sep 17 00:00:00 2001
From: Sebastien Bourdeauducq <sb@m-labs.hk>
Date: Thu, 10 Mar 2016 12:14:48 +0800
Subject: [PATCH 33/38] rtio: fix different address collision detection

---
 artiq/gateware/rtio/core.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/artiq/gateware/rtio/core.py b/artiq/gateware/rtio/core.py
index 6b8d68c22..69babebac 100644
--- a/artiq/gateware/rtio/core.py
+++ b/artiq/gateware/rtio/core.py
@@ -158,8 +158,8 @@ class _OutputManager(Module):
         self.sync.rsys += sequence_error.eq(self.ev.timestamp[fine_ts_width:] <
                                             buf.timestamp[fine_ts_width:])
         if interface.enable_replace:
-            if hasattr(self.ev, "a"):
-                different_addresses = self.ev.a != buf.a
+            if address_width:
+                different_addresses = self.ev.address != buf.address
             else:
                 different_addresses = 0
             if fine_ts_width:

From f3a2b3a67ed682d34af3689e7062d56e840af451 Mon Sep 17 00:00:00 2001
From: Sebastien Bourdeauducq <sb@m-labs.hk>
Date: Thu, 10 Mar 2016 12:16:18 +0800
Subject: [PATCH 34/38] test/rtio: raise exception when pulse is not received

---
 artiq/test/coredevice/test_rtio.py | 14 ++++++++++++--
 1 file changed, 12 insertions(+), 2 deletions(-)

diff --git a/artiq/test/coredevice/test_rtio.py b/artiq/test/coredevice/test_rtio.py
index 8f5dcb48c..881c16ae9 100644
--- a/artiq/test/coredevice/test_rtio.py
+++ b/artiq/test/coredevice/test_rtio.py
@@ -12,6 +12,10 @@ from artiq.test.hardware_testbench import ExperimentCase
 artiq_low_latency = os.getenv("ARTIQ_LOW_LATENCY")
 
 
+class PulseNotReceived(Exception):
+    pass
+
+
 class RTT(EnvExperiment):
     def build(self):
         self.setattr_device("core")
@@ -29,7 +33,10 @@ class RTT(EnvExperiment):
                 delay(1*us)
                 t0 = now_mu()
                 self.ttl_inout.pulse(1*us)
-        self.set_dataset("rtt", mu_to_seconds(self.ttl_inout.timestamp_mu() - t0))
+        t1 = self.ttl_inout.timestamp_mu()
+        if t1 < 0:
+            raise PulseNotReceived()
+        self.set_dataset("rtt", mu_to_seconds(t1 - t0))
 
 
 class Loopback(EnvExperiment):
@@ -48,7 +55,10 @@ class Loopback(EnvExperiment):
                 delay(1*us)
                 t0 = now_mu()
                 self.loop_out.pulse(1*us)
-        self.set_dataset("rtt", mu_to_seconds(self.loop_in.timestamp_mu() - t0))
+        t1 = self.loop_in.timestamp_mu()
+        if t1 < 0:
+            raise PulseNotReceived()
+        self.set_dataset("rtt", mu_to_seconds(t1 - t0))
 
 
 class ClockGeneratorLoopback(EnvExperiment):

From d0cf58922983e1245a75453cf9e81566deb24c96 Mon Sep 17 00:00:00 2001
From: Sebastien Bourdeauducq <sb@m-labs.hk>
Date: Thu, 10 Mar 2016 13:24:00 +0800
Subject: [PATCH 35/38] test/rtio/Loopback: ensure loop_out is low before
 starting test

---
 artiq/test/coredevice/test_rtio.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/artiq/test/coredevice/test_rtio.py b/artiq/test/coredevice/test_rtio.py
index 881c16ae9..17caf7450 100644
--- a/artiq/test/coredevice/test_rtio.py
+++ b/artiq/test/coredevice/test_rtio.py
@@ -48,6 +48,7 @@ class Loopback(EnvExperiment):
     @kernel
     def run(self):
         self.loop_in.input()
+        self.loop_out.off()
         delay(1*us)
         with parallel:
             self.loop_in.gate_rising(2*us)

From 579168f06f1de676383aea1266c1647f6d2023c1 Mon Sep 17 00:00:00 2001
From: Sebastien Bourdeauducq <sb@m-labs.hk>
Date: Thu, 10 Mar 2016 15:09:54 +0800
Subject: [PATCH 36/38] monkey-patch asyncio.proactor_events to handle
 ConnectionAbortedError on Windows. Closes #247

---
 artiq/monkey_patches.py           | 157 ++++++++++++++++++++++++++++++
 artiq/protocols/asyncio_server.py |   2 +
 artiq/tools.py                    | 108 --------------------
 3 files changed, 159 insertions(+), 108 deletions(-)
 create mode 100644 artiq/monkey_patches.py

diff --git a/artiq/monkey_patches.py b/artiq/monkey_patches.py
new file mode 100644
index 000000000..43504daaf
--- /dev/null
+++ b/artiq/monkey_patches.py
@@ -0,0 +1,157 @@
+import sys
+
+
+__all__ = []
+
+
+if sys.version_info[:3] == (3, 5, 1):
+    import asyncio
+    import socket
+    import collections
+    import itertools
+    import os
+
+    # See https://github.com/m-labs/artiq/issues/253
+    @asyncio.coroutines.coroutine
+    def create_server(self, protocol_factory, host=None, port=None,
+                      *,
+                      family=socket.AF_UNSPEC,
+                      flags=socket.AI_PASSIVE,
+                      sock=None,
+                      backlog=100,
+                      ssl=None,
+                      reuse_address=None,
+                      reuse_port=None):
+        """Create a TCP server.
+        The host parameter can be a string, in that case the TCP server is bound
+        to host and port.
+        The host parameter can also be a sequence of strings and in that case
+        the TCP server is bound to all hosts of the sequence. If a host
+        appears multiple times (possibly indirectly e.g. when hostnames
+        resolve to the same IP address), the server is only bound once to that
+        host.
+        Return a Server object which can be used to stop the service.
+        This method is a coroutine.
+        """
+        if isinstance(ssl, bool):
+            raise TypeError('ssl argument must be an SSLContext or None')
+        if host is not None or port is not None:
+            if sock is not None:
+                raise ValueError(
+                    'host/port and sock can not be specified at the same time')
+
+            AF_INET6 = getattr(socket, 'AF_INET6', 0)
+            if reuse_address is None:
+                reuse_address = os.name == 'posix' and sys.platform != 'cygwin'
+            sockets = []
+            if host == '':
+                hosts = [None]
+            elif (isinstance(host, str) or
+                  not isinstance(host, collections.Iterable)):
+                hosts = [host]
+            else:
+                hosts = host
+
+            fs = [self._create_server_getaddrinfo(host, port, family=family,
+                                                  flags=flags)
+                  for host in hosts]
+            infos = yield from asyncio.tasks.gather(*fs, loop=self)
+            infos = set(itertools.chain.from_iterable(infos))
+
+            completed = False
+            try:
+                for res in infos:
+                    af, socktype, proto, canonname, sa = res
+                    try:
+                        sock = socket.socket(af, socktype, proto)
+                    except socket.error:
+                        # Assume it's a bad family/type/protocol combination.
+                        if self._debug:
+                            asyncio.log.logger.warning('create_server() failed to create '
+                                           'socket.socket(%r, %r, %r)',
+                                           af, socktype, proto, exc_info=True)
+                        continue
+                    sockets.append(sock)
+                    if reuse_address:
+                        sock.setsockopt(
+                            socket.SOL_SOCKET, socket.SO_REUSEADDR, True)
+                    if reuse_port:
+                        if not hasattr(socket, 'SO_REUSEPORT'):
+                            raise ValueError(
+                                'reuse_port not supported by socket module')
+                        else:
+                            sock.setsockopt(
+                                socket.SOL_SOCKET, socket.SO_REUSEPORT, True)
+                    # Disable IPv4/IPv6 dual stack support (enabled by
+                    # default on Linux) which makes a single socket
+                    # listen on both address families.
+                    if af == AF_INET6 and hasattr(socket, 'IPPROTO_IPV6'):
+                        sock.setsockopt(socket.IPPROTO_IPV6,
+                                        socket.IPV6_V6ONLY,
+                                        True)
+                    try:
+                        sock.bind(sa)
+                    except OSError as err:
+                        raise OSError(err.errno, 'error while attempting '
+                                      'to bind on address %r: %s'
+                                      % (sa, err.strerror.lower()))
+                completed = True
+            finally:
+                if not completed:
+                    for sock in sockets:
+                        sock.close()
+        else:
+            if sock is None:
+                raise ValueError('Neither host/port nor sock were specified')
+            sockets = [sock]
+
+        server = asyncio.base_events.Server(self, sockets)
+        for sock in sockets:
+            sock.listen(backlog)
+            sock.setblocking(False)
+            self._start_serving(protocol_factory, sock, ssl, server)
+        if self._debug:
+            asyncio.log.logger.info("%r is serving", server)
+        return server
+
+    asyncio.base_events.BaseEventLoop.create_server = create_server
+
+
+    # See https://github.com/m-labs/artiq/issues/247
+    def _loop_writing(self, f=None, data=None):
+        try:
+            assert f is self._write_fut
+            self._write_fut = None
+            self._pending_write = 0
+            if f:
+                f.result()
+            if data is None:
+                data = self._buffer
+                self._buffer = None
+            if not data:
+                if self._closing:
+                    self._loop.call_soon(self._call_connection_lost, None)
+                if self._eof_written:
+                    self._sock.shutdown(socket.SHUT_WR)
+                # Now that we've reduced the buffer size, tell the
+                # protocol to resume writing if it was paused.  Note that
+                # we do this last since the callback is called immediately
+                # and it may add more data to the buffer (even causing the
+                # protocol to be paused again).
+                self._maybe_resume_protocol()
+            else:
+                self._write_fut = self._loop._proactor.send(self._sock, data)
+                if not self._write_fut.done():
+                    assert self._pending_write == 0
+                    self._pending_write = len(data)
+                    self._write_fut.add_done_callback(self._loop_writing)
+                    self._maybe_pause_protocol()
+                else:
+                    self._write_fut.add_done_callback(self._loop_writing)
+        except (ConnectionResetError, ConnectionAbortedError) as exc:
+            self._force_close(exc)
+        except OSError as exc:
+            self._fatal_error(exc, 'Fatal write error on pipe transport')
+    
+    from asyncio import proactor_events
+    proactor_events._ProactorBaseWritePipeTransport._loop_writing = _loop_writing
diff --git a/artiq/protocols/asyncio_server.py b/artiq/protocols/asyncio_server.py
index e7e300eed..3316a8804 100644
--- a/artiq/protocols/asyncio_server.py
+++ b/artiq/protocols/asyncio_server.py
@@ -1,6 +1,8 @@
 import asyncio
 from copy import copy
 
+from artiq.monkey_patches import *
+
 
 class AsyncioServer:
     """Generic TCP server based on asyncio.
diff --git a/artiq/tools.py b/artiq/tools.py
index 8d2424cdc..ef666cabf 100644
--- a/artiq/tools.py
+++ b/artiq/tools.py
@@ -7,8 +7,6 @@ import asyncio
 import time
 import collections
 import os
-import socket
-import itertools
 import atexit
 import string
 
@@ -245,109 +243,3 @@ def get_windows_drives():
             drives.append(letter)
         bitmask >>= 1
     return drives
-
-if sys.version_info[:3] == (3, 5, 1):
-    # See https://github.com/m-labs/artiq/issues/253
-    @asyncio.coroutines.coroutine
-    def create_server(self, protocol_factory, host=None, port=None,
-                      *,
-                      family=socket.AF_UNSPEC,
-                      flags=socket.AI_PASSIVE,
-                      sock=None,
-                      backlog=100,
-                      ssl=None,
-                      reuse_address=None,
-                      reuse_port=None):
-        """Create a TCP server.
-        The host parameter can be a string, in that case the TCP server is bound
-        to host and port.
-        The host parameter can also be a sequence of strings and in that case
-        the TCP server is bound to all hosts of the sequence. If a host
-        appears multiple times (possibly indirectly e.g. when hostnames
-        resolve to the same IP address), the server is only bound once to that
-        host.
-        Return a Server object which can be used to stop the service.
-        This method is a coroutine.
-        """
-        if isinstance(ssl, bool):
-            raise TypeError('ssl argument must be an SSLContext or None')
-        if host is not None or port is not None:
-            if sock is not None:
-                raise ValueError(
-                    'host/port and sock can not be specified at the same time')
-
-            AF_INET6 = getattr(socket, 'AF_INET6', 0)
-            if reuse_address is None:
-                reuse_address = os.name == 'posix' and sys.platform != 'cygwin'
-            sockets = []
-            if host == '':
-                hosts = [None]
-            elif (isinstance(host, str) or
-                  not isinstance(host, collections.Iterable)):
-                hosts = [host]
-            else:
-                hosts = host
-
-            fs = [self._create_server_getaddrinfo(host, port, family=family,
-                                                  flags=flags)
-                  for host in hosts]
-            infos = yield from asyncio.tasks.gather(*fs, loop=self)
-            infos = set(itertools.chain.from_iterable(infos))
-
-            completed = False
-            try:
-                for res in infos:
-                    af, socktype, proto, canonname, sa = res
-                    try:
-                        sock = socket.socket(af, socktype, proto)
-                    except socket.error:
-                        # Assume it's a bad family/type/protocol combination.
-                        if self._debug:
-                            asyncio.log.logger.warning('create_server() failed to create '
-                                           'socket.socket(%r, %r, %r)',
-                                           af, socktype, proto, exc_info=True)
-                        continue
-                    sockets.append(sock)
-                    if reuse_address:
-                        sock.setsockopt(
-                            socket.SOL_SOCKET, socket.SO_REUSEADDR, True)
-                    if reuse_port:
-                        if not hasattr(socket, 'SO_REUSEPORT'):
-                            raise ValueError(
-                                'reuse_port not supported by socket module')
-                        else:
-                            sock.setsockopt(
-                                socket.SOL_SOCKET, socket.SO_REUSEPORT, True)
-                    # Disable IPv4/IPv6 dual stack support (enabled by
-                    # default on Linux) which makes a single socket
-                    # listen on both address families.
-                    if af == AF_INET6 and hasattr(socket, 'IPPROTO_IPV6'):
-                        sock.setsockopt(socket.IPPROTO_IPV6,
-                                        socket.IPV6_V6ONLY,
-                                        True)
-                    try:
-                        sock.bind(sa)
-                    except OSError as err:
-                        raise OSError(err.errno, 'error while attempting '
-                                      'to bind on address %r: %s'
-                                      % (sa, err.strerror.lower()))
-                completed = True
-            finally:
-                if not completed:
-                    for sock in sockets:
-                        sock.close()
-        else:
-            if sock is None:
-                raise ValueError('Neither host/port nor sock were specified')
-            sockets = [sock]
-
-        server = asyncio.base_events.Server(self, sockets)
-        for sock in sockets:
-            sock.listen(backlog)
-            sock.setblocking(False)
-            self._start_serving(protocol_factory, sock, ssl, server)
-        if self._debug:
-            asyncio.log.logger.info("%r is serving", server)
-        return server
-
-    asyncio.base_events.BaseEventLoop.create_server = create_server

From f68c24094f233d9bfcf9e5aeec340391a85e5231 Mon Sep 17 00:00:00 2001
From: Robert Jordens <rj@m-labs.hk>
Date: Thu, 10 Mar 2016 12:25:10 +0100
Subject: [PATCH 37/38] test_spi: drain errors and be more strict on where we
 expect errors

---
 artiq/test/coredevice/test_spi.py | 56 ++++++++++++++++++++++++++-----
 1 file changed, 47 insertions(+), 9 deletions(-)

diff --git a/artiq/test/coredevice/test_spi.py b/artiq/test/coredevice/test_spi.py
index 53b1a3ec0..3686f532b 100644
--- a/artiq/test/coredevice/test_spi.py
+++ b/artiq/test/coredevice/test_spi.py
@@ -2,6 +2,10 @@ from artiq.experiment import *
 from artiq.test.hardware_testbench import ExperimentCase
 
 
+class WrongError(Exception):
+    pass
+
+
 class Collision(EnvExperiment):
     def build(self):
         self.setattr_device("core")
@@ -11,7 +15,10 @@ class Collision(EnvExperiment):
     def run(self):
         self.core.break_realtime()
         t = now_mu()
-        self.spi0.set_config_mu()
+        try:
+            self.spi0.set_config_mu()
+        except RTIOBusy:
+            raise WrongError()
         at_mu(t)
         self.spi0.set_config_mu()
 
@@ -24,18 +31,49 @@ class Busy(EnvExperiment):
 
     @kernel
     def run(self):
-        self.core.break_realtime()
-        t = now_mu()
-        self.spi0.set_config_mu()
-        at_mu(t + self.spi0.ref_period_mu)
-        self.spi0.set_config_mu()  # causes the error
-        self.led.on()
-        self.led.sync()            # registers the error
-        self.core.break_realtime()
+        try:
+            self.core.break_realtime()
+            self.spi0.set_config_mu()
+            t = now_mu()
+            self.spi0.set_config_mu()
+            at_mu(t + self.spi0.ref_period_mu)
+            self.spi0.set_config_mu()  # causes the error
+            self.led.on()
+            self.led.sync()            # registers the error
+            self.core.break_realtime()
+        except RTIOBusy:
+            raise WrongError()         # we don't expect RTIOBusy so far
         self.spi0.set_config_mu()  # raises the error
 
 
+class DrainErrors(EnvExperiment):
+    def build(self):
+        self.setattr_device("core")
+        self.setattr_device("spi0")
+        self.setattr_device("led")
+
+    @kernel
+    def run(self):
+        while True:
+            try:
+                self.core.break_realtime()
+                delay(100*us)
+                self.spi0.set_config_mu()
+                self.led.on()
+                self.led.sync()
+                self.core.break_realtime()
+                self.spi0.set_config_mu()
+                self.led.off()
+                return
+            except:
+                pass
+
+
 class SPITest(ExperimentCase):
+    def tearDown(self):
+        self.execute(DrainErrors)
+        ExperimentCase.tearDown(self)
+
     def test_collision(self):
         with self.assertRaises(RTIOCollision):
             self.execute(Collision)

From a618a6d03ab76770eef9ab3e24a5331ed57a2d82 Mon Sep 17 00:00:00 2001
From: Robert Jordens <rj@m-labs.hk>
Date: Thu, 10 Mar 2016 12:34:06 +0100
Subject: [PATCH 38/38] hardware_testbench: better message when skipping

---
 artiq/test/hardware_testbench.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/artiq/test/hardware_testbench.py b/artiq/test/hardware_testbench.py
index 3cb0d3700..890f7e0e8 100644
--- a/artiq/test/hardware_testbench.py
+++ b/artiq/test/hardware_testbench.py
@@ -107,7 +107,8 @@ class ExperimentCase(unittest.TestCase):
             return exp
         except KeyError as e:
             # skip if ddb does not match requirements
-            raise unittest.SkipTest(*e.args)
+            raise unittest.SkipTest(
+                "device_db entry `{}` not found".format(*e.args))
 
     def execute(self, cls, **kwargs):
         expid = {