Merge branch 'master' into subprocess-termination

* master: (44 commits)
  Revert "conda: restrict binutils-or1k-linux dependency to linux."
  manual/installing: refresh
  use https for m-labs.hk
  gui/log: top cell alignment
  master/log: do not break lines
  conda: fix pyqt package name
  gui/applets: log warning if IPC address not in command
  applets: make sure pyqtgraph imports qt5
  applets: avoid argparse subparser mess
  examples/histogram: artiq -> artiq.experiment
  gui/applets: save dock UID in state
  setup.py: give up trying to check for PyQt
  setup.py: fix PyQt5 package name
  Use Qt5
  applets: fix error message text
  applets: handle dataset mutations
  applets: properly name docks to support state save/restore
  applets: clean shutdown
  protocols/pyon: set support
  protocols/pyon: remove FlatFileDB
  ...

README.rst

@@ -12,6 +12,6 @@ nanosecond timing resolution and sub-microsecond latency.
 Technologies employed include Python, Migen, MiSoC/mor1kx, LLVM and llvmlite.
 
 Website:
-http://m-labs.hk/artiq
+https://m-labs.hk/artiq
 
 Copyright (C) 2014-2016 M-Labs Limited. Licensed under GNU GPL version 3.

artiq/applets/big_number.py

@@ -1,6 +1,6 @@
 #!/usr/bin/env python3.5
 
-from quamash import QtWidgets
+from PyQt5 import QtWidgets
 
 from artiq.applets.simple import SimpleApplet
 
@@ -11,7 +11,7 @@ class NumberWidget(QtWidgets.QLCDNumber):
         self.setDigitCount(args.digit_count)
         self.dataset_name = args.dataset
 
-    def data_changed(self, data, mod):
+    def data_changed(self, data, mods):
         try:
             n = float(data[self.dataset_name][1])
         except (KeyError, ValueError, TypeError):

artiq/applets/plot_hist.py

@@ -0,0 +1,40 @@
+#!/usr/bin/env python3.5
+
+import numpy as np
+import PyQt5  # make sure pyqtgraph imports Qt5
+import pyqtgraph
+
+from artiq.applets.simple import SimpleApplet
+
+
+class HistogramPlot(pyqtgraph.PlotWidget):
+    def __init__(self, args):
+        pyqtgraph.PlotWidget.__init__(self)
+        self.args = args
+
+    def data_changed(self, data, mods):
+        try:
+            y = data[self.args.y][1]
+            if self.args.x is None:
+                x = None
+            else:
+                x = data[self.args.x][1]
+        except KeyError:
+            return
+        if x is None:
+            x = list(range(len(y)+1))
+
+        if len(y) and len(x) == len(y) + 1:
+            self.clear()
+            self.plot(x, y, stepMode=True, fillLevel=0,
+                      brush=(0, 0, 255, 150))
+
+
+def main():
+    applet = SimpleApplet(HistogramPlot)
+    applet.add_dataset("y", "Y values")
+    applet.add_dataset("x", "Bin boundaries", required=False)
+    applet.run()
+
+if __name__ == "__main__":
+    main()

artiq/applets/plot_xy.py

@@ -0,0 +1,60 @@
+#!/usr/bin/env python3.5
+
+import numpy as np
+import PyQt5  # make sure pyqtgraph imports Qt5
+import pyqtgraph
+
+from artiq.applets.simple import SimpleApplet
+
+
+class XYPlot(pyqtgraph.PlotWidget):
+    def __init__(self, args):
+        pyqtgraph.PlotWidget.__init__(self)
+        self.args = args
+
+    def data_changed(self, data, mods):
+        try:
+            y = data[self.args.y][1]
+        except KeyError:
+            return
+        x = data.get(self.args.x, (False, None))[1]
+        if x is None:
+            x = list(range(len(y)))
+        error = data.get(self.args.error, (False, None))[1]
+        fit = data.get(self.args.fit, (False, None))[1]
+
+        if not len(y) or len(y) != len(x):
+            return
+        if error is not None and hasattr(error, "__len__"):
+            if not len(error):
+                error = None
+            elif len(error) != len(y):
+                return
+        if fit is not None:
+            if not len(fit):
+                fit = None
+            elif len(fit) != len(y):
+                return
+
+        self.clear()
+        self.plot(x, y, pen=None, symbol="x")
+        if error is not None:
+            # See https://github.com/pyqtgraph/pyqtgraph/issues/211
+            if hasattr(error, "__len__") and not isinstance(error, np.ndarray):
+                error = np.array(error)
+            errbars = pg.ErrorBarItem(x=np.array(x), y=np.array(y), height=error)
+            self.addItem(errbars)
+        if fit is not None:
+            self.plot(x, fit)
+
+
+def main():
+    applet = SimpleApplet(XYPlot)
+    applet.add_dataset("y", "Y values")
+    applet.add_dataset("x", "X values", required=False)
+    applet.add_dataset("error", "Error bars for each X value", required=False)
+    applet.add_dataset("fit", "Fit values for each X value", required=False)
+    applet.run()
+
+if __name__ == "__main__":
+    main()

artiq/applets/plot_xy_hist.py

@@ -1,69 +1,136 @@
 #!/usr/bin/env python3.5
 
-from pyqtgraph.Qt import QtGui, QtCore
-import pyqtgraph as pg
-
 import numpy as np
+from PyQt5 import QtWidgets
+import pyqtgraph
 
-class XYHistPlot:
+from artiq.applets.simple import SimpleApplet
-    def __init__(self):
-        self.graphics_window = pg.GraphicsWindow(title="XY/Histogram")
-        self.graphics_window.resize(1000,600)
-        self.graphics_window.setWindowTitle("XY/Histogram")
 
-        self.xy_plot = self.graphics_window.addPlot()
+
+def _compute_ys(histogram_bins, histograms_counts):
+        bin_centers = np.empty(len(histogram_bins)-1)
+        for i in range(len(bin_centers)):
+            bin_centers[i] = (histogram_bins[i] + histogram_bins[i+1])/2
+
+        ys = np.empty(histograms_counts.shape[0])
+        for n, counts in enumerate(histograms_counts):
+            ys[n] = sum(bin_centers*counts)/sum(counts)
+        return ys
+
+
+# pyqtgraph.GraphicsWindow fails to behave like a regular Qt widget
+# and breaks embedding. Do not use as top widget.
+class XYHistPlot(QtWidgets.QSplitter):
+    def __init__(self, args):
+        QtWidgets.QSplitter.__init__(self)
+        self.resize(1000,600)
+        self.setWindowTitle("XY/Histogram")
+
+        self.xy_plot = pyqtgraph.PlotWidget()
+        self.insertWidget(0, self.xy_plot)
         self.xy_plot_data = None
         self.arrow = None
+        self.selected_index = None
 
-        self.hist_plot = self.graphics_window.addPlot()
+        self.hist_plot = pyqtgraph.PlotWidget()
+        self.insertWidget(1, self.hist_plot)
         self.hist_plot_data = None
 
-    def set_data(self, xs, histograms_bins, histograms_counts):
+        self.args = args
-        ys = np.empty_like(xs)
-        ys.fill(np.nan)
-        for n, (bins, counts) in enumerate(zip(histograms_bins,
-                                               histograms_counts)):
-            bin_centers = np.empty(len(bins)-1)
-            for i in range(len(bin_centers)):
-                bin_centers[i] = (bins[i] + bins[i+1])/2
-            ys[n] = sum(bin_centers*counts)/sum(bin_centers)
 
+    def _set_full_data(self, xs, histogram_bins, histograms_counts):
+        self.xy_plot.clear()
+        self.hist_plot.clear()
+        self.xy_plot_data = None
+        self.hist_plot_data = None
+        self.arrow = None
+        self.selected_index = None
+
+        self.histogram_bins = histogram_bins
+
+        ys = _compute_ys(self.histogram_bins, histograms_counts)
         self.xy_plot_data = self.xy_plot.plot(x=xs, y=ys,
                                               pen=None,
                                               symbol="x", symbolSize=20)
-        self.xy_plot_data.sigPointsClicked.connect(self.point_clicked)
+        self.xy_plot_data.sigPointsClicked.connect(self._point_clicked)
-        for point, bins, counts in zip(self.xy_plot_data.scatter.points(),
+        for index, (point, counts) in (
-                                       histograms_bins, histograms_counts):
+                enumerate(zip(self.xy_plot_data.scatter.points(),
-            point.histogram_bins = bins
+                              histograms_counts))):
+            point.histogram_index = index
             point.histogram_counts = counts
 
         self.hist_plot_data = self.hist_plot.plot(
-                            stepMode=True, fillLevel=0,
+            stepMode=True, fillLevel=0,
-                            brush=(0, 0, 255, 150))
+            brush=(0, 0, 255, 150))
 
-    def point_clicked(self, data_item, spot_items):
+    def _set_partial_data(self, xs, histograms_counts):
+        ys = _compute_ys(self.histogram_bins, histograms_counts)
+        self.xy_plot_data.setData(x=xs, y=ys,
+                                  pen=None,
+                                  symbol="x", symbolSize=20)
+        for index, (point, counts) in (
+                enumerate(zip(self.xy_plot_data.scatter.points(),
+                              histograms_counts))):
+            point.histogram_index = index
+            point.histogram_counts = counts
+
+    def _point_clicked(self, data_item, spot_items):
         spot_item = spot_items[0]
         position = spot_item.pos()
         if self.arrow is None:
-            self.arrow = pg.ArrowItem(angle=-120, tipAngle=30, baseAngle=20,
+            self.arrow = pyqtgraph.ArrowItem(
-                                      headLen=40, tailLen=40, tailWidth=8,
+                angle=-120, tipAngle=30, baseAngle=20, headLen=40,
-                                      pen=None, brush="y")
+                tailLen=40, tailWidth=8, pen=None, brush="y")
             self.arrow.setPos(position)
             # NB: temporary glitch if addItem is done before setPos
             self.xy_plot.addItem(self.arrow)
         else:
             self.arrow.setPos(position)
-        self.hist_plot_data.setData(x=spot_item.histogram_bins,
+        self.selected_index = spot_item.histogram_index
+        self.hist_plot_data.setData(x=self.histogram_bins,
                                     y=spot_item.histogram_counts)
 
+    def _can_use_partial(self, mods):
+        if self.hist_plot_data is None:
+            return False
+        for mod in mods:
+            if mod["action"] != "setitem":
+                return False
+            if mod["path"] == [self.args.xs, 1]:
+                if mod["key"] == self.selected_index:
+                    return False
+            elif mod["path"][:2] == [self.args.histograms_counts, 1]:
+                if len(mod["path"]) > 2:
+                    index = mod["path"][2]
+                else:
+                    index = mod["key"]
+                if index == self.selected_index:
+                    return False
+            else:
+                return False
+        return True
+
+    def data_changed(self, data, mods):
+        try:
+            xs = data[self.args.xs][1]
+            histogram_bins = data[self.args.histogram_bins][1]
+            histograms_counts = data[self.args.histograms_counts][1]
+        except KeyError:
+            return
+        if self._can_use_partial(mods):
+            self._set_partial_data(xs, histograms_counts)
+        else:
+            self._set_full_data(xs, histogram_bins, histograms_counts)
+        
 
 def main():
-    app = QtGui.QApplication([])
+    applet = SimpleApplet(XYHistPlot)
-    plot = XYHistPlot()
+    applet.add_dataset("xs", "1D array of point abscissas")
-    plot.set_data(np.array([1, 2, 3, 4, 1]),
+    applet.add_dataset("histogram_bins",
-                  np.array([[1, 2, 3], [1, 2, 3], [1, 2, 3], [40, 70, 100], [4, 7, 10, 20]]),
+                       "1D array of histogram bin boundaries")
-                  np.array([[1, 1], [2, 3], [10, 20], [3, 1], [100, 67, 102]]))
+    applet.add_dataset("histograms_counts",
-    app.exec_()
+                       "2D array of histogram counts, for each point")
+    applet.run()
 
-if __name__ == '__main__':
+if __name__ == "__main__":
     main()

artiq/applets/simple.py

@@ -1,65 +1,208 @@
+import logging
 import argparse
 import asyncio
 
-from quamash import QEventLoop, QtWidgets, QtCore
+from quamash import QEventLoop, QtWidgets, QtGui, QtCore
 
-from artiq.protocols.sync_struct import Subscriber
+from artiq.protocols.sync_struct import Subscriber, process_mod
+from artiq.protocols import pyon
+from artiq.protocols.pipe_ipc import AsyncioChildComm
+
+
+logger = logging.getLogger(__name__)
+
+
+class AppletIPCClient(AsyncioChildComm):
+    def set_close_cb(self, close_cb):
+        self.close_cb = close_cb
+
+    def write_pyon(self, obj):
+        self.write(pyon.encode(obj).encode() + b"\n")
+
+    async def read_pyon(self):
+        line = await self.readline()
+        return pyon.decode(line.decode())
+
+    async def embed(self, win_id):
+        # This function is only called when not subscribed to anything,
+        # so the only normal replies are embed_done and terminate.
+        self.write_pyon({"action": "embed",
+                         "win_id": win_id})
+        reply = await self.read_pyon()
+        if reply["action"] == "terminate":
+            self.close_cb()
+        elif reply["action"] != "embed_done":
+            logger.error("unexpected action reply to embed request: %s",
+                         action)
+            self.close_cb()
+
+    async def listen(self):
+        data = None
+        while True:
+            obj = await self.read_pyon()
+            try:
+                action = obj["action"]
+                if action == "terminate":
+                    self.close_cb()
+                    return
+                elif action == "mod":
+                    mod = obj["mod"]
+                    if mod["action"] == "init":
+                        data = self.init_cb(mod["struct"])
+                    else:
+                        process_mod(data, mod)
+                    self.mod_cb(mod)
+                else:
+                    raise ValueError("unknown action in parent message")
+            except:
+                logger.error("error processing parent message",
+                               exc_info=True)
+                self.close_cb()
+
+    def subscribe(self, datasets, init_cb, mod_cb):
+        self.write_pyon({"action": "subscribe",
+                         "datasets": datasets})
+        self.init_cb = init_cb
+        self.mod_cb = mod_cb
+        asyncio.ensure_future(self.listen())
 
 
 class SimpleApplet:
-    def __init__(self, main_widget_class, cmd_description=None):
+    def __init__(self, main_widget_class, cmd_description=None,
+                 default_update_delay=0.0):
         self.main_widget_class = main_widget_class
 
         self.argparser = argparse.ArgumentParser(description=cmd_description)
-        group = self.argparser.add_argument_group("data server")
+
+        self.argparser.add_argument("--update-delay", type=float,
+            default=default_update_delay,
+            help="time to wait after a mod (buffering other mods) "
+                  "before updating (default: %(default).2f)")
+
+        group = self.argparser.add_argument_group("standalone mode (default)")
         group.add_argument(
             "--server", default="::1",
-            help="hostname or IP to connect to")
+            help="hostname or IP of the master to connect to "
+                 "for dataset notifications "
+                 "(ignored in embedded mode)")
         group.add_argument(
             "--port", default=3250, type=int,
             help="TCP port to connect to")
+
+        self.argparser.add_argument("--embed", default=None,
+            help="embed into GUI", metavar="IPC_ADDRESS")
+
         self._arggroup_datasets = self.argparser.add_argument_group("datasets")
 
-    def add_dataset(self, name, help=None):
+        self.dataset_args = set()
-        if help is None:
+
-            self._arggroup_datasets.add_argument(name)
+    def add_dataset(self, name, help=None, required=True):
+        kwargs = dict()
+        if help is not None:
+            kwargs["help"] = help
+        if required:
+            self._arggroup_datasets.add_argument(name, **kwargs)
         else:
-            self._arggroup_datasets.add_argument(name, help=help)
+            self._arggroup_datasets.add_argument("--" + name, **kwargs)
+        self.dataset_args.add(name)
 
     def args_init(self):
         self.args = self.argparser.parse_args()
+        self.datasets = {getattr(self.args, arg.replace("-", "_"))
+                         for arg in self.dataset_args}
 
     def quamash_init(self):
         app = QtWidgets.QApplication([])
         self.loop = QEventLoop(app)
         asyncio.set_event_loop(self.loop)
 
+    def ipc_init(self):
+        if self.args.embed is not None:
+            self.ipc = AppletIPCClient(self.args.embed)
+            self.loop.run_until_complete(self.ipc.connect())
+
+    def ipc_close(self):
+        if self.args.embed is not None:
+            self.ipc.close()
+
     def create_main_widget(self):
         self.main_widget = self.main_widget_class(self.args)
+        # Qt window embedding is ridiculously buggy, and empirical testing
+        # has shown that the following procedure must be followed exactly:
+        # 1. applet creates widget
+        # 2. applet creates native window without showing it, and get its ID
+        # 3. applet sends the ID to host, host embeds the widget
+        # 4. applet shows the widget
+        # Doing embedding the other way around (using QWindow.setParent in the
+        # applet) breaks resizing.
+        if self.args.embed is not None:
+            self.ipc.set_close_cb(self.main_widget.close)
+            win_id = int(self.main_widget.winId())
+            self.loop.run_until_complete(self.ipc.embed(win_id))
         self.main_widget.show()
 
     def sub_init(self, data):
         self.data = data
         return data
 
-    def sub_mod(self, mod):
+    def filter_mod(self, mod):
-        self.main_widget.data_changed(self.data, mod)
+        if self.args.embed is not None:
+            # the parent already filters for us
+            return True
 
-    def create_subscriber(self):
+        if mod["action"] == "init":
-        self.subscriber = Subscriber("datasets",
+            return True
-                                     self.sub_init, self.sub_mod)
+        if mod["path"]:
-        self.loop.run_until_complete(self.subscriber.connect(
+            return mod["path"][0] in self.datasets
-            self.args.server, self.args.port))
+        elif mod["action"] in {"setitem", "delitem"}:
+            return mod["key"] in self.datasets
+        else:
+            return False
+
+    def flush_mod_buffer(self):
+        self.main_widget.data_changed(self.data, self.mod_buffer)
+        del self.mod_buffer
+
+    def sub_mod(self, mod):
+        if not self.filter_mod(mod):
+            return
+
+        if self.args.update_delay:
+            if hasattr(self, "mod_buffer"):
+                self.mod_buffer.append(mod)
+            else:
+                self.mod_buffer = [mod]
+                asyncio.get_event_loop().call_later(self.args.update_delay,
+                                                    self.flush_mod_buffer)
+        else:
+            self.main_widget.data_changed(self.data, [mod])
+
+    def subscribe(self):
+        if self.args.embed is None:
+            self.subscriber = Subscriber("datasets",
+                                         self.sub_init, self.sub_mod)
+            self.loop.run_until_complete(self.subscriber.connect(
+                self.args.server, self.args.port))
+        else:
+            self.ipc.subscribe(self.datasets, self.sub_init, self.sub_mod)
+
+    def unsubscribe(self):
+        if self.args.embed is None:
+            self.loop.run_until_complete(self.subscriber.close())
 
     def run(self):
         self.args_init()
         self.quamash_init()
         try:
-            self.create_main_widget()
+            self.ipc_init()
-            self.create_subscriber()
             try:
-                self.loop.run_forever()
+                self.create_main_widget()
+                self.subscribe()
+                try:
+                    self.loop.run_forever()
+                finally:
+                    self.unsubscribe()
             finally:
-                self.loop.run_until_complete(self.subscriber.close())
+                self.ipc_close()
         finally:
             self.loop.close()

artiq/frontend/artiq_gui.py

@@ -5,17 +5,19 @@ import asyncio
 import atexit
 import os
 
-# Quamash must be imported first so that pyqtgraph picks up the Qt binding
+import PyQt5
-# it has chosen.
 from quamash import QEventLoop, QtGui, QtCore
+assert QtGui is PyQt5.QtGui
+# pyqtgraph will pick up any already imported Qt binding.
 from pyqtgraph import dockarea
 
+
 from artiq import __artiq_dir__ as artiq_dir
 from artiq.tools import *
 from artiq.protocols.pc_rpc import AsyncioClient
 from artiq.gui.models import ModelSubscriber
 from artiq.gui import (state, experiments, shortcuts, explorer,
-                       moninj, datasets, schedule, log, console)
+                       moninj, datasets, applets, schedule, log, console)
 
 
 def get_argparser():
@@ -110,7 +112,10 @@ def main():
                                        rpc_clients["experiment_db"])
 
     d_datasets = datasets.DatasetsDock(win, dock_area, sub_clients["datasets"])
-    smgr.register(d_datasets)
+
+    d_applets = applets.AppletsDock(dock_area, sub_clients["datasets"])
+    atexit_register_coroutine(d_applets.stop)
+    smgr.register(d_applets)
 
     if os.name != "nt":
         d_ttl_dds = moninj.MonInj()
@@ -130,9 +135,11 @@ def main():
     if os.name != "nt":
         dock_area.addDock(d_ttl_dds.dds_dock, "top")
         dock_area.addDock(d_ttl_dds.ttl_dock, "above", d_ttl_dds.dds_dock)
-        dock_area.addDock(d_datasets, "above", d_ttl_dds.ttl_dock)
+        dock_area.addDock(d_applets, "above", d_ttl_dds.ttl_dock)
+        dock_area.addDock(d_datasets, "above", d_applets)
     else:
-        dock_area.addDock(d_datasets, "top")
+        dock_area.addDock(d_applets, "top")
+        dock_area.addDock(d_datasets, "above", d_applets)
     dock_area.addDock(d_shortcuts, "above", d_datasets)
     dock_area.addDock(d_explorer, "above", d_shortcuts)
     dock_area.addDock(d_console, "bottom")

artiq/gui/applets.py

@@ -0,0 +1,327 @@
+import logging
+import asyncio
+import sys
+import shlex
+from functools import partial
+
+from quamash import QtCore, QtGui, QtWidgets
+from pyqtgraph import dockarea
+
+from artiq.protocols.pipe_ipc import AsyncioParentComm
+from artiq.protocols import pyon
+
+
+logger = logging.getLogger(__name__)
+
+
+class AppletIPCServer(AsyncioParentComm):
+    def __init__(self, datasets_sub):
+        AsyncioParentComm.__init__(self)
+        self.datasets_sub = datasets_sub
+        self.datasets = set()
+
+    def write_pyon(self, obj):
+        self.write(pyon.encode(obj).encode() + b"\n")
+
+    async def read_pyon(self):
+        line = await self.readline()
+        return pyon.decode(line.decode())
+
+    def _synthesize_init(self, data):
+        struct = {k: v for k, v in data.items() if k in self.datasets}
+        return {"action": "init",
+                "struct": struct}
+
+    def _on_mod(self, mod):
+        if mod["action"] == "init":
+            mod = self._synthesize_init(mod["struct"])
+        else:
+            if mod["path"]:
+                if mod["path"][0] not in self.datasets:
+                    return
+            elif mod["action"] in {"setitem", "delitem"}:
+                if mod["key"] not in self.datasets:
+                    return
+        self.write_pyon({"action": "mod", "mod": mod})
+
+    async def serve(self, embed_cb):
+        self.datasets_sub.notify_cbs.append(self._on_mod)
+        try:
+            while True:
+                obj = await self.read_pyon()
+                try:
+                    action = obj["action"]
+                    if action == "embed":
+                        embed_cb(obj["win_id"])
+                        self.write_pyon({"action": "embed_done"})
+                    elif action == "subscribe":
+                        self.datasets = obj["datasets"]
+                        if self.datasets_sub.model is not None:
+                            mod = self._synthesize_init(
+                                self.datasets_sub.model.backing_store)
+                            self.write_pyon({"action": "mod", "mod": mod})
+                    else:
+                        raise ValueError("unknown action in applet message")
+                except:
+                    logger.warning("error processing applet message",
+                                   exc_info=True)
+                    self.write_pyon({"action": "error"})
+        except asyncio.CancelledError:
+            pass
+        except:
+            logger.error("error processing data from applet, "
+                         "server stopped", exc_info=True)
+        finally:
+            self.datasets_sub.notify_cbs.remove(self._on_mod)
+
+    def start(self, embed_cb):
+        self.server_task = asyncio.ensure_future(self.serve(embed_cb))
+
+    async def stop(self):
+        self.server_task.cancel()
+        await asyncio.wait([self.server_task])
+
+
+class AppletDock(dockarea.Dock):
+    def __init__(self, datasets_sub, uid, name, command):
+        dockarea.Dock.__init__(self, "applet" + str(uid),
+                               label="Applet: " + name,
+                               closable=True)
+        self.setMinimumSize(QtCore.QSize(500, 400))
+        self.datasets_sub = datasets_sub
+        self.applet_name = name
+        self.command = command
+
+    def rename(self, name):
+        self.applet_name = name
+        self.label.setText("Applet: " + name)
+
+    async def start(self):
+        self.ipc = AppletIPCServer(self.datasets_sub)
+        if "{ipc_address}" not in self.command:
+            logger.warning("IPC address missing from command for %s",
+                           self.applet_name)
+        command = self.command.format(python=sys.executable,
+                                      ipc_address=self.ipc.get_address())
+        logger.debug("starting command %s for %s", command, self.applet_name)
+        try:
+            await self.ipc.create_subprocess(*shlex.split(command))
+        except:
+            logger.warning("Applet %s failed to start", self.applet_name,
+                           exc_info=True)
+        self.ipc.start(self.embed)
+
+    def embed(self, win_id):
+        logger.debug("capturing window 0x%x for %s", win_id, self.applet_name)
+        embed_window = QtGui.QWindow.fromWinId(win_id)
+        embed_widget = QtWidgets.QWidget.createWindowContainer(embed_window)
+        self.addWidget(embed_widget)
+
+    async def terminate(self):
+        if hasattr(self, "ipc"):
+            await self.ipc.stop()
+            self.ipc.write_pyon({"action": "terminate"})
+            try:
+                await asyncio.wait_for(self.ipc.process.wait(), 2.0)
+            except:
+                logger.warning("Applet %s failed to exit, killing",
+                               self.applet_name)
+                try:
+                    self.ipc.process.kill()
+                except ProcessLookupError:
+                    pass
+                await self.ipc.process.wait()
+            del self.ipc
+
+    async def restart(self):
+        await self.terminate()
+        await self.start()
+
+
+_templates = [
+    ("Big number", "{python} -m artiq.applets.big_number "
+                   "--embed {ipc_address} NUMBER_DATASET"),
+    ("Histogram", "{python} -m artiq.applets.plot_hist "
+                  "--embed {ipc_address} COUNTS_DATASET "
+                  "--x BIN_BOUNDARIES_DATASET"),
+    ("XY", "{python} -m artiq.applets.plot_xy "
+           "--embed {ipc_address} Y_DATASET --x X_DATASET "
+           "--error ERROR_DATASET --fit FIT_DATASET"),
+    ("XY + Histogram", "{python} -m artiq.applets.plot_xy_hist "
+                       "--embed {ipc_address} X_DATASET "
+                       "HIST_BIN_BOUNDARIES_DATASET "
+                       "HISTS_COUNTS_DATASET"),
+]
+
+
+class AppletsDock(dockarea.Dock):
+    def __init__(self, dock_area, datasets_sub):
+        self.dock_area = dock_area
+        self.datasets_sub = datasets_sub
+        self.dock_to_checkbox = dict()
+        self.applet_uids = set()
+        self.workaround_pyqtgraph_bug = False
+
+        dockarea.Dock.__init__(self, "Applets")
+        self.setMinimumSize(QtCore.QSize(850, 450))
+
+        self.table = QtWidgets.QTableWidget(0, 3)
+        self.table.setHorizontalHeaderLabels(["Enable", "Name", "Command"])
+        self.table.setSelectionBehavior(QtGui.QAbstractItemView.SelectRows)
+        self.table.setSelectionMode(QtGui.QAbstractItemView.SingleSelection)
+        self.table.horizontalHeader().setStretchLastSection(True)
+        self.table.horizontalHeader().setResizeMode(
+            QtGui.QHeaderView.ResizeToContents)
+        self.table.verticalHeader().setResizeMode(
+            QtGui.QHeaderView.ResizeToContents)
+        self.table.verticalHeader().hide()
+        self.table.setTextElideMode(QtCore.Qt.ElideNone)
+        self.addWidget(self.table)
+
+        self.table.setContextMenuPolicy(QtCore.Qt.ActionsContextMenu)
+        new_action = QtGui.QAction("New applet", self.table)
+        new_action.triggered.connect(self.new)
+        self.table.addAction(new_action)
+        templates_menu = QtGui.QMenu()
+        for name, template in _templates:
+            action = QtGui.QAction(name, self.table)
+            action.triggered.connect(partial(self.new_template, template))
+            templates_menu.addAction(action)
+        restart_action = QtGui.QAction("New applet from template", self.table)
+        restart_action.setMenu(templates_menu)
+        self.table.addAction(restart_action)
+        restart_action = QtGui.QAction("Restart selected applet", self.table)
+        restart_action.setShortcut("CTRL+R")
+        restart_action.setShortcutContext(QtCore.Qt.WidgetShortcut)
+        restart_action.triggered.connect(self.restart)
+        self.table.addAction(restart_action)
+        delete_action = QtGui.QAction("Delete selected applet", self.table)
+        delete_action.setShortcut("DELETE")
+        delete_action.setShortcutContext(QtCore.Qt.WidgetShortcut)
+        delete_action.triggered.connect(self.delete)
+        self.table.addAction(delete_action)
+
+        self.table.cellChanged.connect(self.cell_changed)
+
+    def create(self, uid, name, command):
+        dock = AppletDock(self.datasets_sub, uid, name, command)
+        # If a dock is floated and then dock state is restored, pyqtgraph
+        # leaves a "phantom" window open.
+        if self.workaround_pyqtgraph_bug:
+            self.dock_area.addDock(dock)
+        else:
+            self.dock_area.floatDock(dock)
+        asyncio.ensure_future(dock.start())
+        dock.sigClosed.connect(partial(self.on_dock_closed, dock))
+        return dock
+
+    def cell_changed(self, row, column):
+        if column == 0:
+            item = self.table.item(row, column)
+            if item.checkState() == QtCore.Qt.Checked:
+                command = self.table.item(row, 2)
+                if command:
+                    command = command.text()
+                    name = self.table.item(row, 1)
+                    if name is None:
+                        name = ""
+                    else:
+                        name = name.text()
+                    dock = self.create(item.applet_uid, name, command)
+                    item.applet_dock = dock
+                    self.dock_to_checkbox[dock] = item
+            else:
+                dock = item.applet_dock
+                if dock is not None:
+                    # This calls self.on_dock_closed
+                    dock.close()
+        elif column == 1 or column == 2:
+            new_value = self.table.item(row, column).text()
+            dock = self.table.item(row, 0).applet_dock
+            if dock is not None:
+                if column == 1:
+                    dock.rename(new_value)
+                else:
+                    dock.command = new_value
+
+    def on_dock_closed(self, dock):
+        asyncio.ensure_future(dock.terminate())
+        checkbox_item = self.dock_to_checkbox[dock]
+        checkbox_item.applet_dock = None
+        del self.dock_to_checkbox[dock]
+        checkbox_item.setCheckState(QtCore.Qt.Unchecked)
+
+    def new(self, uid=None):
+        if uid is None:
+            uid = next(iter(set(range(len(self.applet_uids) + 1))
+                            - self.applet_uids))
+        self.applet_uids.add(uid)
+
+        row = self.table.rowCount()
+        self.table.insertRow(row)
+        checkbox = QtWidgets.QTableWidgetItem()
+        checkbox.setFlags(QtCore.Qt.ItemIsSelectable |
+                          QtCore.Qt.ItemIsUserCheckable |
+                          QtCore.Qt.ItemIsEnabled)
+        checkbox.setCheckState(QtCore.Qt.Unchecked)
+        checkbox.applet_uid = uid
+        checkbox.applet_dock = None
+        self.table.setItem(row, 0, checkbox)
+        self.table.setItem(row, 1, QtWidgets.QTableWidgetItem())
+        self.table.setItem(row, 2, QtWidgets.QTableWidgetItem())
+        return row
+
+    def new_template(self, template):
+        row = self.new()
+        self.table.item(row, 2).setText(template)
+
+    def restart(self):
+        selection = self.table.selectedRanges()
+        if selection:
+            row = selection[0].topRow()
+            dock = self.table.item(row, 0).applet_dock
+            if dock is not None:
+                asyncio.ensure_future(dock.restart())
+
+    def delete(self):
+        selection = self.table.selectedRanges()
+        if selection:
+            row = selection[0].topRow()
+            item = self.table.item(row, 0)
+            dock = item.applet_dock
+            if dock is not None:
+                # This calls self.on_dock_closed
+                dock.close()
+            self.applet_uids.remove(item.applet_uid)
+            self.table.removeRow(row)
+
+
+    async def stop(self):
+        for row in range(self.table.rowCount()):
+            dock = self.table.item(row, 0).applet_dock
+            if dock is not None:
+                await dock.terminate()
+
+    def save_state(self):
+        state = []
+        for row in range(self.table.rowCount()):
+            uid = self.table.item(row, 0).applet_uid
+            enabled = self.table.item(row, 0).checkState() == QtCore.Qt.Checked
+            name = self.table.item(row, 1).text()
+            command = self.table.item(row, 2).text()
+            state.append((uid, enabled, name, command))
+        return state
+
+    def restore_state(self, state):
+        self.workaround_pyqtgraph_bug = True
+        for uid, enabled, name, command in state:
+            row = self.new(uid)
+            item = QtWidgets.QTableWidgetItem()
+            item.setText(name)
+            self.table.setItem(row, 1, item)
+            item = QtWidgets.QTableWidgetItem()
+            item.setText(command)
+            self.table.setItem(row, 2, item)
+            if enabled:
+                self.table.item(row, 0).setCheckState(QtCore.Qt.Checked)
+        self.workaround_pyqtgraph_bug = False

artiq/gui/datasets.py

@@ -9,12 +9,6 @@ from pyqtgraph import LayoutWidget
 
 from artiq.tools import short_format
 from artiq.gui.models import DictSyncTreeSepModel
-from artiq.gui.displays import *
-
-try:
-    QSortFilterProxyModel = QtCore.QSortFilterProxyModel
-except AttributeError:
-    QSortFilterProxyModel = QtGui.QSortFilterProxyModel
 
 
 logger = logging.getLogger(__name__)
@@ -35,12 +29,6 @@ class Model(DictSyncTreeSepModel):
            raise ValueError
 
 
-def _get_display_type_name(display_cls):
-    for name, (_, cls) in display_types.items():
-        if cls is display_cls:
-            return name
-
-
 class DatasetsDock(dockarea.Dock):
     def __init__(self, dialog_parent, dock_area, datasets_sub):
         dockarea.Dock.__init__(self, "Datasets")
@@ -62,19 +50,6 @@ class DatasetsDock(dockarea.Dock):
 
         self.table_model = Model(dict())
         datasets_sub.add_setmodel_callback(self.set_model)
-        datasets_sub.notify_cbs.append(self.on_mod)
-
-        add_display_box = QtGui.QGroupBox("Add display")
-        grid.addWidget(add_display_box, 1, 1)
-        display_grid = QtGui.QGridLayout()
-        add_display_box.setLayout(display_grid)
-
-        for n, name in enumerate(display_types.keys()):
-            btn = QtGui.QPushButton(name)
-            display_grid.addWidget(btn, n, 0)
-            btn.clicked.connect(partial(self.create_dialog, name))
-
-        self.displays = dict()
 
     def _search_datasets(self):
         if hasattr(self, "table_model_filter"):
@@ -83,74 +58,6 @@ class DatasetsDock(dockarea.Dock):
 
     def set_model(self, model):
         self.table_model = model
-        self.table_model_filter = QSortFilterProxyModel()
+        self.table_model_filter = QtCore.QSortFilterProxyModel()
         self.table_model_filter.setSourceModel(self.table_model)
         self.table.setModel(self.table_model_filter)
-
-    def update_display_data(self, dsp):
-        filtered_data = {k: self.table_model.backing_store[k][1]
-                         for k in dsp.data_sources()
-                         if k in self.table_model.backing_store}
-        dsp.update_data(filtered_data)
-
-    def on_mod(self, mod):
-        if mod["action"] == "init":
-            for display in self.displays.values():
-                display.update_data(self.table_model.backing_store)
-            return
-
-        if mod["path"]:
-            source = mod["path"][0]
-        elif mod["action"] == "setitem":
-            source = mod["key"]
-        else:
-            return
-
-        for display in self.displays.values():
-            if source in display.data_sources():
-                self.update_display_data(display)
-
-    def create_dialog(self, ty):
-        dlg_class = display_types[ty][0]
-        dlg = dlg_class(self.dialog_parent, None, dict(),
-            sorted(self.table_model.backing_store.keys()),
-            partial(self.create_display, ty, None))
-        dlg.open()
-
-    def create_display(self, ty, prev_name, name, settings):
-        if prev_name is not None and prev_name in self.displays:
-            raise NotImplementedError
-        dsp_class = display_types[ty][1]
-        dsp = dsp_class(name, settings)
-        self.displays[name] = dsp
-        self.update_display_data(dsp)
-
-        def on_close():
-            del self.displays[name]
-        dsp.sigClosed.connect(on_close)
-        self.dock_area.floatDock(dsp)
-        return dsp
-
-    def save_state(self):
-        r = dict()
-        for name, display in self.displays.items():
-            r[name] = {
-                "ty": _get_display_type_name(type(display)),
-                "settings": display.settings,
-                "state": display.save_state()
-            }
-        return r
-
-    def restore_state(self, state):
-        for name, desc in state.items():
-            try:
-                dsp = self.create_display(desc["ty"], None, name,
-                                          desc["settings"])
-            except:
-                logger.warning("Failed to create display '%s'", name,
-                               exc_info=True)
-            try:
-                dsp.restore_state(desc["state"])
-            except:
-                logger.warning("Failed to restore display state of '%s'",
-                               name, exc_info=True)

artiq/gui/displays.py

@@ -1,217 +0,0 @@
-from collections import OrderedDict
-import numpy as np
-
-from quamash import QtGui
-import pyqtgraph as pg
-from pyqtgraph import dockarea
-
-
-class _BaseSettings(QtGui.QDialog):
-    def __init__(self, parent, window_title, prev_name, create_cb):
-        QtGui.QDialog.__init__(self, parent=parent)
-        self.setWindowTitle(window_title)
-
-        self.grid = QtGui.QGridLayout()
-        self.setLayout(self.grid)
-
-        self.grid.addWidget(QtGui.QLabel("Name:"), 0, 0)
-        self.name = QtGui.QLineEdit()
-        self.grid.addWidget(self.name, 0, 1)
-        if prev_name is not None:
-            self.name.setText(prev_name)
-
-        def on_accept():
-            create_cb(self.name.text(), self.get_input())
-        self.accepted.connect(on_accept)
-
-    def add_buttons(self):
-        buttons = QtGui.QDialogButtonBox(
-            QtGui.QDialogButtonBox.Ok | QtGui.QDialogButtonBox.Cancel)
-        self.grid.addWidget(buttons, self.grid.rowCount(), 0, 1, 2)
-        buttons.accepted.connect(self.accept)
-        buttons.rejected.connect(self.reject)
-
-    def accept(self):
-        if self.name.text() and self.validate_input():
-            QtGui.QDialog.accept(self)
-
-    def validate_input(self):
-        raise NotImplementedError
-
-    def get_input(self):
-        raise NotImplementedError
-
-
-class _SimpleSettings(_BaseSettings):
-    def __init__(self, parent, prev_name, prev_settings,
-                 result_list, create_cb):
-        _BaseSettings.__init__(self, parent, self._window_title,
-                               prev_name, create_cb)
-
-        self.result_widgets = dict()
-        for row, (has_none, key) in enumerate(self._result_keys):
-            self.grid.addWidget(QtGui.QLabel(key.capitalize() + ":"))
-            w = QtGui.QComboBox()
-            self.grid.addWidget(w, row + 1, 1)
-            if has_none:
-                w.addItem("<None>")
-            w.addItems(result_list)
-            w.setEditable(True)
-            if key in prev_settings:
-                w.setEditText(prev_settings[key])
-            self.result_widgets[key] = w
-        self.add_buttons()
-
-    def validate_input(self):
-        return all(w.currentText() for w in self.result_widgets.values())
-
-    def get_input(self):
-        return {k: v.currentText() for k, v in self.result_widgets.items()}
-
-
-class NumberDisplaySettings(_SimpleSettings):
-    _window_title = "Number display"
-    _result_keys = [(False, "result")]
-
-
-class NumberDisplay(dockarea.Dock):
-    def __init__(self, name, settings):
-        dockarea.Dock.__init__(self, "Display: " + name, closable=True)
-        self.settings = settings
-        self.number = QtGui.QLCDNumber()
-        self.number.setDigitCount(10)
-        self.addWidget(self.number)
-
-    def data_sources(self):
-        return {self.settings["result"]}
-
-    def update_data(self, data):
-        result = self.settings["result"]
-        try:
-            n = float(data[result])
-        except:
-            n = "---"
-        self.number.display(n)
-
-    def save_state(self):
-        return None
-
-    def restore_state(self, state):
-        pass
-
-
-class XYDisplaySettings(_SimpleSettings):
-    _window_title = "XY plot"
-    _result_keys = [(False, "y"), (True, "x"), (True, "error"), (True, "fit")]
-
-
-class XYDisplay(dockarea.Dock):
-    def __init__(self, name, settings):
-        dockarea.Dock.__init__(self, "XY: " + name, closable=True)
-        self.settings = settings
-        self.plot = pg.PlotWidget()
-        self.addWidget(self.plot)
-
-    def data_sources(self):
-        s = {self.settings["y"]}
-        for k in "x", "error", "fit":
-            if self.settings[k] != "<None>":
-                s.add(self.settings[k])
-        return s
-
-    def update_data(self, data):
-        result_y = self.settings["y"]
-        result_x = self.settings["x"]
-        result_error = self.settings["error"]
-        result_fit = self.settings["fit"]
-
-        try:
-            y = data[result_y]
-        except KeyError:
-            return
-        x = data.get(result_x, None)
-        if x is None:
-            x = list(range(len(y)))
-        error = data.get(result_error, None)
-        fit = data.get(result_fit, None)
-
-        if not len(y) or len(y) != len(x):
-            return
-        if error is not None and hasattr(error, "__len__"):
-            if not len(error):
-                error = None
-            elif len(error) != len(y):
-                return
-        if fit is not None:
-            if not len(fit):
-                fit = None
-            elif len(fit) != len(y):
-                return
-
-        self.plot.clear()
-        self.plot.plot(x, y, pen=None, symbol="x")
-        if error is not None:
-            # See https://github.com/pyqtgraph/pyqtgraph/issues/211
-            if hasattr(error, "__len__") and not isinstance(error, np.ndarray):
-                error = np.array(error)
-            errbars = pg.ErrorBarItem(x=np.array(x), y=np.array(y), height=error)
-            self.plot.addItem(errbars)
-        if fit is not None:
-            self.plot.plot(x, fit)
-
-    def save_state(self):
-        return self.plot.saveState()
-
-    def restore_state(self, state):
-        self.plot.restoreState(state)
-
-
-class HistogramDisplaySettings(_SimpleSettings):
-    _window_title = "Histogram"
-    _result_keys = [(False, "y"), (True, "x")]
-
-
-class HistogramDisplay(dockarea.Dock):
-    def __init__(self, name, settings):
-        dockarea.Dock.__init__(self, "Histogram: " + name, closable=True)
-        self.settings = settings
-        self.plot = pg.PlotWidget()
-        self.addWidget(self.plot)
-
-    def data_sources(self):
-        s = {self.settings["y"]}
-        if self.settings["x"] != "<None>":
-            s.add(self.settings["x"])
-        return s
-
-    def update_data(self, data):
-        result_y = self.settings["y"]
-        result_x = self.settings["x"]
-        try:
-            y = data[result_y]
-            if result_x == "<None>":
-                x = None
-            else:
-                x = data[result_x]
-        except KeyError:
-            return
-        if x is None:
-            x = list(range(len(y)+1))
-
-        if len(y) and len(x) == len(y) + 1:
-            self.plot.clear()
-            self.plot.plot(x, y, stepMode=True, fillLevel=0,
-                           brush=(0, 0, 255, 150))
-
-    def save_state(self):
-        return self.plot.saveState()
-
-    def restore_state(self, state):
-        self.plot.restoreState(state)
-
-
-display_types = OrderedDict([
-    ("Number", (NumberDisplaySettings, NumberDisplay)),
-    ("XY", (XYDisplaySettings, XYDisplay)),
-    ("Histogram", (HistogramDisplaySettings, HistogramDisplay))
-])

artiq/gui/entries.py

@@ -10,6 +10,100 @@ from artiq.gui.tools import disable_scroll_wheel
 logger = logging.getLogger(__name__)
 
 
+class _StringEntry(QtGui.QLineEdit):
+    def __init__(self, argument):
+        QtGui.QLineEdit.__init__(self)
+        self.setText(argument["state"])
+        def update(text):
+            argument["state"] = text
+        self.textEdited.connect(update)
+
+    @staticmethod
+    def state_to_value(state):
+        return state
+
+    @staticmethod
+    def default_state(procdesc):
+        return procdesc.get("default", "")
+
+
+class _BooleanEntry(QtGui.QCheckBox):
+    def __init__(self, argument):
+        QtGui.QCheckBox.__init__(self)
+        self.setChecked(argument["state"])
+        def update(checked):
+            argument["state"] = bool(checked)
+        self.stateChanged.connect(update)
+
+    @staticmethod
+    def state_to_value(state):
+        return state
+
+    @staticmethod
+    def default_state(procdesc):
+        return procdesc.get("default", False)
+
+
+class _EnumerationEntry(QtGui.QComboBox):
+    def __init__(self, argument):
+        QtGui.QComboBox.__init__(self)
+        disable_scroll_wheel(self)
+        choices = argument["desc"]["choices"]
+        self.addItems(choices)
+        idx = choices.index(argument["state"])
+        self.setCurrentIndex(idx)
+        def update(index):
+            argument["state"] = choices[index]
+        self.currentIndexChanged.connect(update)
+
+    @staticmethod
+    def state_to_value(state):
+        return state
+
+    @staticmethod
+    def default_state(procdesc):
+        if "default" in procdesc:
+            return procdesc["default"]
+        else:
+            return procdesc["choices"][0]
+
+
+class _NumberEntry(QtGui.QDoubleSpinBox):
+    def __init__(self, argument):
+        QtGui.QDoubleSpinBox.__init__(self)
+        disable_scroll_wheel(self)
+        procdesc = argument["desc"]
+        scale = procdesc["scale"]
+        self.setDecimals(procdesc["ndecimals"])
+        self.setSingleStep(procdesc["step"]/scale)
+        if procdesc["min"] is not None:
+            self.setMinimum(procdesc["min"]/scale)
+        else:
+            self.setMinimum(float("-inf"))
+        if procdesc["max"] is not None:
+            self.setMaximum(procdesc["max"]/scale)
+        else:
+            self.setMaximum(float("inf"))
+        if procdesc["unit"]:
+            self.setSuffix(" " + procdesc["unit"])
+
+        self.setValue(argument["state"]/scale)
+        def update(value):
+            argument["state"] = value*scale
+        self.valueChanged.connect(update)
+
+    @staticmethod
+    def state_to_value(state):
+        return state
+
+    @staticmethod
+    def default_state(procdesc):
+        if "default" in procdesc:
+            return procdesc["default"]
+        else:
+            return 0.0
+
+
 class _NoScan(LayoutWidget):
     def __init__(self, procdesc, state):
         LayoutWidget.__init__(self)
@@ -38,7 +132,7 @@ class _NoScan(LayoutWidget):
         self.value.valueChanged.connect(update)
 
 
-class _Range(LayoutWidget):
+class _RangeScan(LayoutWidget):
     def __init__(self, procdesc, state):
         LayoutWidget.__init__(self)
 
@@ -90,7 +184,8 @@ class _Range(LayoutWidget):
         self.max.valueChanged.connect(update_max)
         self.npoints.valueChanged.connect(update_npoints)
 
-class _Explicit(LayoutWidget):
+
+class _ExplicitScan(LayoutWidget):
     def __init__(self, state):
         LayoutWidget.__init__(self)
 
@@ -109,7 +204,7 @@ class _Explicit(LayoutWidget):
         self.value.textEdited.connect(update)
 
 
-class ScanController(LayoutWidget):
+class _ScanEntry(LayoutWidget):
     def __init__(self, argument):
         LayoutWidget.__init__(self)
         self.argument = argument
@@ -121,9 +216,9 @@ class ScanController(LayoutWidget):
         state = argument["state"]
         self.widgets = OrderedDict()
         self.widgets["NoScan"] = _NoScan(procdesc, state["NoScan"])
-        self.widgets["LinearScan"] = _Range(procdesc, state["LinearScan"])
+        self.widgets["LinearScan"] = _RangeScan(procdesc, state["LinearScan"])
-        self.widgets["RandomScan"] = _Range(procdesc, state["RandomScan"])
+        self.widgets["RandomScan"] = _RangeScan(procdesc, state["RandomScan"])
-        self.widgets["ExplicitScan"] = _Explicit(state["ExplicitScan"])
+        self.widgets["ExplicitScan"] = _ExplicitScan(state["ExplicitScan"])
         for widget in self.widgets.values():
             self.stack.addWidget(widget)
 
@@ -181,3 +276,13 @@ class ScanController(LayoutWidget):
                 self.stack.setCurrentWidget(self.widgets[ty])
                 self.argument["state"]["selected"] = ty
                 break
+
+
+argty_to_entry = {
+    "PYONValue": _StringEntry,
+    "BooleanValue": _BooleanEntry,
+    "EnumerationValue": _EnumerationEntry,
+    "NumberValue": _NumberEntry,
+    "StringValue": _StringEntry,
+    "Scannable": _ScanEntry
+}

artiq/gui/experiments.py

@@ -7,117 +7,13 @@ from quamash import QtGui, QtCore
 
 from pyqtgraph import dockarea, LayoutWidget
 
-from artiq.gui.tools import log_level_to_name, disable_scroll_wheel
+from artiq.gui.tools import log_level_to_name
-from artiq.gui.scan import ScanController
+from artiq.gui.entries import argty_to_entry
 
 
 logger = logging.getLogger(__name__)
 
 
-class _StringEntry(QtGui.QLineEdit):
-    def __init__(self, argument):
-        QtGui.QLineEdit.__init__(self)
-        self.setText(argument["state"])
-        def update(text):
-            argument["state"] = text
-        self.textEdited.connect(update)
-
-    @staticmethod
-    def state_to_value(state):
-        return state
-
-    @staticmethod
-    def default_state(procdesc):
-        return procdesc.get("default", "")
-
-
-class _BooleanEntry(QtGui.QCheckBox):
-    def __init__(self, argument):
-        QtGui.QCheckBox.__init__(self)
-        self.setChecked(argument["state"])
-        def update(checked):
-            argument["state"] = bool(checked)
-        self.stateChanged.connect(update)
-
-    @staticmethod
-    def state_to_value(state):
-        return state
-
-    @staticmethod
-    def default_state(procdesc):
-        return procdesc.get("default", False)
-
-
-class _EnumerationEntry(QtGui.QComboBox):
-    def __init__(self, argument):
-        QtGui.QComboBox.__init__(self)
-        disable_scroll_wheel(self)
-        choices = argument["desc"]["choices"]
-        self.addItems(choices)
-        idx = choices.index(argument["state"])
-        self.setCurrentIndex(idx)
-        def update(index):
-            argument["state"] = choices[index]
-        self.currentIndexChanged.connect(update)
-
-    @staticmethod
-    def state_to_value(state):
-        return state
-
-    @staticmethod
-    def default_state(procdesc):
-        if "default" in procdesc:
-            return procdesc["default"]
-        else:
-            return procdesc["choices"][0]
-
-
-class _NumberEntry(QtGui.QDoubleSpinBox):
-    def __init__(self, argument):
-        QtGui.QDoubleSpinBox.__init__(self)
-        disable_scroll_wheel(self)
-        procdesc = argument["desc"]
-        scale = procdesc["scale"]
-        self.setDecimals(procdesc["ndecimals"])
-        self.setSingleStep(procdesc["step"]/scale)
-        if procdesc["min"] is not None:
-            self.setMinimum(procdesc["min"]/scale)
-        else:
-            self.setMinimum(float("-inf"))
-        if procdesc["max"] is not None:
-            self.setMaximum(procdesc["max"]/scale)
-        else:
-            self.setMaximum(float("inf"))
-        if procdesc["unit"]:
-            self.setSuffix(" " + procdesc["unit"])
-
-        self.setValue(argument["state"]/scale)
-        def update(value):
-            argument["state"] = value*scale
-        self.valueChanged.connect(update)
-
-    @staticmethod
-    def state_to_value(state):
-        return state
-
-    @staticmethod
-    def default_state(procdesc):
-        if "default" in procdesc:
-            return procdesc["default"]
-        else:
-            return 0.0
-
-
-_argty_to_entry = {
-    "PYONValue": _StringEntry,
-    "BooleanValue": _BooleanEntry,
-    "EnumerationValue": _EnumerationEntry,
-    "NumberValue": _NumberEntry,
-    "StringValue": _StringEntry,
-    "Scannable": ScanController
-}
-
-
 # Experiment URLs come in two forms:
 # 1. repo:<experiment name>
 #    (file name and class name to be retrieved from explist)
@@ -153,7 +49,7 @@ class _ArgumentEditor(QtGui.QTreeWidget):
             self.addTopLevelItem(QtGui.QTreeWidgetItem(["No arguments"]))
 
         for name, argument in arguments.items():
-            entry = _argty_to_entry[argument["desc"]["ty"]](argument)
+            entry = argty_to_entry[argument["desc"]["ty"]](argument)
             widget_item = QtGui.QTreeWidgetItem([name])
             self._arg_to_entry_widgetitem[name] = entry, widget_item
 
@@ -211,14 +107,14 @@ class _ArgumentEditor(QtGui.QTreeWidget):
         argument = self.manager.get_submission_arguments(self.expurl)[name]
 
         procdesc = arginfo[name][0]
-        state = _argty_to_entry[procdesc["ty"]].default_state(procdesc)
+        state = argty_to_entry[procdesc["ty"]].default_state(procdesc)
         argument["desc"] = procdesc
         argument["state"] = state
 
         old_entry, widget_item = self._arg_to_entry_widgetitem[name]
         old_entry.deleteLater()
 
-        entry = _argty_to_entry[procdesc["ty"]](argument)
+        entry = argty_to_entry[procdesc["ty"]](argument)
         self._arg_to_entry_widgetitem[name] = entry, widget_item
         self.setItemWidget(widget_item, 1, entry)
 
@@ -466,7 +362,7 @@ class ExperimentManager:
     def initialize_submission_arguments(self, expurl, arginfo):
         arguments = OrderedDict()
         for name, (procdesc, group) in arginfo.items():
-            state = _argty_to_entry[procdesc["ty"]].default_state(procdesc)
+            state = argty_to_entry[procdesc["ty"]].default_state(procdesc)
             arguments[name] = {
                 "desc": procdesc,
                 "group": group,
@@ -512,7 +408,7 @@ class ExperimentManager:
 
         argument_values = dict()
         for name, argument in arguments.items():
-            entry_cls = _argty_to_entry[argument["desc"]["ty"]]
+            entry_cls = argty_to_entry[argument["desc"]["ty"]]
             argument_values[name] = entry_cls.state_to_value(argument["state"])
 
         expid = {

artiq/gui/log.py

@@ -9,11 +9,6 @@ from pyqtgraph import dockarea, LayoutWidget
 
 from artiq.gui.tools import log_level_to_name
 
-try:
-    QSortFilterProxyModel = QtCore.QSortFilterProxyModel
-except AttributeError:
-    QSortFilterProxyModel = QtGui.QSortFilterProxyModel
-
 
 def _make_wrappable(row, width=30):
     level, source, time, msg = row
@@ -34,8 +29,7 @@ class Model(QtCore.QAbstractTableModel):
         timer.timeout.connect(self.timer_tick)
         timer.start(100)
 
-        self.fixed_font = QtGui.QFont()
+        self.fixed_font = QtGui.QFontDatabase.systemFont(QtGui.QFontDatabase.FixedFont)
-        self.fixed_font.setFamily("Monospace")
 
         self.white = QtGui.QBrush(QtGui.QColor(255, 255, 255))
         self.black = QtGui.QBrush(QtGui.QColor(0, 0, 0))
@@ -87,6 +81,8 @@ class Model(QtCore.QAbstractTableModel):
             if (role == QtCore.Qt.FontRole
                     and index.column() == 1):
                 return self.fixed_font
+            elif role == QtCore.Qt.TextAlignmentRole:
+                return QtCore.Qt.AlignLeft | QtCore.Qt.AlignTop
             elif role == QtCore.Qt.BackgroundRole:
                 level = self.entries[index.row()][0]
                 if level >= logging.ERROR:
@@ -114,9 +110,9 @@ class Model(QtCore.QAbstractTableModel):
                     time.strftime("%m/%d %H:%M:%S", time.localtime(v[2])))
 
 
-class _LogFilterProxyModel(QSortFilterProxyModel):
+class _LogFilterProxyModel(QtCore.QSortFilterProxyModel):
     def __init__(self, min_level, freetext):
-        QSortFilterProxyModel.__init__(self)
+        QtCore.QSortFilterProxyModel.__init__(self)
         self.min_level = min_level
         self.freetext = freetext
 

artiq/gui/shortcuts.py

@@ -1,13 +1,8 @@
 import logging
 from functools import partial
 
-from quamash import QtGui, QtCore
+from quamash import QtGui, QtCore, QtWidgets
 from pyqtgraph import dockarea
-try:
-    from quamash import QtWidgets
-    QShortcut = QtWidgets.QShortcut
-except:
-    QShortcut = QtGui.QShortcut
 
 
 logger = logging.getLogger(__name__)
@@ -66,7 +61,7 @@ class ShortcutsDock(dockarea.Dock):
                 "open": open,
                 "submit": submit
             }
-            shortcut = QShortcut("F" + str(i+1), main_window)
+            shortcut = QtWidgets.QShortcut("F" + str(i+1), main_window)
             shortcut.setContext(QtCore.Qt.ApplicationShortcut)
             shortcut.activated.connect(partial(self._activated, i))
 

artiq/master/log.py

@@ -24,9 +24,9 @@ class LogBufferHandler(logging.Handler):
 
     def emit(self, record):
         message = self.format(record)
-        for part in message.split("\n"):
+        self.log_buffer.log(record.levelno, record.source, record.created,
-            self.log_buffer.log(record.levelno, record.source, record.created,
+                            message)
-                                part)
+
 
 def log_args(parser):
     group = parser.add_argument_group("logging")

artiq/protocols/pyon.py

@@ -35,6 +35,7 @@ _encode_map = {
     bytes: "bytes",
     tuple: "tuple",
     list: "list",
+    set: "set",
     dict: "dict",
     wrapping_int: "number",
     Fraction: "fraction",
@@ -98,6 +99,12 @@ class _Encoder:
         r += "]"
         return r
 
+    def encode_set(self, x):
+        r = "{"
+        r += ", ".join([self.encode(item) for item in x])
+        r += "}"
+        return r
+
     def encode_dict(self, x):
         r = "{"
         if not self.pretty or len(x) < 2:
@@ -149,9 +156,7 @@ class _Encoder:
 
 def encode(x, pretty=False):
     """Serializes a Python object and returns the corresponding string in
-    Python syntax.
+    Python syntax."""
-
-    """
     return _Encoder(pretty).encode(x)
 
 
@@ -181,9 +186,7 @@ _eval_dict = {
 
 def decode(s):
     """Parses a string in the Python syntax, reconstructs the corresponding
-    object, and returns it.
+    object, and returns it."""
-
-    """
     return eval(s, _eval_dict, {})
 
 
@@ -202,23 +205,3 @@ def load_file(filename):
     """Parses the specified file and returns the decoded Python object."""
     with open(filename, "r") as f:
         return decode(f.read())
-
-
-class FlatFileDB:
-    def __init__(self, filename):
-        self.filename = filename
-        self.data = pyon.load_file(self.filename)
-
-    def save(self):
-        pyon.store_file(self.filename, self.data)
-
-    def get(self, key):
-        return self.data[key]
-
-    def set(self, key, value):
-        self.data[key] = value
-        self.save()
-
-    def delete(self, key):
-        del self.data[key]
-        self.save()

artiq/test/test_serialization.py

@@ -10,6 +10,7 @@ from artiq.protocols import pyon
 _pyon_test_object = {
     (1, 2): [(3, 4.2), (2, )],
     Fraction(3, 4): np.linspace(5, 10, 1),
+    {"testing", "sets"},
     "a": np.int8(9), "b": np.int16(-98), "c": np.int32(42), "d": np.int64(-5),
     "e": np.uint8(8), "f": np.uint16(5), "g": np.uint32(4), "h": np.uint64(9),
     "x": np.float16(9.0), "y": np.float32(9.0), "z": np.float64(9.0),

conda/artiq-kc705-nist_clock/meta.yaml

@@ -22,6 +22,6 @@ requirements:
     - artiq {{ "{tag} py_{number}+git{hash}".format(tag=environ.get("GIT_DESCRIBE_TAG"), number=environ.get("GIT_DESCRIBE_NUMBER"), hash=environ.get("GIT_DESCRIBE_HASH")[1:]) if "GIT_DESCRIBE_TAG" in environ else "" }}
 
 about:
-  home: http://m-labs.hk/artiq
+  home: https://m-labs.hk/artiq
   license: GPL
   summary: 'Bitstream, BIOS and runtime for NIST_QC2 on the KC705 board'

conda/artiq-kc705-nist_qc1/meta.yaml

@@ -22,6 +22,6 @@ requirements:
     - artiq {{ "{tag} py_{number}+git{hash}".format(tag=environ.get("GIT_DESCRIBE_TAG"), number=environ.get("GIT_DESCRIBE_NUMBER"), hash=environ.get("GIT_DESCRIBE_HASH")[1:]) if "GIT_DESCRIBE_TAG" in environ else "" }}
 
 about:
-  home: http://m-labs.hk/artiq
+  home: https://m-labs.hk/artiq
   license: GPL
   summary: 'Bitstream, BIOS and runtime for NIST_QC1 on the KC705 board'

conda/artiq-kc705-nist_qc2/meta.yaml

@@ -22,6 +22,6 @@ requirements:
     - artiq {{ "{tag} py_{number}+git{hash}".format(tag=environ.get("GIT_DESCRIBE_TAG"), number=environ.get("GIT_DESCRIBE_NUMBER"), hash=environ.get("GIT_DESCRIBE_HASH")[1:]) if "GIT_DESCRIBE_TAG" in environ else "" }}
 
 about:
-  home: http://m-labs.hk/artiq
+  home: https://m-labs.hk/artiq
   license: GPL
   summary: 'Bitstream, BIOS and runtime for NIST_QC2 on the KC705 board'

conda/artiq/meta.yaml

@@ -46,11 +46,12 @@ requirements:
     - sphinx-argparse
     - h5py
     - dateutil
+    - pyqt5
     - quamash
     - pyqtgraph
     - pygit2
     - aiohttp
-    - binutils-or1k-linux # [linux]
+    - binutils-or1k-linux
     - pythonparser
     - levenshtein
 
@@ -59,6 +60,6 @@ test:
     - artiq
 
 about:
-  home: http://m-labs.hk/artiq
+  home: https://m-labs.hk/artiq
   license: GPL
   summary: 'ARTIQ (Advanced Real-Time Infrastructure for Quantum physics) is a next-generation control system for quantum information experiments. It is being developed in partnership with the Ion Storage Group at NIST, and its applicability reaches beyond ion trapping.'

doc/manual/installing.rst

@@ -14,6 +14,11 @@ But you can also :ref:`install from sources <install-from-sources>`.
 Installing using conda
 ----------------------
 
+.. warning::
+    Conda packages are supported for Linux (64-bit) and Windows (32- and 64-bit). Users of other
+    operating systems (32-bit Linux, BSD, ...) should install from source.
+
+
 Installing Anaconda or Miniconda
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
@@ -21,10 +26,6 @@ Installing Anaconda or Miniconda
 
 * Or install the more minimalistic Miniconda (choose Python 3.5) from http://conda.pydata.org/miniconda.html
 
-.. warning::
-    If you are installing on Windows, choose the Windows 32-bit version regardless of whether you have
-    a 32-bit or 64-bit Windows.
-
 After installing either Anaconda or Miniconda, open a new terminal and make sure the following command works::
 
     $ conda

doc/manual/introduction.rst

@@ -20,4 +20,4 @@ Technologies employed include Python, Migen, MiSoC/mor1kx, LLVM and llvmlite.
 ARTIQ is licensed under 3-clause BSD.
 
 Website:
-http://m-labs.hk/artiq
+https://m-labs.hk/artiq

examples/master/repository/histograms.py

@@ -0,0 +1,35 @@
+from time import sleep
+
+import numpy as np
+
+from artiq.experiment import *
+
+
+class Histograms(EnvExperiment):
+    """Histograms demo"""
+    def build(self):
+        pass
+
+    def run(self):
+        nbins = 50
+        npoints = 20
+
+        bin_boundaries = np.linspace(-10, 30, nbins + 1)
+        self.set_dataset("hd_bins", bin_boundaries,
+                         broadcast=True, save=False)
+
+        xs = np.empty(npoints)
+        xs.fill(np.nan)
+        xs = self.set_dataset("hd_xs", xs,
+                              broadcast=True, save=False)
+
+        counts = np.empty((npoints, nbins))
+        counts = self.set_dataset("hd_counts", counts, 
+                              broadcast=True, save=False)
+
+        for i in range(npoints):
+            histogram, _ = np.histogram(np.random.normal(i, size=1000),
+                                        bin_boundaries)
+            counts[i] = histogram
+            xs[i] = i % 8
+            sleep(0.3)

setup.py

@@ -10,6 +10,7 @@ if sys.version_info[:3] < (3, 5, 1):
     raise Exception("You need Python 3.5.1+")
 
 
+# Depends on PyQt5, but setuptools cannot check for it.
 requirements = [
     "sphinx", "sphinx-argparse", "pyserial", "numpy", "scipy",
     "python-dateutil", "prettytable", "h5py",
@@ -45,7 +46,7 @@ setup(
     cmdclass=versioneer.get_cmdclass(),
     author="M-Labs / NIST Ion Storage Group",
     author_email="sb@m-labs.hk",
-    url="http://m-labs.hk/artiq",
+    url="https://m-labs.hk/artiq",
     description="A control system for trapped-ion experiments",
     long_description=open("README.rst").read(),
     license="GPL",