From f85554486a1d428e0b11c81ded785b34eafc02b5 Mon Sep 17 00:00:00 2001 From: Sebastien Bourdeauducq Date: Wed, 14 Oct 2015 19:29:58 +0800 Subject: [PATCH 01/73] gui: dataset filtering with QSortFilterProxyModel --- artiq/gui/datasets.py | 14 ++++---------- 1 file changed, 4 insertions(+), 10 deletions(-) diff --git a/artiq/gui/datasets.py b/artiq/gui/datasets.py index ebe8d295c..3d547902b 100644 --- a/artiq/gui/datasets.py +++ b/artiq/gui/datasets.py @@ -74,15 +74,7 @@ class DatasetsDock(dockarea.Dock): self.displays = dict() def _search_datasets(self): - model = self.table_model - search = self.search.displayText() - for row in range(model.rowCount(model.index(0, 0))): - index = model.index(row, 0) - dataset = model.data(index, QtCore.Qt.DisplayRole) - if search in dataset: - self.table.showRow(row) - else: - self.table.hideRow(row) + self.table_model_filter.setFilterFixedString(self.search.displayText()) def get_dataset(self, key): return self.table_model.backing_store[key][1] @@ -97,7 +89,9 @@ class DatasetsDock(dockarea.Dock): def init_datasets_model(self, init): self.table_model = DatasetsModel(self.table, init) - self.table.setModel(self.table_model) + self.table_model_filter = QtCore.QSortFilterProxyModel() + self.table_model_filter.setSourceModel(self.table_model) + self.table.setModel(self.table_model_filter) return self.table_model def update_display_data(self, dsp): From f3b3bf3036bfa242ea92c17226dae6d185b52260 Mon Sep 17 00:00:00 2001 From: Sebastien Bourdeauducq Date: Wed, 14 Oct 2015 21:21:19 +0800 Subject: [PATCH 02/73] gui: log filtering --- artiq/frontend/artiq_gui.py | 1 + artiq/gui/log.py | 58 +++++++++++++++++++++++++++++++++---- 2 files changed, 54 insertions(+), 5 deletions(-) diff --git a/artiq/frontend/artiq_gui.py b/artiq/frontend/artiq_gui.py index 10db8eb41..4a4b3369c 100755 --- a/artiq/frontend/artiq_gui.py +++ b/artiq/frontend/artiq_gui.py @@ -116,6 +116,7 @@ def main(): atexit.register(lambda: loop.run_until_complete(d_schedule.sub_close())) d_log = LogDock() + smgr.register(d_log) loop.run_until_complete(d_log.sub_connect( args.server, args.port_notify)) atexit.register(lambda: loop.run_until_complete(d_log.sub_close())) diff --git a/artiq/gui/log.py b/artiq/gui/log.py index 51e6aefac..bbb3ad2b0 100644 --- a/artiq/gui/log.py +++ b/artiq/gui/log.py @@ -3,7 +3,7 @@ import logging import time from quamash import QtGui, QtCore -from pyqtgraph import dockarea +from pyqtgraph import dockarea, LayoutWidget from artiq.protocols.sync_struct import Subscriber from artiq.gui.tools import ListSyncModel @@ -20,6 +20,7 @@ def _level_to_name(level): return "INFO" return "DEBUG" + class _LogModel(ListSyncModel): def __init__(self, parent, init): ListSyncModel.__init__(self, @@ -66,10 +67,39 @@ class _LogModel(ListSyncModel): return v[3] +class _LevelFilterProxyModel(QtCore.QSortFilterProxyModel): + def __init__(self, min_level): + QtCore.QSortFilterProxyModel.__init__(self) + self.min_level = min_level + + def filterAcceptsRow(self, sourceRow, sourceParent): + model = self.sourceModel() + index = model.index(sourceRow, 0, sourceParent) + data = model.data(index, QtCore.Qt.DisplayRole) + return getattr(logging, data) >= self.min_level + + def set_min_level(self, min_level): + self.min_level = min_level + self.invalidateFilter() + + class LogDock(dockarea.Dock): def __init__(self): dockarea.Dock.__init__(self, "Log", size=(1000, 300)) + grid = LayoutWidget() + self.addWidget(grid) + + grid.addWidget(QtGui.QLabel("Minimum level: "), 0, 0) + grid.layout.setColumnStretch(0, 0) + grid.layout.setColumnStretch(1, 0) + grid.layout.setColumnStretch(2, 1) + self.filterbox = QtGui.QComboBox() + for item in "DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL": + self.filterbox.addItem(item) + grid.addWidget(self.filterbox, 0, 1) + self.filterbox.currentIndexChanged.connect(self.filter_changed) + self.log = QtGui.QTableView() self.log.setSelectionMode(QtGui.QAbstractItemView.NoSelection) self.log.horizontalHeader().setResizeMode( @@ -78,7 +108,7 @@ class LogDock(dockarea.Dock): QtGui.QAbstractItemView.ScrollPerPixel) self.log.setShowGrid(False) self.log.setTextElideMode(QtCore.Qt.ElideNone) - self.addWidget(self.log) + grid.addWidget(self.log, 1, 0, colspan=3) self.scroll_at_bottom = False async def sub_connect(self, host, port): @@ -88,6 +118,10 @@ class LogDock(dockarea.Dock): async def sub_close(self): await self.subscriber.close() + def filter_changed(self): + self.table_model_filter.set_min_level( + getattr(logging, self.filterbox.currentText())) + def rows_inserted_before(self): scrollbar = self.log.verticalScrollBar() self.scroll_at_bottom = scrollbar.value() == scrollbar.maximum() @@ -98,7 +132,21 @@ class LogDock(dockarea.Dock): def init_log_model(self, init): table_model = _LogModel(self.log, init) - self.log.setModel(table_model) - table_model.rowsAboutToBeInserted.connect(self.rows_inserted_before) - table_model.rowsInserted.connect(self.rows_inserted_after) + self.table_model_filter = _LevelFilterProxyModel( + getattr(logging, self.filterbox.currentText())) + self.table_model_filter.setSourceModel(table_model) + self.log.setModel(self.table_model_filter) + self.table_model_filter.rowsAboutToBeInserted.connect(self.rows_inserted_before) + self.table_model_filter.rowsInserted.connect(self.rows_inserted_after) return table_model + + def save_state(self): + return {"min_level_idx": self.filterbox.currentIndex()} + + def restore_state(self, state): + try: + idx = state["min_level_idx"] + except KeyError: + pass + else: + self.filterbox.setCurrentIndex(idx) From 49acd10dfde1c4d94d6ce7919ce0170736867d14 Mon Sep 17 00:00:00 2001 From: Sebastien Bourdeauducq Date: Wed, 14 Oct 2015 21:32:16 +0800 Subject: [PATCH 03/73] gui: fix data update in DictSyncModel/ListSyncModel --- artiq/gui/tools.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/artiq/gui/tools.py b/artiq/gui/tools.py index 242c66600..4899b1173 100644 --- a/artiq/gui/tools.py +++ b/artiq/gui/tools.py @@ -95,7 +95,7 @@ class DictSyncModel(QtCore.QAbstractTableModel): new_row = self._find_row(k, v) if old_row == new_row: self.dataChanged.emit(self.index(old_row, 0), - self.index(old_row, len(self.headers))) + self.index(old_row, len(self.headers)-1)) else: self.beginMoveRows(QtCore.QModelIndex(), old_row, old_row, QtCore.QModelIndex(), new_row) @@ -157,7 +157,7 @@ class ListSyncModel(QtCore.QAbstractTableModel): def __setitem__(self, k, v): self.dataChanged.emit(self.index(k, 0), - self.index(k, len(self.headers))) + self.index(k, len(self.headers)-1)) self.backing_store[k] = v def __delitem__(self, k): From 9e2e233fef6a0549c4fe63f4c41d51586172916b Mon Sep 17 00:00:00 2001 From: Sebastien Bourdeauducq Date: Thu, 15 Oct 2015 23:47:31 +0800 Subject: [PATCH 04/73] master: log file support --- artiq/master/log.py | 28 ++++++++++++++++++++++++---- 1 file changed, 24 insertions(+), 4 deletions(-) diff --git a/artiq/master/log.py b/artiq/master/log.py index 4388819d8..3e6bec2b0 100644 --- a/artiq/master/log.py +++ b/artiq/master/log.py @@ -1,4 +1,5 @@ import logging +import logging.handlers from artiq.protocols.sync_struct import Notifier @@ -74,22 +75,41 @@ class SourceFilter: def log_args(parser): - group = parser.add_argument_group("verbosity") + group = parser.add_argument_group("logging") group.add_argument("-v", "--verbose", default=0, action="count", - help="increase logging level for the master process") + help="increase logging level of the master process") group.add_argument("-q", "--quiet", default=0, action="count", - help="decrease logging level for the master process") + help="decrease logging level of the master process") + group.add_argument("--log-file", default="", + help="store logs in rotated files; set the " + "base filename") + group.add_argument("--log-max-size", type=int, default=1024, + help="maximum size of each log file in KiB " + "(default: %(default)d)") + group.add_argument("--log-backup-count", type=int, default=6, + help="number of old log files to keep (. is added " + "to the base filename (default: %(default)d)") def init_log(args): root_logger = logging.getLogger() root_logger.setLevel(logging.NOTSET) # we use our custom filter only flt = SourceFilter(logging.WARNING + args.quiet*10 - args.verbose*10) + full_fmt = logging.Formatter( + "%(levelname)s:%(source)s:%(name)s:%(message)s") handlers = [] console_handler = logging.StreamHandler() - console_handler.setFormatter(logging.Formatter("%(levelname)s:%(source)s:%(name)s:%(message)s")) + console_handler.setFormatter(full_fmt) handlers.append(console_handler) + + if args.log_file: + file_handler = logging.handlers.RotatingFileHandler( + args.log_file, + maxBytes=args.log_max_size*1024, + backupCount=args.log_backup_count) + file_handler.setFormatter(full_fmt) + handlers.append(file_handler) log_buffer = LogBuffer(1000) buffer_handler = LogBufferHandler(log_buffer) From cbda753f440f3b4e0e7d33eb1641ed63f2bcf1d3 Mon Sep 17 00:00:00 2001 From: Sebastien Bourdeauducq Date: Fri, 16 Oct 2015 00:53:35 +0800 Subject: [PATCH 05/73] master: TCP server for remote logging --- artiq/frontend/artiq_master.py | 21 ++++++--- artiq/master/log.py | 42 +++-------------- artiq/protocols/logging.py | 67 ++++++++++++++++++++++++++++ doc/manual/default_network_ports.rst | 2 + 4 files changed, 91 insertions(+), 41 deletions(-) create mode 100644 artiq/protocols/logging.py diff --git a/artiq/frontend/artiq_master.py b/artiq/frontend/artiq_master.py index dc2fb0ac5..186d14e68 100755 --- a/artiq/frontend/artiq_master.py +++ b/artiq/frontend/artiq_master.py @@ -5,9 +5,10 @@ import argparse import atexit import os -from artiq.protocols.pc_rpc import Server +from artiq.protocols.pc_rpc import Server as RPCServer from artiq.protocols.sync_struct import Publisher -from artiq.master.log import log_args, init_log +from artiq.protocols.logging import Server as LoggingServer +from artiq.master.log import log_args, init_log, log_worker from artiq.master.databases import DeviceDB, DatasetDB from artiq.master.scheduler import Scheduler from artiq.master.worker_db import get_last_rid @@ -27,6 +28,9 @@ def get_argparser(): group.add_argument( "--port-control", default=3251, type=int, help="TCP port to listen to for control (default: %(default)d)") + group.add_argument( + "--port-logging", default=1066, type=int, + help="TCP port to listen to for remote logging (default: %(default)d)") group = parser.add_argument_group("databases") group.add_argument("--device-db", default="device_db.pyon", @@ -49,7 +53,7 @@ def get_argparser(): def main(): args = get_argparser().parse_args() - log_buffer, log_forwarder = init_log(args) + log_buffer = init_log(args) if os.name == "nt": loop = asyncio.ProactorEventLoop() asyncio.set_event_loop(loop) @@ -67,7 +71,7 @@ def main(): else: repo_backend = FilesystemBackend(args.repository) repository = Repository(repo_backend, device_db.get_device_db, - log_forwarder.log_worker) + log_worker) atexit.register(repository.close) repository.scan_async() @@ -76,14 +80,14 @@ def main(): "get_device": device_db.get, "get_dataset": dataset_db.get, "update_dataset": dataset_db.update, - "log": log_forwarder.log_worker + "log": log_worker } scheduler = Scheduler(get_last_rid() + 1, worker_handlers, repo_backend) worker_handlers["scheduler_submit"] = scheduler.submit scheduler.start() atexit.register(lambda: loop.run_until_complete(scheduler.stop())) - server_control = Server({ + server_control = RPCServer({ "master_device_db": device_db, "master_dataset_db": dataset_db, "master_schedule": scheduler, @@ -104,6 +108,11 @@ def main(): args.bind, args.port_notify)) atexit.register(lambda: loop.run_until_complete(server_notify.stop())) + server_logging = LoggingServer() + loop.run_until_complete(server_logging.start( + args.bind, args.port_logging)) + atexit.register(lambda: loop.run_until_complete(server_logging.stop())) + loop.run_forever() if __name__ == "__main__": diff --git a/artiq/master/log.py b/artiq/master/log.py index 3e6bec2b0..ca8e1fd71 100644 --- a/artiq/master/log.py +++ b/artiq/master/log.py @@ -2,6 +2,7 @@ import logging import logging.handlers from artiq.protocols.sync_struct import Notifier +from artiq.protocols.logging import parse_log_message, log_with_name class LogBuffer: @@ -25,38 +26,11 @@ class LogBufferHandler(logging.Handler): self.log_buffer.log(record.levelno, record.source, record.created, message) -name_to_level = { - "CRITICAL": logging.CRITICAL, - "ERROR": logging.ERROR, - "WARN": logging.WARNING, - "WARNING": logging.WARNING, - "INFO": logging.INFO, - "DEBUG": logging.DEBUG, -} - - -def parse_log_message(msg): - for name, level in name_to_level.items(): - if msg.startswith(name + ":"): - remainder = msg[len(name) + 1:] - try: - idx = remainder.index(":") - except: - continue - return level, remainder[:idx], remainder[idx+1:] - return logging.INFO, "print", msg - - -fwd_logger = logging.getLogger("fwd") - - -class LogForwarder: - def log_worker(self, rid, message): - level, name, message = parse_log_message(message) - fwd_logger.name = name - fwd_logger.log(level, message, - extra={"source": "worker({})".format(rid)}) - log_worker.worker_pass_rid = True +def log_worker(rid, message): + level, name, message = parse_log_message(message) + log_with_name(name, level, message, + extra={"source": "worker({})".format(rid)}) +log_worker.worker_pass_rid = True class SourceFilter: @@ -120,6 +94,4 @@ def init_log(args): handler.addFilter(flt) root_logger.addHandler(handler) - log_forwarder = LogForwarder() - - return log_buffer, log_forwarder + return log_buffer diff --git a/artiq/protocols/logging.py b/artiq/protocols/logging.py new file mode 100644 index 000000000..42cca1572 --- /dev/null +++ b/artiq/protocols/logging.py @@ -0,0 +1,67 @@ +import asyncio +import logging + +from artiq.protocols.asyncio_server import AsyncioServer + + +_fwd_logger = logging.getLogger("fwd") + + +def log_with_name(name, *args, **kwargs): + _fwd_logger.name = name + _fwd_logger.log(*args, **kwargs) + + +_name_to_level = { + "CRITICAL": logging.CRITICAL, + "ERROR": logging.ERROR, + "WARN": logging.WARNING, + "WARNING": logging.WARNING, + "INFO": logging.INFO, + "DEBUG": logging.DEBUG, +} + + +def parse_log_message(msg): + for name, level in _name_to_level.items(): + if msg.startswith(name + ":"): + remainder = msg[len(name) + 1:] + try: + idx = remainder.index(":") + except: + continue + return level, remainder[:idx], remainder[idx+1:] + return logging.INFO, "print", msg + + +_init_string = b"ARTIQ logging\n" + + +class Server(AsyncioServer): + async def _handle_connection_cr(self, reader, writer): + try: + line = await reader.readline() + if line != _init_string: + return + + while True: + line = await reader.readline() + if not line: + break + try: + line = line.decode() + except: + return + line = line[:-1] + linesplit = line.split(":", 4) + if len(linesplit) != 4: + return + source, levelname, name, message = linesplit + try: + level = _name_to_level[levelname] + except KeyError: + return + log_with_name(name, level, message, + extra={"source": source}) + finally: + writer.close() diff --git a/doc/manual/default_network_ports.rst b/doc/manual/default_network_ports.rst index 35e576d2b..ab50291af 100644 --- a/doc/manual/default_network_ports.rst +++ b/doc/manual/default_network_ports.rst @@ -8,6 +8,8 @@ Default network ports +--------------------------+--------------+ | Core device (mon/inj) | 3250 (UDP) | +--------------------------+--------------+ +| Master (logging) | 1066 | ++--------------------------+--------------+ | InfluxDB bridge | 3248 | +--------------------------+--------------+ | Controller manager | 3249 | From f3e61e265cf919bb28ed7f96732241112189d691 Mon Sep 17 00:00:00 2001 From: whitequark Date: Fri, 16 Oct 2015 07:06:22 +0300 Subject: [PATCH 06/73] conda: move everything except artiq to conda-recipes (closes #138). --- .travis/get-anaconda.sh | 1 + conda/README.md | 17 -------------- conda/aiohttp/bld.bat | 2 -- conda/aiohttp/build.sh | 3 --- conda/aiohttp/meta.yaml | 36 ----------------------------- conda/binutils-or1k-linux/README.md | 8 ------- conda/binutils-or1k-linux/bld.bat | 10 -------- conda/binutils-or1k-linux/build.sh | 6 ----- conda/binutils-or1k-linux/meta.yaml | 20 ---------------- conda/chardet/bld.bat | 2 -- conda/chardet/build.sh | 1 - conda/chardet/meta.yaml | 33 -------------------------- conda/dateutil/bld.bat | 1 - conda/dateutil/build.sh | 1 - conda/dateutil/meta.yaml | 30 ------------------------ conda/flterm/build.sh | 5 ---- conda/flterm/meta.yaml | 12 ---------- conda/libgit2/bld.bat | 20 ---------------- conda/libgit2/build.sh | 7 ------ conda/libgit2/meta.yaml | 27 ---------------------- conda/libssh2/bld.bat | 17 -------------- conda/libssh2/build.sh | 7 ------ conda/libssh2/meta.yaml | 23 ------------------ conda/lit/bld.bat | 2 -- conda/lit/build.sh | 1 - conda/lit/meta.yaml | 27 ---------------------- conda/llvmdev-or1k/bld.bat | 25 -------------------- conda/llvmdev-or1k/build.sh | 10 -------- conda/llvmdev-or1k/meta.yaml | 22 ------------------ conda/llvmlite-artiq/bld.bat | 8 ------- conda/llvmlite-artiq/build.sh | 3 --- conda/llvmlite-artiq/meta.yaml | 27 ---------------------- conda/prettytable/bld.bat | 8 ------- conda/prettytable/build.sh | 3 --- conda/prettytable/meta.yaml | 26 --------------------- conda/pydaqmx/bld.bat | 2 -- conda/pydaqmx/build.sh | 2 -- conda/pydaqmx/meta.yaml | 22 ------------------ conda/pyelftools/bld.bat | 1 - conda/pyelftools/build.sh | 1 - conda/pyelftools/meta.yaml | 26 --------------------- conda/pygit2/bld.bat | 3 --- conda/pygit2/build.sh | 2 -- conda/pygit2/meta.yaml | 28 ---------------------- conda/pyqtgraph/bld.bat | 2 -- conda/pyqtgraph/build.sh | 3 --- conda/pyqtgraph/meta.yaml | 27 ---------------------- conda/pythonparser/bld.bat | 2 -- conda/pythonparser/build.sh | 2 -- conda/pythonparser/meta.yaml | 24 ------------------- conda/quamash/bld.bat | 1 - conda/quamash/build.sh | 3 --- conda/quamash/meta.yaml | 29 ----------------------- conda/sphinx-argparse/bld.bat | 1 - conda/sphinx-argparse/build.sh | 1 - conda/sphinx-argparse/meta.yaml | 28 ---------------------- doc/manual/installing.rst | 7 +++--- 57 files changed, 5 insertions(+), 663 deletions(-) delete mode 100644 conda/README.md delete mode 100644 conda/aiohttp/bld.bat delete mode 100644 conda/aiohttp/build.sh delete mode 100644 conda/aiohttp/meta.yaml delete mode 100755 conda/binutils-or1k-linux/README.md delete mode 100644 conda/binutils-or1k-linux/bld.bat delete mode 100755 conda/binutils-or1k-linux/build.sh delete mode 100644 conda/binutils-or1k-linux/meta.yaml delete mode 100644 conda/chardet/bld.bat delete mode 100644 conda/chardet/build.sh delete mode 100644 conda/chardet/meta.yaml delete mode 100644 conda/dateutil/bld.bat delete mode 100644 conda/dateutil/build.sh delete mode 100644 conda/dateutil/meta.yaml delete mode 100644 conda/flterm/build.sh delete mode 100644 conda/flterm/meta.yaml delete mode 100644 conda/libgit2/bld.bat delete mode 100644 conda/libgit2/build.sh delete mode 100644 conda/libgit2/meta.yaml delete mode 100644 conda/libssh2/bld.bat delete mode 100644 conda/libssh2/build.sh delete mode 100644 conda/libssh2/meta.yaml delete mode 100644 conda/lit/bld.bat delete mode 100644 conda/lit/build.sh delete mode 100644 conda/lit/meta.yaml delete mode 100644 conda/llvmdev-or1k/bld.bat delete mode 100644 conda/llvmdev-or1k/build.sh delete mode 100644 conda/llvmdev-or1k/meta.yaml delete mode 100644 conda/llvmlite-artiq/bld.bat delete mode 100755 conda/llvmlite-artiq/build.sh delete mode 100644 conda/llvmlite-artiq/meta.yaml delete mode 100644 conda/prettytable/bld.bat delete mode 100644 conda/prettytable/build.sh delete mode 100644 conda/prettytable/meta.yaml delete mode 100644 conda/pydaqmx/bld.bat delete mode 100644 conda/pydaqmx/build.sh delete mode 100644 conda/pydaqmx/meta.yaml delete mode 100644 conda/pyelftools/bld.bat delete mode 100644 conda/pyelftools/build.sh delete mode 100644 conda/pyelftools/meta.yaml delete mode 100644 conda/pygit2/bld.bat delete mode 100644 conda/pygit2/build.sh delete mode 100644 conda/pygit2/meta.yaml delete mode 100644 conda/pyqtgraph/bld.bat delete mode 100644 conda/pyqtgraph/build.sh delete mode 100644 conda/pyqtgraph/meta.yaml delete mode 100644 conda/pythonparser/bld.bat delete mode 100644 conda/pythonparser/build.sh delete mode 100644 conda/pythonparser/meta.yaml delete mode 100644 conda/quamash/bld.bat delete mode 100644 conda/quamash/build.sh delete mode 100644 conda/quamash/meta.yaml delete mode 100644 conda/sphinx-argparse/bld.bat delete mode 100644 conda/sphinx-argparse/build.sh delete mode 100644 conda/sphinx-argparse/meta.yaml diff --git a/.travis/get-anaconda.sh b/.travis/get-anaconda.sh index 790a723ad..69f107dce 100755 --- a/.travis/get-anaconda.sh +++ b/.travis/get-anaconda.sh @@ -10,4 +10,5 @@ conda update -q conda conda info -a conda install conda-build jinja2 conda create -q -n py35 python=$TRAVIS_PYTHON_VERSION +conda config --add channels https://conda.anaconda.org/m-labs/channel/main conda config --add channels https://conda.anaconda.org/m-labs/channel/dev diff --git a/conda/README.md b/conda/README.md deleted file mode 100644 index 017742a8b..000000000 --- a/conda/README.md +++ /dev/null @@ -1,17 +0,0 @@ -Uploading conda packages (Python 3.5) -===================================== - -Preparing: - - 1. [Install miniconda][miniconda] - 2. `conda update -q conda` - 3. `conda install conda-build jinja2 anaconda` - 4. `conda create -q -n py35 python=3.5` - 5. `conda config --add channels https://conda.anaconda.org/m-labs/channel/dev` - -Building: - - 1. `conda build pkgname --python 3.5`; this command displays a path to the freshly built package - 2. `anaconda upload -c main -c dev` - -[miniconda]: http://conda.pydata.org/docs/install/quick.html#linux-miniconda-install diff --git a/conda/aiohttp/bld.bat b/conda/aiohttp/bld.bat deleted file mode 100644 index c40a9bbef..000000000 --- a/conda/aiohttp/bld.bat +++ /dev/null @@ -1,2 +0,0 @@ -"%PYTHON%" setup.py install -if errorlevel 1 exit 1 diff --git a/conda/aiohttp/build.sh b/conda/aiohttp/build.sh deleted file mode 100644 index 8e25a1455..000000000 --- a/conda/aiohttp/build.sh +++ /dev/null @@ -1,3 +0,0 @@ -#!/bin/bash - -$PYTHON setup.py install diff --git a/conda/aiohttp/meta.yaml b/conda/aiohttp/meta.yaml deleted file mode 100644 index 2b196ffc1..000000000 --- a/conda/aiohttp/meta.yaml +++ /dev/null @@ -1,36 +0,0 @@ -package: - name: aiohttp - version: "0.17.2" - -source: - fn: aiohttp-0.17.2.tar.gz - url: https://pypi.python.org/packages/source/a/aiohttp/aiohttp-0.17.2.tar.gz - md5: 7640928fd4b5c1ccf1f8bcad276d39d6 - -build: - number: 0 - -requirements: - build: - - python - - setuptools - - chardet - - run: - - python - - chardet - -test: - # Python imports - imports: - - aiohttp - - requires: - - chardet - - gunicorn # [not win] - - nose - -about: - home: https://github.com/KeepSafe/aiohttp/ - license: Apache Software License - summary: 'http client/server for asyncio' diff --git a/conda/binutils-or1k-linux/README.md b/conda/binutils-or1k-linux/README.md deleted file mode 100755 index d812cc7b2..000000000 --- a/conda/binutils-or1k-linux/README.md +++ /dev/null @@ -1,8 +0,0 @@ -binutils-or1k-linux -=================== - -To build this package on Windows: - -* Install cygwin -* Install the following packages: gcc-core g++-core make texinfo patch -* Run cygwin terminal and execute $ conda build binutils-or1k-linux \ No newline at end of file diff --git a/conda/binutils-or1k-linux/bld.bat b/conda/binutils-or1k-linux/bld.bat deleted file mode 100644 index 6c709129f..000000000 --- a/conda/binutils-or1k-linux/bld.bat +++ /dev/null @@ -1,10 +0,0 @@ -FOR /F "tokens=* USEBACKQ" %%F IN (`cygpath -u %PREFIX%`) DO ( -SET var=%%F -) -set PREFIX=%var% -FOR /F "tokens=* USEBACKQ" %%F IN (`cygpath -u %RECIPE_DIR%`) DO ( -SET var=%%F -) -set RECIPE_DIR=%var% -sh %RECIPE_DIR%/build.sh -if errorlevel 1 exit 1 diff --git a/conda/binutils-or1k-linux/build.sh b/conda/binutils-or1k-linux/build.sh deleted file mode 100755 index faa6aa8e4..000000000 --- a/conda/binutils-or1k-linux/build.sh +++ /dev/null @@ -1,6 +0,0 @@ -patch -p1 < $RECIPE_DIR/../../misc/binutils-2.25.1-or1k-R_PCREL-pcrel_offset.patch -mkdir build -cd build -../configure --target=or1k-linux --prefix=$PREFIX -make -j2 -make install diff --git a/conda/binutils-or1k-linux/meta.yaml b/conda/binutils-or1k-linux/meta.yaml deleted file mode 100644 index d8e8f9e71..000000000 --- a/conda/binutils-or1k-linux/meta.yaml +++ /dev/null @@ -1,20 +0,0 @@ -package: - name: binutils-or1k-linux - version: 2.25.1 - -source: - fn: binutils-2.25.1.tar.bz2 - url: https://ftp.gnu.org/gnu/binutils/binutils-2.25.1.tar.bz2 - sha256: b5b14added7d78a8d1ca70b5cb75fef57ce2197264f4f5835326b0df22ac9f22 - -build: - number: 0 - -requirements: - build: - - system # [not win] - -about: - home: https://www.gnu.org/software/binutils/ - license: GPL - summary: 'A set of programming tools for creating and managing binary programs, object files, libraries, profile data, and assembly source code.' diff --git a/conda/chardet/bld.bat b/conda/chardet/bld.bat deleted file mode 100644 index c40a9bbef..000000000 --- a/conda/chardet/bld.bat +++ /dev/null @@ -1,2 +0,0 @@ -"%PYTHON%" setup.py install -if errorlevel 1 exit 1 diff --git a/conda/chardet/build.sh b/conda/chardet/build.sh deleted file mode 100644 index 5a5aeeb48..000000000 --- a/conda/chardet/build.sh +++ /dev/null @@ -1 +0,0 @@ -$PYTHON setup.py install diff --git a/conda/chardet/meta.yaml b/conda/chardet/meta.yaml deleted file mode 100644 index e9b7c795c..000000000 --- a/conda/chardet/meta.yaml +++ /dev/null @@ -1,33 +0,0 @@ -package: - name: chardet - version: 2.2.1 - -source: - fn: chardet-2.2.1.tar.gz - url: https://pypi.python.org/packages/source/c/chardet/chardet-2.2.1.tar.gz - md5: 4a758402eaefd0331bdedc7ecb6f452c - -build: - entry_points: - - chardetect = chardet.chardetect:main - number: 0 - -requirements: - build: - - python - - setuptools - - run: - - python - -test: - # Python imports - imports: - - chardet - - commands: - - chardetect run_test.py - -about: - home: https://github.com/chardet/chardet - license: GNU Library or Lesser General Public License (LGPL) diff --git a/conda/dateutil/bld.bat b/conda/dateutil/bld.bat deleted file mode 100644 index 39b5e1fee..000000000 --- a/conda/dateutil/bld.bat +++ /dev/null @@ -1 +0,0 @@ -%PYTHON% setup.py install diff --git a/conda/dateutil/build.sh b/conda/dateutil/build.sh deleted file mode 100644 index 5a5aeeb48..000000000 --- a/conda/dateutil/build.sh +++ /dev/null @@ -1 +0,0 @@ -$PYTHON setup.py install diff --git a/conda/dateutil/meta.yaml b/conda/dateutil/meta.yaml deleted file mode 100644 index fd9d40a3e..000000000 --- a/conda/dateutil/meta.yaml +++ /dev/null @@ -1,30 +0,0 @@ -package: - name: dateutil - version: 2.4.2 - -source: - fn: python-dateutil-2.4.2.tar.gz - url: https://pypi.python.org/packages/source/p/python-dateutil/python-dateutil-2.4.2.tar.gz - md5: 4ef68e1c485b09e9f034e10473e5add2 - -build: - number: 0 - -requirements: - build: - - python - - setuptools - - six >=1.5 - run: - - python - - six >=1.5 - -test: - imports: - - dateutil - - dateutil.zoneinfo - -about: - home: https://dateutil.readthedocs.org - license: BSD - summary: 'Extensions to the standard Python datetime module' diff --git a/conda/flterm/build.sh b/conda/flterm/build.sh deleted file mode 100644 index 1121beb65..000000000 --- a/conda/flterm/build.sh +++ /dev/null @@ -1,5 +0,0 @@ -#!/bin/bash - -make -C $SRC_DIR/tools flterm -mkdir -p $PREFIX/bin -cp $SRC_DIR/tools/flterm $PREFIX/bin/ diff --git a/conda/flterm/meta.yaml b/conda/flterm/meta.yaml deleted file mode 100644 index 16afb47cb..000000000 --- a/conda/flterm/meta.yaml +++ /dev/null @@ -1,12 +0,0 @@ -package: - name: flterm - version: 0 - -source: - git_url: https://github.com/m-labs/misoc - git_tag: master - -about: - home: https://github.com/m-labs/misoc/blob/master/tools/flterm.c - license: 3-clause BSD - summary: 'Serial terminal to connect to MiSoC uart.' diff --git a/conda/libgit2/bld.bat b/conda/libgit2/bld.bat deleted file mode 100644 index 268c18cd9..000000000 --- a/conda/libgit2/bld.bat +++ /dev/null @@ -1,20 +0,0 @@ -mkdir build -cd build -REM Configure step -if "%ARCH%"=="32" ( -set CMAKE_GENERATOR=Visual Studio 12 2013 -) else ( -set CMAKE_GENERATOR=Visual Studio 12 2013 Win64 -) -set CMAKE_GENERATOR_TOOLSET=v120_xp -cmake -G "%CMAKE_GENERATOR%" -DCMAKE_INSTALL_PREFIX=%PREFIX% -DSTDCALL=OFF -DCMAKE_PREFIX_PATH=$PREFIX %SRC_DIR% -if errorlevel 1 exit 1 -REM Build step -cmake --build . -if errorlevel 1 exit 1 -REM Install step -cmake --build . --target install -if errorlevel 1 exit 1 -REM Hack to help pygit2 to find libgit2 -mkdir %PREFIX%\Scripts -copy "%PREFIX%\bin\git2.dll" "%PREFIX%\Scripts\" \ No newline at end of file diff --git a/conda/libgit2/build.sh b/conda/libgit2/build.sh deleted file mode 100644 index dc4a85aa0..000000000 --- a/conda/libgit2/build.sh +++ /dev/null @@ -1,7 +0,0 @@ -#!/bin/bash - -mkdir build -cd build -cmake .. -DCMAKE_INSTALL_PREFIX=$PREFIX -DCMAKE_PREFIX_PATH=$PREFIX -make -j2 -make install diff --git a/conda/libgit2/meta.yaml b/conda/libgit2/meta.yaml deleted file mode 100644 index 5741b44b4..000000000 --- a/conda/libgit2/meta.yaml +++ /dev/null @@ -1,27 +0,0 @@ -package: - name: libgit2 - version: 0.22.3 - -source: - git_url: https://github.com/libgit2/libgit2 - git_tag: v0.22.3 - -build: - number: 1 - -requirements: - build: - - system # [linux] - - cmake # [linux] - - openssl - - libssh2 - - zlib - run: - - openssl - - zlib - - libssh2 - -about: - home: https://libgit2.github.com/ - license: GPLv2 with a special Linking Exception - summary: 'libgit2 is a portable, pure C implementation of the Git core methods provided as a re-entrant linkable library with a solid API, allowing you to write native speed custom Git applications in any language with bindings.' diff --git a/conda/libssh2/bld.bat b/conda/libssh2/bld.bat deleted file mode 100644 index ed957bd42..000000000 --- a/conda/libssh2/bld.bat +++ /dev/null @@ -1,17 +0,0 @@ -mkdir build -cd build -REM Configure step -if "%ARCH%"=="32" ( -set CMAKE_GENERATOR=Visual Studio 12 2013 -) else ( -set CMAKE_GENERATOR=Visual Studio 12 2013 Win64 -) -set CMAKE_GENERATOR_TOOLSET=v120_xp -cmake -G "%CMAKE_GENERATOR%" -DCMAKE_INSTALL_PREFIX=%PREFIX% -DOPENSSL_ROOT_DIR=%PREFIX%\Library -DBUILD_SHARED_LIBS=ON -DBUILD_TESTING=OFF -DBUILD_EXAMPLES=OFF -DCMAKE_PREFIX_PATH=$PREFIX %SRC_DIR% -if errorlevel 1 exit 1 -REM Build step -cmake --build . -if errorlevel 1 exit 1 -REM Install step -cmake --build . --target install -if errorlevel 1 exit 1 \ No newline at end of file diff --git a/conda/libssh2/build.sh b/conda/libssh2/build.sh deleted file mode 100644 index 773dda78b..000000000 --- a/conda/libssh2/build.sh +++ /dev/null @@ -1,7 +0,0 @@ -#!/bin/bash - -mkdir build -cd build -cmake .. -DCMAKE_INSTALL_PREFIX=$PREFIX -DOPENSSL_ROOT_DIR=$PREFIX -DBUILD_SHARED_LIBS=ON -DBUILD_TESTING=OFF -DBUILD_EXAMPLES=OFF -DCMAKE_PREFIX_PATH=$PREFIX -make -j2 -make install diff --git a/conda/libssh2/meta.yaml b/conda/libssh2/meta.yaml deleted file mode 100644 index 28c0f59b6..000000000 --- a/conda/libssh2/meta.yaml +++ /dev/null @@ -1,23 +0,0 @@ -package: - name: libssh2 - version: 1.6.0 - -source: - git_url: https://github.com/libssh2/libssh2 - git_tag: libssh2-1.6.0 - -build: - number: 1 - -requirements: - build: - - system # [linux] - - cmake # [linux] - - openssl - run: - - openssl - -about: - home: http://www.libssh2.org/ - license: BSD - summary: 'libssh2 is a client-side C library implementing the SSH2 protocol' diff --git a/conda/lit/bld.bat b/conda/lit/bld.bat deleted file mode 100644 index c40a9bbef..000000000 --- a/conda/lit/bld.bat +++ /dev/null @@ -1,2 +0,0 @@ -"%PYTHON%" setup.py install -if errorlevel 1 exit 1 diff --git a/conda/lit/build.sh b/conda/lit/build.sh deleted file mode 100644 index 5a5aeeb48..000000000 --- a/conda/lit/build.sh +++ /dev/null @@ -1 +0,0 @@ -$PYTHON setup.py install diff --git a/conda/lit/meta.yaml b/conda/lit/meta.yaml deleted file mode 100644 index 14cf41555..000000000 --- a/conda/lit/meta.yaml +++ /dev/null @@ -1,27 +0,0 @@ -package: - name: lit - version: 0.4.1 - -source: - fn: lit-0.4.1.tar.gz - url: https://pypi.python.org/packages/source/l/lit/lit-0.4.1.tar.gz - md5: ea6f00470e1bf7ed9e4edcff0f650fe6 - -build: - number: 0 - -requirements: - build: - - python - - setuptools - - run: - - python - -test: - commands: - - lit --version - -about: - home: http://llvm.org/docs/CommandGuide/lit.html - license: MIT diff --git a/conda/llvmdev-or1k/bld.bat b/conda/llvmdev-or1k/bld.bat deleted file mode 100644 index 654b44d64..000000000 --- a/conda/llvmdev-or1k/bld.bat +++ /dev/null @@ -1,25 +0,0 @@ -mkdir build -cd build -set BUILD_CONFIG=Release -REM Configure step -if "%ARCH%"=="32" ( -set CMAKE_GENERATOR=Visual Studio 12 2013 -) else ( -set CMAKE_GENERATOR=Visual Studio 12 2013 Win64 -) -set CMAKE_GENERATOR_TOOLSET=v120_xp -@rem Reduce build times and package size by removing unused stuff -set CMAKE_CUSTOM=-DLLVM_TARGETS_TO_BUILD="OR1K;X86" -DLLVM_INCLUDE_TESTS=OFF ^ --DLLVM_INCLUDE_TOOLS=OFF -DLLVM_INCLUDE_UTILS=OFF ^ --DLLVM_INCLUDE_DOCS=OFF -DLLVM_INCLUDE_EXAMPLES=OFF ^ --DLLVM_ENABLE_ASSERTIONS=ON -cmake -G "%CMAKE_GENERATOR%" -T "%CMAKE_GENERATOR_TOOLSET%" ^ --DCMAKE_BUILD_TYPE="%BUILD_CONFIG%" -DCMAKE_PREFIX_PATH=%LIBRARY_PREFIX% ^ --DCMAKE_INSTALL_PREFIX:PATH=%LIBRARY_PREFIX% %CMAKE_CUSTOM% %SRC_DIR% -if errorlevel 1 exit 1 -REM Build step -cmake --build . --config "%BUILD_CONFIG%" -if errorlevel 1 exit 1 -REM Install step -cmake --build . --config "%BUILD_CONFIG%" --target install -if errorlevel 1 exit 1 diff --git a/conda/llvmdev-or1k/build.sh b/conda/llvmdev-or1k/build.sh deleted file mode 100644 index 391f592cc..000000000 --- a/conda/llvmdev-or1k/build.sh +++ /dev/null @@ -1,10 +0,0 @@ -#!/bin/bash - -cd tools -git clone https://github.com/openrisc/clang-or1k clang -cd .. -mkdir build -cd build -cmake .. -DCMAKE_INSTALL_PREFIX=$PREFIX -DLLVM_TARGETS_TO_BUILD="OR1K;X86" -DCMAKE_BUILD_TYPE=Rel -DLLVM_ENABLE_ASSERTIONS=ON -make -j2 -make install diff --git a/conda/llvmdev-or1k/meta.yaml b/conda/llvmdev-or1k/meta.yaml deleted file mode 100644 index 09ca7046a..000000000 --- a/conda/llvmdev-or1k/meta.yaml +++ /dev/null @@ -1,22 +0,0 @@ -package: - name: llvmdev-or1k - version: "3.5.0" - -source: - git_url: https://github.com/openrisc/llvm-or1k - git_tag: master - -build: - number: 5 - -requirements: - build: - - system [linux] - - cmake [linux] - run: - - system [linux] - -about: - home: http://llvm.org/ - license: Open Source (http://llvm.org/releases/3.5.0/LICENSE.TXT) - summary: Development headers and libraries for LLVM diff --git a/conda/llvmlite-artiq/bld.bat b/conda/llvmlite-artiq/bld.bat deleted file mode 100644 index 8b58512c1..000000000 --- a/conda/llvmlite-artiq/bld.bat +++ /dev/null @@ -1,8 +0,0 @@ -@rem Let CMake know about the LLVM install path, for find_package() -set CMAKE_PREFIX_PATH=%LIBRARY_PREFIX% - -@rem Ensure there are no build leftovers (CMake can complain) -if exist ffi\build rmdir /S /Q ffi\build - -%PYTHON% setup.py install -if errorlevel 1 exit 1 diff --git a/conda/llvmlite-artiq/build.sh b/conda/llvmlite-artiq/build.sh deleted file mode 100755 index 8e25a1455..000000000 --- a/conda/llvmlite-artiq/build.sh +++ /dev/null @@ -1,3 +0,0 @@ -#!/bin/bash - -$PYTHON setup.py install diff --git a/conda/llvmlite-artiq/meta.yaml b/conda/llvmlite-artiq/meta.yaml deleted file mode 100644 index 45c964b0f..000000000 --- a/conda/llvmlite-artiq/meta.yaml +++ /dev/null @@ -1,27 +0,0 @@ -package: - name: llvmlite-artiq - version: "0.5.1" - -source: - git_url: https://github.com/m-labs/llvmlite - git_tag: artiq - -requirements: - build: - - python - - llvmdev-or1k - - setuptools - run: - - python - -build: - number: 5 - -test: - imports: - - llvmlite_artiq - - llvmlite_artiq.binding - -about: - home: https://pypi.python.org/pypi/llvmlite/ - license: BSD diff --git a/conda/prettytable/bld.bat b/conda/prettytable/bld.bat deleted file mode 100644 index 87b1481d7..000000000 --- a/conda/prettytable/bld.bat +++ /dev/null @@ -1,8 +0,0 @@ -"%PYTHON%" setup.py install -if errorlevel 1 exit 1 - -:: Add more build steps here, if they are necessary. - -:: See -:: http://docs.continuum.io/conda/build.html -:: for a list of environment variables that are set during the build process. diff --git a/conda/prettytable/build.sh b/conda/prettytable/build.sh deleted file mode 100644 index 8e25a1455..000000000 --- a/conda/prettytable/build.sh +++ /dev/null @@ -1,3 +0,0 @@ -#!/bin/bash - -$PYTHON setup.py install diff --git a/conda/prettytable/meta.yaml b/conda/prettytable/meta.yaml deleted file mode 100644 index ca359db92..000000000 --- a/conda/prettytable/meta.yaml +++ /dev/null @@ -1,26 +0,0 @@ -package: - name: prettytable - version: !!str 0.7.2 - -source: - fn: prettytable-0.7.2.tar.bz2 - url: https://pypi.python.org/packages/source/P/PrettyTable/prettytable-0.7.2.tar.bz2 - md5: 760dc900590ac3c46736167e09fa463a - -requirements: - build: - - python - - setuptools - - run: - - python - -test: - imports: - - prettytable - - -about: - home: http://code.google.com/p/prettytable/ - license: BSD License - summary: 'A simple Python library for easily displaying tabular data in a visually appealing ASCII table format.' diff --git a/conda/pydaqmx/bld.bat b/conda/pydaqmx/bld.bat deleted file mode 100644 index add2c3c60..000000000 --- a/conda/pydaqmx/bld.bat +++ /dev/null @@ -1,2 +0,0 @@ -"%PYTHON%" setup.py build -"%PYTHON%" setup.py install diff --git a/conda/pydaqmx/build.sh b/conda/pydaqmx/build.sh deleted file mode 100644 index f1d91245e..000000000 --- a/conda/pydaqmx/build.sh +++ /dev/null @@ -1,2 +0,0 @@ -$PYTHON setup.py build -$PYTHON setup.py install diff --git a/conda/pydaqmx/meta.yaml b/conda/pydaqmx/meta.yaml deleted file mode 100644 index 36f4636f9..000000000 --- a/conda/pydaqmx/meta.yaml +++ /dev/null @@ -1,22 +0,0 @@ -package: - name: pydaqmx - version: "1.3.1" - -source: - git_url: https://github.com/clade/pydaqmx - git_tag: master - -build: - number: 0 - -requirements: - build: - - python - - setuptools - run: - - python - -about: - home: http://pythonhosted.org/PyDAQmx/ - license: BSD - summary: PyDAQmx allows users to use data acquisition hardware from National Instruments with Python. It provides an interface between the NIDAQmx driver and Python. The package works on Windows and Linux.' diff --git a/conda/pyelftools/bld.bat b/conda/pyelftools/bld.bat deleted file mode 100644 index 39b5e1fee..000000000 --- a/conda/pyelftools/bld.bat +++ /dev/null @@ -1 +0,0 @@ -%PYTHON% setup.py install diff --git a/conda/pyelftools/build.sh b/conda/pyelftools/build.sh deleted file mode 100644 index 5a5aeeb48..000000000 --- a/conda/pyelftools/build.sh +++ /dev/null @@ -1 +0,0 @@ -$PYTHON setup.py install diff --git a/conda/pyelftools/meta.yaml b/conda/pyelftools/meta.yaml deleted file mode 100644 index f65b271dd..000000000 --- a/conda/pyelftools/meta.yaml +++ /dev/null @@ -1,26 +0,0 @@ -package: - name: pyelftools - version: 0.23 - -source: - git_url: https://github.com/eliben/pyelftools.git - git_tag: v0.23 - -build: - number: 0 - -requirements: - build: - - python - - setuptools - run: - - python - -test: - imports: - - elftools - -about: - home: https://github.com/eliben/pyelftools.git - license: Public domain - summary: 'Library for analyzing ELF files and DWARF debugging information' diff --git a/conda/pygit2/bld.bat b/conda/pygit2/bld.bat deleted file mode 100644 index 0b9010888..000000000 --- a/conda/pygit2/bld.bat +++ /dev/null @@ -1,3 +0,0 @@ -set LIBGIT2=%PREFIX% -set VS100COMNTOOLS=%VS120COMNTOOLS% -%PYTHON% setup.py install \ No newline at end of file diff --git a/conda/pygit2/build.sh b/conda/pygit2/build.sh deleted file mode 100644 index 833768d01..000000000 --- a/conda/pygit2/build.sh +++ /dev/null @@ -1,2 +0,0 @@ -export LIBGIT2=$PREFIX -$PYTHON setup.py install diff --git a/conda/pygit2/meta.yaml b/conda/pygit2/meta.yaml deleted file mode 100644 index fcc222f29..000000000 --- a/conda/pygit2/meta.yaml +++ /dev/null @@ -1,28 +0,0 @@ -package: - name: pygit2 - version: 0.22.1 - -source: - git_url: https://github.com/libgit2/pygit2 - git_tag: v0.22.1 - -build: - number: 1 - -requirements: - build: - - system # [linux] - - python - - libgit2 - - cffi >=0.8.1 - - pkgconfig # [linux] - run: - - system # [linux] - - python - - libgit2 - - cffi >=0.8.1 - -about: - home: http://www.pygit2.org/ - license: GPLv2 with a special Linking Exception - summary: 'Pygit2 is a set of Python bindings to the libgit2 shared library, libgit2 implements the core of Git.' diff --git a/conda/pyqtgraph/bld.bat b/conda/pyqtgraph/bld.bat deleted file mode 100644 index c40a9bbef..000000000 --- a/conda/pyqtgraph/bld.bat +++ /dev/null @@ -1,2 +0,0 @@ -"%PYTHON%" setup.py install -if errorlevel 1 exit 1 diff --git a/conda/pyqtgraph/build.sh b/conda/pyqtgraph/build.sh deleted file mode 100644 index 8e25a1455..000000000 --- a/conda/pyqtgraph/build.sh +++ /dev/null @@ -1,3 +0,0 @@ -#!/bin/bash - -$PYTHON setup.py install diff --git a/conda/pyqtgraph/meta.yaml b/conda/pyqtgraph/meta.yaml deleted file mode 100644 index f060acf0b..000000000 --- a/conda/pyqtgraph/meta.yaml +++ /dev/null @@ -1,27 +0,0 @@ -package: - name: pyqtgraph - version: 0.9.10.1036edf - -source: - git_url: https://github.com/pyqtgraph/pyqtgraph.git - git_rev: 1036edf - -requirements: - build: - - python - - setuptools - - numpy - - run: - - python - - numpy - - pyqt >=4.7 - -test: - imports: - - pyqtgraph - -about: - home: http://www.pyqtgraph.org - license: MIT License - summary: 'Scientific Graphics and GUI Library for Python' diff --git a/conda/pythonparser/bld.bat b/conda/pythonparser/bld.bat deleted file mode 100644 index c8c1ee0d1..000000000 --- a/conda/pythonparser/bld.bat +++ /dev/null @@ -1,2 +0,0 @@ -pip install regex -%PYTHON% setup.py install diff --git a/conda/pythonparser/build.sh b/conda/pythonparser/build.sh deleted file mode 100644 index 1e07e90fb..000000000 --- a/conda/pythonparser/build.sh +++ /dev/null @@ -1,2 +0,0 @@ -pip install regex -$PYTHON setup.py install diff --git a/conda/pythonparser/meta.yaml b/conda/pythonparser/meta.yaml deleted file mode 100644 index 6ef508192..000000000 --- a/conda/pythonparser/meta.yaml +++ /dev/null @@ -1,24 +0,0 @@ -package: - name: pythonparser - version: 0.0 - -source: - git_url: https://github.com/m-labs/pythonparser - git_tag: master - -build: - number: 0 - -requirements: - build: - - python - - setuptools - -test: - imports: - - pythonparser - -about: - home: http://m-labs.hk/pythonparser/ - license: BSD - summary: 'PythonParser is a Python parser written specifically for use in tooling. It parses source code into an AST that is a superset of Python’s built-in ast module, but returns precise location information for every token.' diff --git a/conda/quamash/bld.bat b/conda/quamash/bld.bat deleted file mode 100644 index 39b5e1fee..000000000 --- a/conda/quamash/bld.bat +++ /dev/null @@ -1 +0,0 @@ -%PYTHON% setup.py install diff --git a/conda/quamash/build.sh b/conda/quamash/build.sh deleted file mode 100644 index 8e25a1455..000000000 --- a/conda/quamash/build.sh +++ /dev/null @@ -1,3 +0,0 @@ -#!/bin/bash - -$PYTHON setup.py install diff --git a/conda/quamash/meta.yaml b/conda/quamash/meta.yaml deleted file mode 100644 index 724e9b674..000000000 --- a/conda/quamash/meta.yaml +++ /dev/null @@ -1,29 +0,0 @@ -package: - name: quamash - version: 0.5.1 - -source: - fn: Quamash-0.5.1.tar.gz - url: https://pypi.python.org/packages/source/Q/Quamash/Quamash-0.5.1.tar.gz#md5=c5fa317f615eafd492560771bc2caeca - md5: c5fa317f615eafd492560771bc2caeca - -build: - number: 0 - -requirements: - build: - - python - - setuptools - - pyqt 4.* - run: - - python - - pyqt 4.* - -test: - imports: - - quamash - -about: - home: https://github.com/harvimt/quamash - license: BSD - summary: 'Implementation of the PEP 3156 Event-Loop with Qt' diff --git a/conda/sphinx-argparse/bld.bat b/conda/sphinx-argparse/bld.bat deleted file mode 100644 index 39b5e1fee..000000000 --- a/conda/sphinx-argparse/bld.bat +++ /dev/null @@ -1 +0,0 @@ -%PYTHON% setup.py install diff --git a/conda/sphinx-argparse/build.sh b/conda/sphinx-argparse/build.sh deleted file mode 100644 index 5a5aeeb48..000000000 --- a/conda/sphinx-argparse/build.sh +++ /dev/null @@ -1 +0,0 @@ -$PYTHON setup.py install diff --git a/conda/sphinx-argparse/meta.yaml b/conda/sphinx-argparse/meta.yaml deleted file mode 100644 index 6ead92292..000000000 --- a/conda/sphinx-argparse/meta.yaml +++ /dev/null @@ -1,28 +0,0 @@ -package: - name: sphinx-argparse - version: 0.1.13 - -source: - fn: sphinx-argparse-0.1.13.tar.gz - url: https://pypi.python.org/packages/source/s/sphinx-argparse/sphinx-argparse-0.1.13.tar.gz - md5: 5ec84e75e1c4b2ae7ca5fb92a6abd738 - -build: - number: 0 - -requirements: - build: - - python - - setuptools - - sphinx - run: - - python - - sphinx - -test: - imports: - - sphinxarg - -about: - license: MIT - summary: 'Sphinx extension that automatically documents argparse commands and options' diff --git a/doc/manual/installing.rst b/doc/manual/installing.rst index 8cb1727a2..6d2498121 100644 --- a/doc/manual/installing.rst +++ b/doc/manual/installing.rst @@ -13,12 +13,12 @@ Installing using conda Installing Anaconda or Miniconda ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -* You can either install Anaconda (chose Python 3.5) from https://store.continuum.io/cshop/anaconda/ +* You can either install Anaconda (choose Python 3.5) from https://store.continuum.io/cshop/anaconda/ -* Or install the more minimalistic Miniconda (chose Python 3.5) from http://conda.pydata.org/miniconda.html +* Or install the more minimalistic Miniconda (choose Python 3.5) from http://conda.pydata.org/miniconda.html .. warning:: - If you are installing on Windows, chose the Windows 32-bit version regardless of whether you have + If you are installing on Windows, choose the Windows 32-bit version regardless of whether you have a 32-bit or 64-bit Windows. After installing either Anaconda or Miniconda, open a new terminal and make sure the following command works:: @@ -40,6 +40,7 @@ Installing the host side software For this, you need to add our binstar repository to your conda configuration:: + $ conda config --add channels http://conda.anaconda.org/m-labs/channel/main $ conda config --add channels http://conda.anaconda.org/m-labs/channel/dev Then you can install the ARTIQ package, it will pull all the necessary dependencies:: From c0796249b3497dd21b7b1f14120f8e9700975b5b Mon Sep 17 00:00:00 2001 From: Sebastien Bourdeauducq Date: Fri, 16 Oct 2015 18:34:37 +0800 Subject: [PATCH 07/73] protocols/logging: document, take level numbers for consistency with master publish --- artiq/protocols/logging.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/artiq/protocols/logging.py b/artiq/protocols/logging.py index 42cca1572..6c3fdded3 100644 --- a/artiq/protocols/logging.py +++ b/artiq/protocols/logging.py @@ -38,6 +38,11 @@ _init_string = b"ARTIQ logging\n" class Server(AsyncioServer): + """Remote logging TCP server. + + Takes one log entry per line, in the format: + source:levelno:name:message + """ async def _handle_connection_cr(self, reader, writer): try: line = await reader.readline() @@ -56,10 +61,10 @@ class Server(AsyncioServer): linesplit = line.split(":", 4) if len(linesplit) != 4: return - source, levelname, name, message = linesplit + source, level, name, message = linesplit try: - level = _name_to_level[levelname] - except KeyError: + level = int(level) + except: return log_with_name(name, level, message, extra={"source": source}) From cee8f288de87d44b3c4d94a2eccf1cf0b4edf2db Mon Sep 17 00:00:00 2001 From: Sebastien Bourdeauducq Date: Fri, 16 Oct 2015 18:35:02 +0800 Subject: [PATCH 08/73] protocols/logging: add LogForwarder --- artiq/protocols/logging.py | 37 +++++++++++++++++++++++++++++++++++++ 1 file changed, 37 insertions(+) diff --git a/artiq/protocols/logging.py b/artiq/protocols/logging.py index 6c3fdded3..44376da7c 100644 --- a/artiq/protocols/logging.py +++ b/artiq/protocols/logging.py @@ -2,6 +2,7 @@ import asyncio import logging from artiq.protocols.asyncio_server import AsyncioServer +from artiq.tools import TaskObject _fwd_logger = logging.getLogger("fwd") @@ -70,3 +71,39 @@ class Server(AsyncioServer): extra={"source": source}) finally: writer.close() + + +class LogForwarder(logging.Handler, TaskObject): + def __init__(self, host, port, reconnect_timer=5.0, queue_size=1000, + **kwargs): + logging.Handler.__init__(self, **kwargs) + self.host = host + self.port = port + self.setFormatter(logging.Formatter( + "%(source)s:%(levelno)d:%(name)s:%(message)s")) + self._queue = asyncio.Queue(queue_size) + self.reconnect_timer = reconnect_timer + + def emit(self, record): + message = self.format(record) + try: + self._queue.put_nowait(message) + except asyncio.QueueFull: + pass + + async def _do(self): + while True: + try: + reader, writer = await asyncio.open_connection(self.host, + self.port) + writer.write(_init_string) + while True: + message = await self._queue.get() + "\n" + writer.write(message.encode()) + await writer.drain() + except asyncio.CancelledError: + return + except: + await asyncio.sleep(self.reconnect_timer) + finally: + writer.close() From 9bb94f0f59b06082a73290c5ef40ba715e7faa23 Mon Sep 17 00:00:00 2001 From: Sebastien Bourdeauducq Date: Fri, 16 Oct 2015 18:35:30 +0800 Subject: [PATCH 09/73] master/log: move formatter into LogBufferHandler --- artiq/master/log.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/artiq/master/log.py b/artiq/master/log.py index ca8e1fd71..f588e839e 100644 --- a/artiq/master/log.py +++ b/artiq/master/log.py @@ -20,6 +20,7 @@ class LogBufferHandler(logging.Handler): def __init__(self, log_buffer, *args, **kwargs): logging.Handler.__init__(self, *args, **kwargs) self.log_buffer = log_buffer + self.setFormatter(logging.Formatter("%(name)s:%(message)s")) def emit(self, record): message = self.format(record) @@ -87,7 +88,6 @@ def init_log(args): log_buffer = LogBuffer(1000) buffer_handler = LogBufferHandler(log_buffer) - buffer_handler.setFormatter(logging.Formatter("%(name)s:%(message)s")) handlers.append(buffer_handler) for handler in handlers: From 9e96a687e22e2182d8fbf7caa945d17b5ff737c1 Mon Sep 17 00:00:00 2001 From: Sebastien Bourdeauducq Date: Fri, 16 Oct 2015 18:35:58 +0800 Subject: [PATCH 10/73] ctlmgr: forward log to master --- artiq/frontend/artiq_ctlmgr.py | 80 ++++++++++++++++++++++++---------- 1 file changed, 56 insertions(+), 24 deletions(-) diff --git a/artiq/frontend/artiq_ctlmgr.py b/artiq/frontend/artiq_ctlmgr.py index fb25519b1..0a3149b78 100755 --- a/artiq/frontend/artiq_ctlmgr.py +++ b/artiq/frontend/artiq_ctlmgr.py @@ -10,34 +10,13 @@ import socket from artiq.protocols.sync_struct import Subscriber from artiq.protocols.pc_rpc import AsyncioClient, Server -from artiq.tools import verbosity_args, init_logger +from artiq.protocols.logging import LogForwarder from artiq.tools import TaskObject, Condition logger = logging.getLogger(__name__) -def get_argparser(): - parser = argparse.ArgumentParser(description="ARTIQ controller manager") - verbosity_args(parser) - parser.add_argument( - "-s", "--server", default="::1", - help="hostname or IP of the master to connect to") - parser.add_argument( - "--port", default=3250, type=int, - help="TCP port to use to connect to the master") - parser.add_argument( - "--retry-master", default=5.0, type=float, - help="retry timer for reconnecting to master") - parser.add_argument( - "--bind", default="::1", - help="hostname or IP address to bind to") - parser.add_argument( - "--bind-port", default=3249, type=int, - help="TCP port to listen to for control (default: %(default)d)") - return parser - - class Controller: def __init__(self, name, ddb_entry): self.name = name @@ -252,9 +231,54 @@ class ControllerManager(TaskObject): self.controller_db.current_controllers.active[k].retry_now.notify() +def get_argparser(): + parser = argparse.ArgumentParser(description="ARTIQ controller manager") + + group = parser.add_argument_group("verbosity") + group.add_argument("-v", "--verbose", default=0, action="count", + help="increase logging level") + group.add_argument("-q", "--quiet", default=0, action="count", + help="decrease logging level") + + parser.add_argument( + "-s", "--server", default="::1", + help="hostname or IP of the master to connect to") + parser.add_argument( + "--port-notify", default=3250, type=int, + help="TCP port to connect to for notifications") + parser.add_argument( + "--port-logging", default=1066, type=int, + help="TCP port to connect to for logging") + parser.add_argument( + "--retry-master", default=5.0, type=float, + help="retry timer for reconnecting to master") + parser.add_argument( + "--bind", default="::1", + help="hostname or IP address to bind to") + parser.add_argument( + "--bind-port", default=3249, type=int, + help="TCP port to listen to for control (default: %(default)d)") + return parser + + +class SourceAdder: + def filter(self, record): + if not hasattr(record, "source"): + record.source = "ctlmgr" + return True + + def main(): args = get_argparser().parse_args() - init_logger(args) + + root_logger = logging.getLogger() + root_logger.setLevel(logging.WARNING + args.quiet*10 - args.verbose*10) + source_adder = SourceAdder() + console_handler = logging.StreamHandler() + console_handler.setFormatter(logging.Formatter( + "%(levelname)s:%(source)s:%(name)s:%(message)s")) + console_handler.addFilter(source_adder) + root_logger.addHandler(console_handler) if os.name == "nt": loop = asyncio.ProactorEventLoop() @@ -263,7 +287,15 @@ def main(): loop = asyncio.get_event_loop() atexit.register(lambda: loop.close()) - ctlmgr = ControllerManager(args.server, args.port, args.retry_master) + logfwd = LogForwarder(args.server, args.port_logging, + args.retry_master) + logfwd.addFilter(source_adder) + root_logger.addHandler(logfwd) + logfwd.start() + atexit.register(lambda: loop.run_until_complete(logfwd.stop())) + + ctlmgr = ControllerManager(args.server, args.port_notify, + args.retry_master) ctlmgr.start() atexit.register(lambda: loop.run_until_complete(ctlmgr.stop())) From 786dc14057e994b7e9a3598a2da954af5de85997 Mon Sep 17 00:00:00 2001 From: Sebastien Bourdeauducq Date: Fri, 16 Oct 2015 20:07:31 +0800 Subject: [PATCH 11/73] protocols/logging: fix split, warn on format errors --- artiq/protocols/logging.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/artiq/protocols/logging.py b/artiq/protocols/logging.py index 44376da7c..ae2d5a0e2 100644 --- a/artiq/protocols/logging.py +++ b/artiq/protocols/logging.py @@ -5,6 +5,7 @@ from artiq.protocols.asyncio_server import AsyncioServer from artiq.tools import TaskObject +logger = logging.getLogger(__name__) _fwd_logger = logging.getLogger("fwd") @@ -59,13 +60,17 @@ class Server(AsyncioServer): except: return line = line[:-1] - linesplit = line.split(":", 4) + linesplit = line.split(":", 3) if len(linesplit) != 4: + logger.warning("received improperly formatted message, " + "dropping connection") return source, level, name, message = linesplit try: level = int(level) except: + logger.warning("received improperly formatted level, " + "dropping connection") return log_with_name(name, level, message, extra={"source": source}) From f332c1d3cc8482c7ddbf56f5fb776a19e7917196 Mon Sep 17 00:00:00 2001 From: Sebastien Bourdeauducq Date: Fri, 16 Oct 2015 20:08:11 +0800 Subject: [PATCH 12/73] ctlmgr: forward controller logs --- artiq/frontend/artiq_ctlmgr.py | 45 ++++++++++++++++++++++++---------- artiq/master/log.py | 20 +++------------ artiq/protocols/logging.py | 16 ++++++++++++ 3 files changed, 51 insertions(+), 30 deletions(-) diff --git a/artiq/frontend/artiq_ctlmgr.py b/artiq/frontend/artiq_ctlmgr.py index 0a3149b78..b4e5dacc2 100755 --- a/artiq/frontend/artiq_ctlmgr.py +++ b/artiq/frontend/artiq_ctlmgr.py @@ -5,12 +5,15 @@ import atexit import argparse import os import logging +import subprocess import shlex import socket from artiq.protocols.sync_struct import Subscriber from artiq.protocols.pc_rpc import AsyncioClient, Server -from artiq.protocols.logging import LogForwarder +from artiq.protocols.logging import (LogForwarder, + parse_log_message, log_with_name, + SourceFilter) from artiq.tools import TaskObject, Condition @@ -75,6 +78,23 @@ class Controller: else: break + async def forward_logs(self, stream): + source = "controller({})".format(self.name) + while True: + try: + entry = (await stream.readline()) + if not entry: + break + entry = entry[:-1] + level, name, message = parse_log_message(entry.decode()) + log_with_name(name, level, message, extra={"source": source}) + except: + logger.debug("exception in log forwarding", exc_info=True) + break + logger.debug("stopped log forwarding of stream %s of %s", + stream, self.name) + + async def launcher(self): try: while True: @@ -82,7 +102,12 @@ class Controller: self.name, self.command) try: self.process = await asyncio.create_subprocess_exec( - *shlex.split(self.command)) + *shlex.split(self.command), + stdout=subprocess.PIPE, stderr=subprocess.PIPE) + asyncio.ensure_future(self.forward_logs( + self.process.stdout)) + asyncio.ensure_future(self.forward_logs( + self.process.stderr)) await self._wait_and_ping() except FileNotFoundError: logger.warning("Controller %s failed to start", self.name) @@ -236,9 +261,9 @@ def get_argparser(): group = parser.add_argument_group("verbosity") group.add_argument("-v", "--verbose", default=0, action="count", - help="increase logging level") + help="increase logging level of the manager process") group.add_argument("-q", "--quiet", default=0, action="count", - help="decrease logging level") + help="decrease logging level of the manager process") parser.add_argument( "-s", "--server", default="::1", @@ -261,19 +286,13 @@ def get_argparser(): return parser -class SourceAdder: - def filter(self, record): - if not hasattr(record, "source"): - record.source = "ctlmgr" - return True - - def main(): args = get_argparser().parse_args() root_logger = logging.getLogger() - root_logger.setLevel(logging.WARNING + args.quiet*10 - args.verbose*10) - source_adder = SourceAdder() + root_logger.setLevel(logging.NOTSET) + source_adder = SourceFilter(logging.WARNING + args.quiet*10 - args.verbose*10, + "ctlmgr") console_handler = logging.StreamHandler() console_handler.setFormatter(logging.Formatter( "%(levelname)s:%(source)s:%(name)s:%(message)s")) diff --git a/artiq/master/log.py b/artiq/master/log.py index f588e839e..f8e6939ef 100644 --- a/artiq/master/log.py +++ b/artiq/master/log.py @@ -2,7 +2,7 @@ import logging import logging.handlers from artiq.protocols.sync_struct import Notifier -from artiq.protocols.logging import parse_log_message, log_with_name +from artiq.protocols.logging import parse_log_message, log_with_name, SourceFilter class LogBuffer: @@ -34,21 +34,6 @@ def log_worker(rid, message): log_worker.worker_pass_rid = True -class SourceFilter: - def __init__(self, master_level): - self.master_level = master_level - - def filter(self, record): - if not hasattr(record, "source"): - record.source = "master" - if record.source == "master": - return record.levelno >= self.master_level - else: - # log messages that are forwarded from a source have already - # been filtered, and may have a level below the master level. - return True - - def log_args(parser): group = parser.add_argument_group("logging") group.add_argument("-v", "--verbose", default=0, action="count", @@ -69,7 +54,8 @@ def log_args(parser): def init_log(args): root_logger = logging.getLogger() root_logger.setLevel(logging.NOTSET) # we use our custom filter only - flt = SourceFilter(logging.WARNING + args.quiet*10 - args.verbose*10) + flt = SourceFilter(logging.WARNING + args.quiet*10 - args.verbose*10, + "master") full_fmt = logging.Formatter( "%(levelname)s:%(source)s:%(name)s:%(message)s") diff --git a/artiq/protocols/logging.py b/artiq/protocols/logging.py index ae2d5a0e2..61b9c07fb 100644 --- a/artiq/protocols/logging.py +++ b/artiq/protocols/logging.py @@ -78,6 +78,22 @@ class Server(AsyncioServer): writer.close() +class SourceFilter: + def __init__(self, local_level, local_source): + self.local_level = local_level + self.local_source = local_source + + def filter(self, record): + if not hasattr(record, "source"): + record.source = self.local_source + if record.source == self.local_source: + return record.levelno >= self.local_level + else: + # log messages that are forwarded from a source have already + # been filtered, and may have a level below the local level. + return True + + class LogForwarder(logging.Handler, TaskObject): def __init__(self, host, port, reconnect_timer=5.0, queue_size=1000, **kwargs): From 6c856025cc1866e6f18db2849e79336ff08efdce Mon Sep 17 00:00:00 2001 From: Sebastien Bourdeauducq Date: Fri, 16 Oct 2015 21:28:39 +0800 Subject: [PATCH 13/73] protocol/logging: workaround for asyncio's inability to detect connection closes on writes --- artiq/protocols/logging.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/artiq/protocols/logging.py b/artiq/protocols/logging.py index 61b9c07fb..6333942da 100644 --- a/artiq/protocols/logging.py +++ b/artiq/protocols/logging.py @@ -117,11 +117,19 @@ class LogForwarder(logging.Handler, TaskObject): try: reader, writer = await asyncio.open_connection(self.host, self.port) + detect_close = asyncio.ensure_future(reader.read(1)) writer.write(_init_string) while True: message = await self._queue.get() + "\n" writer.write(message.encode()) await writer.drain() + # HACK: detect connection termination through the completion + # of a read operation. For some reason, write/drain operations + # on a closed socket do not raise exceptions, but print + # "asyncio:socket.send() raised exception." + if detect_close.done(): + await asyncio.sleep(self.reconnect_timer) + break except asyncio.CancelledError: return except: From f6fd7ecef22509f8c64025bfa689595d7eefeb61 Mon Sep 17 00:00:00 2001 From: Sebastien Bourdeauducq Date: Sat, 17 Oct 2015 10:21:03 +0800 Subject: [PATCH 14/73] logging: handle newlines in messages --- artiq/master/log.py | 4 +++- artiq/protocols/logging.py | 12 +++++++----- 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/artiq/master/log.py b/artiq/master/log.py index f8e6939ef..0211f8584 100644 --- a/artiq/master/log.py +++ b/artiq/master/log.py @@ -24,7 +24,9 @@ class LogBufferHandler(logging.Handler): def emit(self, record): message = self.format(record) - self.log_buffer.log(record.levelno, record.source, record.created, message) + for part in message.split("\n"): + self.log_buffer.log(record.levelno, record.source, record.created, + part) def log_worker(rid, message): diff --git a/artiq/protocols/logging.py b/artiq/protocols/logging.py index 6333942da..f65212550 100644 --- a/artiq/protocols/logging.py +++ b/artiq/protocols/logging.py @@ -101,16 +101,18 @@ class LogForwarder(logging.Handler, TaskObject): self.host = host self.port = port self.setFormatter(logging.Formatter( - "%(source)s:%(levelno)d:%(name)s:%(message)s")) + "%(name)s:%(message)s")) self._queue = asyncio.Queue(queue_size) self.reconnect_timer = reconnect_timer def emit(self, record): message = self.format(record) - try: - self._queue.put_nowait(message) - except asyncio.QueueFull: - pass + for part in message.split("\n"): + part = "{}:{}:{}".format(record.source, record.levelno, part) + try: + self._queue.put_nowait(part) + except asyncio.QueueFull: + break async def _do(self): while True: From 60ed88f3d99f6eddcd381e7ba1ec1783a038d67e Mon Sep 17 00:00:00 2001 From: Sebastien Bourdeauducq Date: Sat, 17 Oct 2015 10:22:05 +0800 Subject: [PATCH 15/73] worker: separate stdout/stderr for forwarding --- artiq/master/worker_impl.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/artiq/master/worker_impl.py b/artiq/master/worker_impl.py index 8b22d3acf..be0c70315 100644 --- a/artiq/master/worker_impl.py +++ b/artiq/master/worker_impl.py @@ -158,7 +158,8 @@ def examine(device_mgr, dataset_mgr, file): def main(): - sys.stdout = sys.stderr = LogForwarder() + sys.stdout = LogForwarder() + sys.stderr = LogForwarder() start_time = None rid = None From 04a049b93db3be14b360f85ef624d709a8b2193c Mon Sep 17 00:00:00 2001 From: Sebastien Bourdeauducq Date: Sat, 17 Oct 2015 10:26:54 +0800 Subject: [PATCH 16/73] master: timestamp logfiles --- artiq/master/log.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/artiq/master/log.py b/artiq/master/log.py index 0211f8584..ae87e015d 100644 --- a/artiq/master/log.py +++ b/artiq/master/log.py @@ -58,12 +58,10 @@ def init_log(args): root_logger.setLevel(logging.NOTSET) # we use our custom filter only flt = SourceFilter(logging.WARNING + args.quiet*10 - args.verbose*10, "master") - full_fmt = logging.Formatter( - "%(levelname)s:%(source)s:%(name)s:%(message)s") - handlers = [] console_handler = logging.StreamHandler() - console_handler.setFormatter(full_fmt) + console_handler.setFormatter(logging.Formatter( + "%(levelname)s:%(source)s:%(name)s:%(message)s")) handlers.append(console_handler) if args.log_file: @@ -71,7 +69,8 @@ def init_log(args): args.log_file, maxBytes=args.log_max_size*1024, backupCount=args.log_backup_count) - file_handler.setFormatter(full_fmt) + file_handler.setFormatter(logging.Formatter( + "%(asctime)s %(levelname)s:%(source)s:%(name)s:%(message)s")) handlers.append(file_handler) log_buffer = LogBuffer(1000) From 797bf9830c6bc40ef74df30baedfdea0658826a6 Mon Sep 17 00:00:00 2001 From: Sebastien Bourdeauducq Date: Sat, 17 Oct 2015 10:39:21 +0800 Subject: [PATCH 17/73] language: add basic logging initializer for experiments --- artiq/language/__init__.py | 4 +++- artiq/language/logging.py | 23 ++++++++++++++++++++ examples/master/repository/arguments_demo.py | 15 ++++++------- 3 files changed, 33 insertions(+), 9 deletions(-) create mode 100644 artiq/language/logging.py diff --git a/artiq/language/__init__.py b/artiq/language/__init__.py index 39edc6164..c7be4eb9b 100644 --- a/artiq/language/__init__.py +++ b/artiq/language/__init__.py @@ -1,10 +1,11 @@ # Copyright (C) 2014, 2015 Robert Jordens -from artiq.language import core, environment, units, scan +from artiq.language import core, environment, units, scan, logging from artiq.language.core import * from artiq.language.environment import * from artiq.language.units import * from artiq.language.scan import * +from artiq.language.logging import * __all__ = [] @@ -12,3 +13,4 @@ __all__.extend(core.__all__) __all__.extend(environment.__all__) __all__.extend(units.__all__) __all__.extend(scan.__all__) +__all__.extend(logging.__all__) diff --git a/artiq/language/logging.py b/artiq/language/logging.py new file mode 100644 index 000000000..c64a3dbf6 --- /dev/null +++ b/artiq/language/logging.py @@ -0,0 +1,23 @@ +from artiq.language.environment import * + +import logging + + +__all__ = ["LogExperiment"] + + +class LogExperiment: + def init_logger(self): + """Call this from build() to add a logging level enumeration + widget, initialize logging globally, and create a logger. + + Your class must also derive from ``HasEnvironment`` (or + ``EnvExperiment``). + + The created logger is called ``self.logger``.""" + level = self.get_argument("log_level", EnumerationValue( + ["DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL"])) + + if level is not None: + logging.basicConfig(level=getattr(logging, level)) + self.logger = logging.getLogger(self.__class__.__name__) diff --git a/examples/master/repository/arguments_demo.py b/examples/master/repository/arguments_demo.py index ffb6e2ac2..a0114c211 100644 --- a/examples/master/repository/arguments_demo.py +++ b/examples/master/repository/arguments_demo.py @@ -1,5 +1,3 @@ -import logging - from artiq import * @@ -35,8 +33,10 @@ class SubComponent2(HasEnvironment): print(self.sc2_enum) -class ArgumentsDemo(EnvExperiment): +class ArgumentsDemo(EnvExperiment, LogExperiment): def build(self): + self.init_logger() + self.setattr_argument("free_value", FreeValue(None)) self.setattr_argument("number", NumberValue(42e-6, unit="s", scale=1e-6, @@ -53,11 +53,10 @@ class ArgumentsDemo(EnvExperiment): self.sc2 = SubComponent2(parent=self) def run(self): - logging.basicConfig(level=logging.DEBUG) - logging.error("logging test: error") - logging.warning("logging test: warning") - logging.info("logging test: info") - logging.debug("logging test: debug") + self.logger.error("logging test: error") + self.logger.warning("logging test: warning") + self.logger.info("logging test: info") + self.logger.debug("logging test: debug") print(self.free_value) print(self.boolean) From 5899825eef17bb3bee3b506d43f6135a0249467f Mon Sep 17 00:00:00 2001 From: Sebastien Bourdeauducq Date: Sat, 17 Oct 2015 10:41:52 +0800 Subject: [PATCH 18/73] import order --- artiq/language/logging.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/artiq/language/logging.py b/artiq/language/logging.py index c64a3dbf6..067d9864e 100644 --- a/artiq/language/logging.py +++ b/artiq/language/logging.py @@ -1,7 +1,7 @@ -from artiq.language.environment import * - import logging +from artiq.language.environment import * + __all__ = ["LogExperiment"] From 7f5e264971715c6e9fc2baf3d12bc71ed6fa07b2 Mon Sep 17 00:00:00 2001 From: Sebastien Bourdeauducq Date: Sat, 17 Oct 2015 10:58:15 +0800 Subject: [PATCH 19/73] ctlmgr: add hostname to log source --- artiq/frontend/artiq_ctlmgr.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/artiq/frontend/artiq_ctlmgr.py b/artiq/frontend/artiq_ctlmgr.py index b4e5dacc2..6e2827d35 100755 --- a/artiq/frontend/artiq_ctlmgr.py +++ b/artiq/frontend/artiq_ctlmgr.py @@ -8,6 +8,7 @@ import logging import subprocess import shlex import socket +import platform from artiq.protocols.sync_struct import Subscriber from artiq.protocols.pc_rpc import AsyncioClient, Server @@ -292,7 +293,7 @@ def main(): root_logger = logging.getLogger() root_logger.setLevel(logging.NOTSET) source_adder = SourceFilter(logging.WARNING + args.quiet*10 - args.verbose*10, - "ctlmgr") + "ctlmgr({})".format(platform.node())) console_handler = logging.StreamHandler() console_handler.setFormatter(logging.Formatter( "%(levelname)s:%(source)s:%(name)s:%(message)s")) From da83212c79576734bf80fc3bfc0ae9b14057c06a Mon Sep 17 00:00:00 2001 From: Sebastien Bourdeauducq Date: Sat, 17 Oct 2015 19:11:17 +0800 Subject: [PATCH 20/73] gui: autodetect whether QSortFilterProxyModel is in QtCore or QtGui --- artiq/gui/datasets.py | 7 ++++++- artiq/gui/log.py | 9 +++++++-- 2 files changed, 13 insertions(+), 3 deletions(-) diff --git a/artiq/gui/datasets.py b/artiq/gui/datasets.py index 3d547902b..4af8a59ac 100644 --- a/artiq/gui/datasets.py +++ b/artiq/gui/datasets.py @@ -12,6 +12,11 @@ from artiq.tools import short_format from artiq.gui.tools import DictSyncModel from artiq.gui.displays import * +try: + QSortFilterProxyModel = QtCore.QSortFilterProxyModel +except AttributeError: + QSortFilterProxyModel = QtGui.QSortFilterProxyModel + logger = logging.getLogger(__name__) @@ -89,7 +94,7 @@ class DatasetsDock(dockarea.Dock): def init_datasets_model(self, init): self.table_model = DatasetsModel(self.table, init) - self.table_model_filter = QtCore.QSortFilterProxyModel() + self.table_model_filter = QSortFilterProxyModel() self.table_model_filter.setSourceModel(self.table_model) self.table.setModel(self.table_model_filter) return self.table_model diff --git a/artiq/gui/log.py b/artiq/gui/log.py index bbb3ad2b0..b9c75df05 100644 --- a/artiq/gui/log.py +++ b/artiq/gui/log.py @@ -8,6 +8,11 @@ from pyqtgraph import dockarea, LayoutWidget from artiq.protocols.sync_struct import Subscriber from artiq.gui.tools import ListSyncModel +try: + QSortFilterProxyModel = QtCore.QSortFilterProxyModel +except AttributeError: + QSortFilterProxyModel = QtGui.QSortFilterProxyModel + def _level_to_name(level): if level >= logging.CRITICAL: @@ -67,9 +72,9 @@ class _LogModel(ListSyncModel): return v[3] -class _LevelFilterProxyModel(QtCore.QSortFilterProxyModel): +class _LevelFilterProxyModel(QSortFilterProxyModel): def __init__(self, min_level): - QtCore.QSortFilterProxyModel.__init__(self) + QSortFilterProxyModel.__init__(self) self.min_level = min_level def filterAcceptsRow(self, sourceRow, sourceParent): From b9c1d3ef1298fcb3963310fefc9cee0d3034d36e Mon Sep 17 00:00:00 2001 From: Sebastien Bourdeauducq Date: Sun, 18 Oct 2015 00:52:16 +0800 Subject: [PATCH 21/73] language, gui: do not automatically insert scale prefixes --- artiq/gui/explorer.py | 7 +++---- artiq/gui/scan.py | 18 ++++++++---------- artiq/gui/tools.py | 18 ------------------ artiq/language/environment.py | 3 +-- artiq/language/scan.py | 3 +-- examples/master/repository/arguments_demo.py | 4 ++-- 6 files changed, 15 insertions(+), 38 deletions(-) diff --git a/artiq/gui/explorer.py b/artiq/gui/explorer.py index 75ea4f338..76482a94d 100644 --- a/artiq/gui/explorer.py +++ b/artiq/gui/explorer.py @@ -6,7 +6,7 @@ from pyqtgraph import LayoutWidget from artiq.protocols.sync_struct import Subscriber from artiq.protocols import pyon -from artiq.gui.tools import si_prefix, DictSyncModel +from artiq.gui.tools import DictSyncModel from artiq.gui.scan import ScanController @@ -85,9 +85,8 @@ class _NumberEntry(QtGui.QDoubleSpinBox): self.setMaximum(procdesc["max"]/self.scale) else: self.setMaximum(float("inf")) - suffix = si_prefix(self.scale) + procdesc["unit"] - if suffix: - self.setSuffix(" " + suffix) + if procdesc["unit"]: + self.setSuffix(" " + procdesc["unit"]) if "default" in procdesc: self.set_argument_value(procdesc["default"]) diff --git a/artiq/gui/scan.py b/artiq/gui/scan.py index 7ed69ed46..de1d281b2 100644 --- a/artiq/gui/scan.py +++ b/artiq/gui/scan.py @@ -1,11 +1,9 @@ from quamash import QtGui from pyqtgraph import LayoutWidget -from artiq.gui.tools import si_prefix - class _Range(LayoutWidget): - def __init__(self, global_min, global_max, global_step, suffix, scale, ndecimals): + def __init__(self, global_min, global_max, global_step, unit, scale, ndecimals): LayoutWidget.__init__(self) self.scale = scale @@ -21,8 +19,8 @@ class _Range(LayoutWidget): spinbox.setMaximum(float("inf")) if global_step is not None: spinbox.setSingleStep(global_step/self.scale) - if suffix: - spinbox.setSuffix(" " + suffix) + if unit: + spinbox.setSuffix(" " + unit) self.addWidget(QtGui.QLabel("Min:"), 0, 0) self.min = QtGui.QDoubleSpinBox() @@ -68,7 +66,7 @@ class ScanController(LayoutWidget): gmin, gmax = procdesc["global_min"], procdesc["global_max"] gstep = procdesc["global_step"] - suffix = si_prefix(self.scale) + procdesc["unit"] + unit = procdesc["unit"] ndecimals = procdesc["ndecimals"] self.v_noscan = QtGui.QDoubleSpinBox() @@ -82,17 +80,17 @@ class ScanController(LayoutWidget): else: self.v_noscan.setMaximum(float("inf")) self.v_noscan.setSingleStep(gstep/self.scale) - if suffix: - self.v_noscan.setSuffix(" " + suffix) + if unit: + self.v_noscan.setSuffix(" " + unit) self.v_noscan_gr = LayoutWidget() self.v_noscan_gr.addWidget(QtGui.QLabel("Value:"), 0, 0) self.v_noscan_gr.addWidget(self.v_noscan, 0, 1) self.stack.addWidget(self.v_noscan_gr) - self.v_linear = _Range(gmin, gmax, gstep, suffix, self.scale, ndecimals) + self.v_linear = _Range(gmin, gmax, gstep, unit, self.scale, ndecimals) self.stack.addWidget(self.v_linear) - self.v_random = _Range(gmin, gmax, gstep, suffix, self.scale, ndecimals) + self.v_random = _Range(gmin, gmax, gstep, unit, self.scale, ndecimals) self.stack.addWidget(self.v_random) self.v_explicit = QtGui.QLineEdit() diff --git a/artiq/gui/tools.py b/artiq/gui/tools.py index 4899b1173..67c7efb42 100644 --- a/artiq/gui/tools.py +++ b/artiq/gui/tools.py @@ -1,22 +1,4 @@ from quamash import QtCore -import numpy as np - - -def si_prefix(scale): - try: - return { - 1e-12: "p", - 1e-9: "n", - 1e-6: "u", - 1e-3: "m", - 1.0: "", - 1e3: "k", - 1e6: "M", - 1e9: "G", - 1e12: "T" - }[scale] - except KeyError: - return "[x{}]".format(scale) class _SyncSubstruct: diff --git a/artiq/language/environment.py b/artiq/language/environment.py index 8c8f11b82..bd7cea725 100644 --- a/artiq/language/environment.py +++ b/artiq/language/environment.py @@ -73,8 +73,7 @@ class NumberValue(_SimpleArgProcessor): :param unit: A string representing the unit of the value, for user interface (UI) purposes. - :param scale: The scale of value for UI purposes. The corresponding SI - prefix is shown in front of the unit, and the displayed value is + :param scale: The scale of value for UI purposes. The displayed value is divided by the scale. :param step: The step with which the value should be modified by up/down buttons in a UI. The default is the scale divided by 10. diff --git a/artiq/language/scan.py b/artiq/language/scan.py index 7bc5d5665..86f5ac00b 100644 --- a/artiq/language/scan.py +++ b/artiq/language/scan.py @@ -140,8 +140,7 @@ class Scannable: by 10. :param unit: A string representing the unit of the scanned variable, for user interface (UI) purposes. - :param scale: The scale of value for UI purposes. The corresponding SI - prefix is shown in front of the unit, and the displayed value is + :param scale: The scale of value for UI purposes. The displayed value is divided by the scale. :param ndecimals: The number of decimals a UI should use. """ diff --git a/examples/master/repository/arguments_demo.py b/examples/master/repository/arguments_demo.py index a0114c211..efeb7977a 100644 --- a/examples/master/repository/arguments_demo.py +++ b/examples/master/repository/arguments_demo.py @@ -4,7 +4,7 @@ from artiq import * class SubComponent1(HasEnvironment): def build(self): self.setattr_argument("sc1_scan", Scannable(default=NoScan(3250), - scale=1e3, unit="Hz"), + scale=1e3, unit="kHz"), "Flux capacitor") self.setattr_argument("sc1_enum", EnumerationValue(["1", "2", "3"]), "Flux capacitor") @@ -39,7 +39,7 @@ class ArgumentsDemo(EnvExperiment, LogExperiment): self.setattr_argument("free_value", FreeValue(None)) self.setattr_argument("number", NumberValue(42e-6, - unit="s", scale=1e-6, + unit="us", scale=1e-6, ndecimals=4)) self.setattr_argument("string", StringValue("Hello World")) self.setattr_argument("scan", Scannable(global_max=400, From d7b41207da12c6a406995c2a4d54d217e466e919 Mon Sep 17 00:00:00 2001 From: Sebastien Bourdeauducq Date: Sun, 18 Oct 2015 13:31:36 +0800 Subject: [PATCH 22/73] tools/short_format: display booleans directly --- artiq/tools.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/artiq/tools.py b/artiq/tools.py index 819979bb0..0e5b24a30 100644 --- a/artiq/tools.py +++ b/artiq/tools.py @@ -49,7 +49,7 @@ def short_format(v): if v is None: return "None" t = type(v) - if np.issubdtype(t, int) or np.issubdtype(t, float): + if t is bool or np.issubdtype(t, int) or np.issubdtype(t, float): return str(v) elif t is str: return "\"" + elide(v, 15) + "\"" From 661b9bfbfacda64b6bf00a0d51e321601d9dbcf8 Mon Sep 17 00:00:00 2001 From: Sebastien Bourdeauducq Date: Sun, 18 Oct 2015 13:32:29 +0800 Subject: [PATCH 23/73] tools/short_format: increase max string length --- artiq/tools.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/artiq/tools.py b/artiq/tools.py index 0e5b24a30..3c2701e90 100644 --- a/artiq/tools.py +++ b/artiq/tools.py @@ -52,7 +52,7 @@ def short_format(v): if t is bool or np.issubdtype(t, int) or np.issubdtype(t, float): return str(v) elif t is str: - return "\"" + elide(v, 15) + "\"" + return "\"" + elide(v, 50) + "\"" else: r = t.__name__ if t is list or t is dict or t is set: From 5947f54855dd2c67614b3cb4b1d4baf7ad7977fa Mon Sep 17 00:00:00 2001 From: Sebastien Bourdeauducq Date: Sun, 18 Oct 2015 14:34:30 +0800 Subject: [PATCH 24/73] pc_rpc: autotarget support --- artiq/frontend/artiq_rpctool.py | 14 ++---------- artiq/protocols/pc_rpc.py | 39 +++++++++++++++++++++++++-------- 2 files changed, 32 insertions(+), 21 deletions(-) diff --git a/artiq/frontend/artiq_rpctool.py b/artiq/frontend/artiq_rpctool.py index acfa886d1..e097fb6e8 100755 --- a/artiq/frontend/artiq_rpctool.py +++ b/artiq/frontend/artiq_rpctool.py @@ -6,7 +6,7 @@ import sys import numpy as np # Needed to use numpy in RPC call arguments on cmd line import pprint -from artiq.protocols.pc_rpc import Client +from artiq.protocols.pc_rpc import AutoTarget, Client def get_argparser(): @@ -85,19 +85,9 @@ def main(): args = get_argparser().parse_args() remote = Client(args.server, args.port, None) - targets, description = remote.get_rpc_id() - if args.action != "list-targets": - # If no target specified and remote has only one, then use this one. - # Exit otherwise. - if len(targets) > 1 and args.target is None: - print("Remote server has several targets, please supply one with " - "-t") - sys.exit(1) - elif args.target is None: - args.target = targets[0] - remote.select_rpc_target(args.target) + remote.select_rpc_target(AutoTarget) if args.action == "list-targets": list_targets(targets, description) diff --git a/artiq/protocols/pc_rpc.py b/artiq/protocols/pc_rpc.py index beab88224..d39fbb68b 100644 --- a/artiq/protocols/pc_rpc.py +++ b/artiq/protocols/pc_rpc.py @@ -27,6 +27,12 @@ from artiq.protocols.asyncio_server import AsyncioServer as _AsyncioServer logger = logging.getLogger(__name__) +class AutoTarget: + """Use this as target value in clients for them to automatically connect + to the target exposed by the server. Servers must have only one target.""" + pass + + class RemoteError(Exception): """Raised when a RPC failed or raised an exception on the remote (server) side.""" @@ -42,6 +48,20 @@ class IncompatibleServer(Exception): _init_string = b"ARTIQ pc_rpc\n" +def _validate_target_name(target_name, target_names): + if target_name is AutoTarget: + if len(target_names) > 1: + raise ValueError("Server has multiple targets: " + + " ".join(sorted(target_names))) + else: + target_name = target_names[0] + elif target_name not in target_names: + raise IncompatibleServer( + "valid target name(s): " + + " ".join(sorted(target_names))) + return target_name + + class Client: """This class proxies the methods available on the server so that they can be used as if they were local methods. @@ -67,11 +87,13 @@ class Client: :param port: TCP port to use. :param target_name: Target name to select. ``IncompatibleServer`` is raised if the target does not exist. + Use ``AutoTarget`` for automatic selection if the server has only one + target. Use ``None`` to skip selecting a target. The list of targets can then be retrieved using ``get_rpc_id`` and then one can be selected later using ``select_rpc_target``. """ - def __init__(self, host, port, target_name): + def __init__(self, host, port, target_name=AutoTarget): self.__socket = socket.create_connection((host, port)) try: @@ -89,8 +111,7 @@ class Client: def select_rpc_target(self, target_name): """Selects a RPC target by name. This function should be called exactly once if the object was created with ``target_name=None``.""" - if target_name not in self.__target_names: - raise IncompatibleServer + target_name = _validate_target_name(target_name, self.__target_names) self.__socket.sendall((target_name + "\n").encode()) def get_rpc_id(self): @@ -180,8 +201,7 @@ class AsyncioClient: """Selects a RPC target by name. This function should be called exactly once if the connection was created with ``target_name=None``. """ - if target_name not in self.__target_names: - raise IncompatibleServer + target_name = _validate_target_name(target_name, self.__target_names) self.__writer.write((target_name + "\n").encode()) def get_rpc_id(self): @@ -259,7 +279,8 @@ class BestEffortClient: except: logger.warning("first connection attempt to %s:%d[%s] failed, " "retrying in the background", - self.__host, self.__port, self.__target_name) + self.__host, self.__port, self.__target_name, + exc_info=True) self.__start_conretry() else: self.__conretry_thread = None @@ -273,9 +294,9 @@ class BestEffortClient: (self.__host, self.__port), timeout) self.__socket.sendall(_init_string) server_identification = self.__recv() - if self.__target_name not in server_identification["targets"]: - raise IncompatibleServer - self.__socket.sendall((self.__target_name + "\n").encode()) + target_name = _validate_target_name(self.__target_name, + server_identification["targets"]) + self.__socket.sendall((target_name + "\n").encode()) def __start_conretry(self): self.__conretry_thread = threading.Thread(target=self.__conretry) From a5606768ad9c64c41ef9298ca05f0df68abeadf3 Mon Sep 17 00:00:00 2001 From: Sebastien Bourdeauducq Date: Sun, 18 Oct 2015 14:35:49 +0800 Subject: [PATCH 25/73] device_db: make target_name parameter optional for controllers --- artiq/master/worker_db.py | 11 ++++++++--- examples/master/device_db.pyon | 5 ----- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/artiq/master/worker_db.py b/artiq/master/worker_db.py index b9f76bae0..3ba176e09 100644 --- a/artiq/master/worker_db.py +++ b/artiq/master/worker_db.py @@ -9,7 +9,7 @@ import numpy import h5py from artiq.protocols.sync_struct import Notifier -from artiq.protocols.pc_rpc import Client, BestEffortClient +from artiq.protocols.pc_rpc import AutoTarget, Client, BestEffortClient logger = logging.getLogger(__name__) @@ -25,8 +25,13 @@ def _create_device(desc, device_mgr): if desc["best_effort"]: cl = BestEffortClient else: - cl = Client - return cl(desc["host"], desc["port"], desc["target_name"]) + cls = Client + # Automatic target can be specified either by the absence of + # the target_name parameter, or a None value. + target_name = desc.get("target_name", None) + if target_name is None: + target_name = AutoTarget + return cls(desc["host"], desc["port"], target_name) else: raise ValueError("Unsupported type in device DB: " + ty) diff --git a/examples/master/device_db.pyon b/examples/master/device_db.pyon index cafe5e538..effe4ef95 100644 --- a/examples/master/device_db.pyon +++ b/examples/master/device_db.pyon @@ -95,7 +95,6 @@ "best_effort": false, "host": "::1", "port": 4000, - "target_name": "pdq2", "command": "pdq2_controller -p {port} --bind {bind} --simulation --dump qc_q1_0.bin" }, "qc_q1_1": { @@ -103,7 +102,6 @@ "best_effort": false, "host": "::1", "port": 4001, - "target_name": "pdq2", "command": "pdq2_controller -p {port} --bind {bind} --simulation --dump qc_q1_1.bin" }, "qc_q1_2": { @@ -111,7 +109,6 @@ "best_effort": false, "host": "::1", "port": 4002, - "target_name": "pdq2", "command": "pdq2_controller -p {port} --bind {bind} --simulation --dump qc_q1_2.bin" }, "qc_q1_3": { @@ -119,7 +116,6 @@ "best_effort": false, "host": "::1", "port": 4003, - "target_name": "pdq2", "command": "pdq2_controller -p {port} --bind {bind} --simulation --dump qc_q1_3.bin" }, "electrodes": { @@ -138,7 +134,6 @@ "best_effort": true, "host": "::1", "port": 3253, - "target_name": "lda", "command": "lda_controller -p {port} --bind {bind} --simulation" }, From 03e317780bed3c34c669b063cf3e0377c469fad3 Mon Sep 17 00:00:00 2001 From: Sebastien Bourdeauducq Date: Sun, 18 Oct 2015 14:37:08 +0800 Subject: [PATCH 26/73] device_db: make best_effort parameter optional for controllers --- artiq/master/worker_db.py | 4 ++-- examples/master/device_db.pyon | 4 ---- 2 files changed, 2 insertions(+), 6 deletions(-) diff --git a/artiq/master/worker_db.py b/artiq/master/worker_db.py index 3ba176e09..950ec147f 100644 --- a/artiq/master/worker_db.py +++ b/artiq/master/worker_db.py @@ -22,8 +22,8 @@ def _create_device(desc, device_mgr): device_class = getattr(module, desc["class"]) return device_class(device_mgr, **desc["arguments"]) elif ty == "controller": - if desc["best_effort"]: - cl = BestEffortClient + if desc.get("best_effort", False): + cls = BestEffortClient else: cls = Client # Automatic target can be specified either by the absence of diff --git a/examples/master/device_db.pyon b/examples/master/device_db.pyon index effe4ef95..4e5618f3b 100644 --- a/examples/master/device_db.pyon +++ b/examples/master/device_db.pyon @@ -92,28 +92,24 @@ "qc_q1_0": { "type": "controller", - "best_effort": false, "host": "::1", "port": 4000, "command": "pdq2_controller -p {port} --bind {bind} --simulation --dump qc_q1_0.bin" }, "qc_q1_1": { "type": "controller", - "best_effort": false, "host": "::1", "port": 4001, "command": "pdq2_controller -p {port} --bind {bind} --simulation --dump qc_q1_1.bin" }, "qc_q1_2": { "type": "controller", - "best_effort": false, "host": "::1", "port": 4002, "command": "pdq2_controller -p {port} --bind {bind} --simulation --dump qc_q1_2.bin" }, "qc_q1_3": { "type": "controller", - "best_effort": false, "host": "::1", "port": 4003, "command": "pdq2_controller -p {port} --bind {bind} --simulation --dump qc_q1_3.bin" From da70f8b88cbf9a86758fc587f788d1cdce9963ef Mon Sep 17 00:00:00 2001 From: Sebastien Bourdeauducq Date: Mon, 19 Oct 2015 20:20:53 +0800 Subject: [PATCH 27/73] test/pc_rpc: test AutoTarget --- artiq/test/pc_rpc.py | 26 ++++++++++++++++---------- 1 file changed, 16 insertions(+), 10 deletions(-) diff --git a/artiq/test/pc_rpc.py b/artiq/test/pc_rpc.py index 19000c659..423b4f31d 100644 --- a/artiq/test/pc_rpc.py +++ b/artiq/test/pc_rpc.py @@ -17,12 +17,12 @@ test_object = [5, 2.1, None, True, False, class RPCCase(unittest.TestCase): - def _run_server_and_test(self, test): + def _run_server_and_test(self, test, *args): # running this file outside of unittest starts the echo server with subprocess.Popen([sys.executable, sys.modules[__name__].__file__]) as proc: try: - test() + test(*args) finally: try: proc.wait(timeout=1) @@ -30,12 +30,12 @@ class RPCCase(unittest.TestCase): proc.kill() raise - def _blocking_echo(self): + def _blocking_echo(self, target): for attempt in range(100): time.sleep(.2) try: remote = pc_rpc.Client(test_address, test_port, - "test") + target) except ConnectionRefusedError: pass else: @@ -50,14 +50,17 @@ class RPCCase(unittest.TestCase): remote.close_rpc() def test_blocking_echo(self): - self._run_server_and_test(self._blocking_echo) + self._run_server_and_test(self._blocking_echo, "test") - async def _asyncio_echo(self): + def test_blocking_echo_autotarget(self): + self._run_server_and_test(self._blocking_echo, pc_rpc.AutoTarget) + + async def _asyncio_echo(self, target): remote = pc_rpc.AsyncioClient() for attempt in range(100): await asyncio.sleep(.2) try: - await remote.connect_rpc(test_address, test_port, "test") + await remote.connect_rpc(test_address, test_port, target) except ConnectionRefusedError: pass else: @@ -71,16 +74,19 @@ class RPCCase(unittest.TestCase): finally: remote.close_rpc() - def _loop_asyncio_echo(self): + def _loop_asyncio_echo(self, target): loop = asyncio.new_event_loop() asyncio.set_event_loop(loop) try: - loop.run_until_complete(self._asyncio_echo()) + loop.run_until_complete(self._asyncio_echo(target)) finally: loop.close() def test_asyncio_echo(self): - self._run_server_and_test(self._loop_asyncio_echo) + self._run_server_and_test(self._loop_asyncio_echo, "test") + + def test_asyncio_echo_autotarget(self): + self._run_server_and_test(self._loop_asyncio_echo, pc_rpc.AutoTarget) class FireAndForgetCase(unittest.TestCase): From 71d2e3a69fd889d467840f7416854b9fc68255d0 Mon Sep 17 00:00:00 2001 From: Sebastien Bourdeauducq Date: Tue, 20 Oct 2015 00:35:02 +0800 Subject: [PATCH 28/73] protocols/sync_struct: disconnection can also cause BrokenPipeError --- artiq/protocols/sync_struct.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/artiq/protocols/sync_struct.py b/artiq/protocols/sync_struct.py index e2c1021ad..e7100701d 100644 --- a/artiq/protocols/sync_struct.py +++ b/artiq/protocols/sync_struct.py @@ -236,7 +236,7 @@ class Publisher(AsyncioServer): await writer.drain() finally: self._recipients[notifier_name].remove(queue) - except ConnectionResetError: + except (ConnectionResetError, BrokenPipeError): # subscribers disconnecting are a normal occurence pass finally: From 1bc406162029983b79bdbfb7d03547393802f6d2 Mon Sep 17 00:00:00 2001 From: Sebastien Bourdeauducq Date: Tue, 20 Oct 2015 00:35:33 +0800 Subject: [PATCH 29/73] protocols: better workaround for asyncio issue 263 --- artiq/protocols/logging.py | 11 ++--------- artiq/protocols/sync_struct.py | 2 ++ artiq/tools.py | 6 ++++++ 3 files changed, 10 insertions(+), 9 deletions(-) diff --git a/artiq/protocols/logging.py b/artiq/protocols/logging.py index f65212550..e69f76223 100644 --- a/artiq/protocols/logging.py +++ b/artiq/protocols/logging.py @@ -2,7 +2,7 @@ import asyncio import logging from artiq.protocols.asyncio_server import AsyncioServer -from artiq.tools import TaskObject +from artiq.tools import TaskObject, workaround_asyncio263 logger = logging.getLogger(__name__) @@ -119,19 +119,12 @@ class LogForwarder(logging.Handler, TaskObject): try: reader, writer = await asyncio.open_connection(self.host, self.port) - detect_close = asyncio.ensure_future(reader.read(1)) writer.write(_init_string) while True: message = await self._queue.get() + "\n" writer.write(message.encode()) + await workaround_asyncio263() await writer.drain() - # HACK: detect connection termination through the completion - # of a read operation. For some reason, write/drain operations - # on a closed socket do not raise exceptions, but print - # "asyncio:socket.send() raised exception." - if detect_close.done(): - await asyncio.sleep(self.reconnect_timer) - break except asyncio.CancelledError: return except: diff --git a/artiq/protocols/sync_struct.py b/artiq/protocols/sync_struct.py index e7100701d..e42534cdb 100644 --- a/artiq/protocols/sync_struct.py +++ b/artiq/protocols/sync_struct.py @@ -16,6 +16,7 @@ from functools import partial from artiq.protocols import pyon from artiq.protocols.asyncio_server import AsyncioServer +from artiq.tools import workaround_asyncio263 _init_string = b"ARTIQ sync_struct\n" @@ -233,6 +234,7 @@ class Publisher(AsyncioServer): line = await queue.get() writer.write(line) # raise exception on connection error + await workaround_asyncio263() await writer.drain() finally: self._recipients[notifier_name].remove(queue) diff --git a/artiq/tools.py b/artiq/tools.py index 3c2701e90..9e443fb6b 100644 --- a/artiq/tools.py +++ b/artiq/tools.py @@ -175,3 +175,9 @@ class Condition: for fut in self._waiters: if not fut.done(): fut.set_result(False) + + +# See: https://github.com/python/asyncio/issues/263 +@asyncio.coroutine +def workaround_asyncio263(): + yield From d35bc5a40a5c5145fb9502106b361a96dd38bf53 Mon Sep 17 00:00:00 2001 From: Sebastien Bourdeauducq Date: Tue, 20 Oct 2015 11:25:38 +0800 Subject: [PATCH 30/73] examples/device_db: add comment about controller hosts. Closes #148 --- examples/master/device_db.pyon | 3 +++ 1 file changed, 3 insertions(+) diff --git a/examples/master/device_db.pyon b/examples/master/device_db.pyon index 4e5618f3b..f1502abb4 100644 --- a/examples/master/device_db.pyon +++ b/examples/master/device_db.pyon @@ -92,6 +92,9 @@ "qc_q1_0": { "type": "controller", + # ::1 is the IPv6 localhost address. If this controller is running on a remote machine, + # replace it with the IP or hostname of the machine. If using the hostname, make sure + # that it always resolves to a network-visible IP address (see documentation). "host": "::1", "port": 4000, "command": "pdq2_controller -p {port} --bind {bind} --simulation --dump qc_q1_0.bin" From 451f39f5b866b76ff892f358a0d2f36748766dba Mon Sep 17 00:00:00 2001 From: Sebastien Bourdeauducq Date: Tue, 20 Oct 2015 17:56:23 +0800 Subject: [PATCH 31/73] gui/explorer: disable calendar popup --- artiq/gui/explorer.py | 1 - 1 file changed, 1 deletion(-) diff --git a/artiq/gui/explorer.py b/artiq/gui/explorer.py index 76482a94d..3d7ba9db9 100644 --- a/artiq/gui/explorer.py +++ b/artiq/gui/explorer.py @@ -234,7 +234,6 @@ class ExplorerDock(dockarea.Dock): self.datetime = QtGui.QDateTimeEdit() self.datetime.setDisplayFormat("MMM d yyyy hh:mm:ss") - self.datetime.setCalendarPopup(True) self.datetime.setDate(QtCore.QDate.currentDate()) self.datetime.dateTimeChanged.connect(self.enable_duedate) self.datetime_en = QtGui.QCheckBox("Due date:") From fbe33b71f4d0796d0ded4947b53889737abd5479 Mon Sep 17 00:00:00 2001 From: Sebastien Bourdeauducq Date: Tue, 20 Oct 2015 18:09:36 +0800 Subject: [PATCH 32/73] gui: add some tooltips --- artiq/gui/explorer.py | 1 + artiq/gui/log.py | 1 + 2 files changed, 2 insertions(+) diff --git a/artiq/gui/explorer.py b/artiq/gui/explorer.py index 3d7ba9db9..123ad9dca 100644 --- a/artiq/gui/explorer.py +++ b/artiq/gui/explorer.py @@ -251,6 +251,7 @@ class ExplorerDock(dockarea.Dock): grid.addWidget(self.pipeline, 2, 1) self.flush = QtGui.QCheckBox("Flush") + self.flush.setToolTip("Flush the pipeline before starting the experiment") grid.addWidget(self.flush, 2, 2, colspan=2) submit = QtGui.QPushButton("Submit") diff --git a/artiq/gui/log.py b/artiq/gui/log.py index b9c75df05..3c4b1bd99 100644 --- a/artiq/gui/log.py +++ b/artiq/gui/log.py @@ -102,6 +102,7 @@ class LogDock(dockarea.Dock): self.filterbox = QtGui.QComboBox() for item in "DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL": self.filterbox.addItem(item) + self.filterbox.setToolTip("Display entries at or above this level") grid.addWidget(self.filterbox, 0, 1) self.filterbox.currentIndexChanged.connect(self.filter_changed) From d13b368a6575ac7ac932b8fe5dc680d7503bcd53 Mon Sep 17 00:00:00 2001 From: Sebastien Bourdeauducq Date: Tue, 20 Oct 2015 18:11:50 +0800 Subject: [PATCH 33/73] build logging into worker --- artiq/frontend/artiq_client.py | 6 +++++ artiq/gui/explorer.py | 11 +++++++++- artiq/language/__init__.py | 4 +--- artiq/language/logging.py | 23 -------------------- artiq/master/worker.py | 13 ++++------- artiq/master/worker_impl.py | 7 +++--- artiq/protocols/logging.py | 2 +- artiq/test/scheduler.py | 2 ++ artiq/test/worker.py | 4 +++- examples/master/dataset_db.pyon | 2 +- examples/master/repository/arguments_demo.py | 14 ++++++------ 11 files changed, 39 insertions(+), 49 deletions(-) delete mode 100644 artiq/language/logging.py diff --git a/artiq/frontend/artiq_client.py b/artiq/frontend/artiq_client.py index 02cf67775..bcf6e2851 100755 --- a/artiq/frontend/artiq_client.py +++ b/artiq/frontend/artiq_client.py @@ -1,6 +1,7 @@ #!/usr/bin/env python3 import argparse +import logging import time import asyncio import sys @@ -51,6 +52,10 @@ def get_argparser(): "(defaults to head, ignored without -R)") parser_add.add_argument("-c", "--class-name", default=None, help="name of the class to run") + parser_add.add_argument("-v", "--verbose", default=0, action="count", + help="increase logging level of the experiment") + parser_add.add_argument("-q", "--quiet", default=0, action="count", + help="decrease logging level of the experiment") parser_add.add_argument("file", help="file containing the experiment to run") parser_add.add_argument("arguments", nargs="*", @@ -110,6 +115,7 @@ def _action_submit(remote, args): sys.exit(1) expid = { + "log_level": logging.WARNING + args.quiet*10 - args.verbose*10, "file": args.file, "class_name": args.class_name, "arguments": arguments, diff --git a/artiq/gui/explorer.py b/artiq/gui/explorer.py index 123ad9dca..babf60833 100644 --- a/artiq/gui/explorer.py +++ b/artiq/gui/explorer.py @@ -1,4 +1,5 @@ import asyncio +import logging from quamash import QtGui, QtCore from pyqtgraph import dockarea @@ -252,7 +253,14 @@ class ExplorerDock(dockarea.Dock): self.flush = QtGui.QCheckBox("Flush") self.flush.setToolTip("Flush the pipeline before starting the experiment") - grid.addWidget(self.flush, 2, 2, colspan=2) + grid.addWidget(self.flush, 2, 2) + + self.log_level = QtGui.QComboBox() + for item in "DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL": + self.log_level.addItem(item) + self.log_level.setCurrentIndex(1) + self.log_level.setToolTip("Minimum level for log entry production") + grid.addWidget(self.log_level, 2, 3) submit = QtGui.QPushButton("Submit") grid.addWidget(submit, 3, 0, colspan=4) @@ -317,6 +325,7 @@ class ExplorerDock(dockarea.Dock): async def submit(self, pipeline_name, file, class_name, arguments, priority, due_date, flush): expid = { + "log_level": getattr(logging, self.log_level.currentText()), "repo_rev": None, "file": file, "class_name": class_name, diff --git a/artiq/language/__init__.py b/artiq/language/__init__.py index c7be4eb9b..39edc6164 100644 --- a/artiq/language/__init__.py +++ b/artiq/language/__init__.py @@ -1,11 +1,10 @@ # Copyright (C) 2014, 2015 Robert Jordens -from artiq.language import core, environment, units, scan, logging +from artiq.language import core, environment, units, scan from artiq.language.core import * from artiq.language.environment import * from artiq.language.units import * from artiq.language.scan import * -from artiq.language.logging import * __all__ = [] @@ -13,4 +12,3 @@ __all__.extend(core.__all__) __all__.extend(environment.__all__) __all__.extend(units.__all__) __all__.extend(scan.__all__) -__all__.extend(logging.__all__) diff --git a/artiq/language/logging.py b/artiq/language/logging.py deleted file mode 100644 index 067d9864e..000000000 --- a/artiq/language/logging.py +++ /dev/null @@ -1,23 +0,0 @@ -import logging - -from artiq.language.environment import * - - -__all__ = ["LogExperiment"] - - -class LogExperiment: - def init_logger(self): - """Call this from build() to add a logging level enumeration - widget, initialize logging globally, and create a logger. - - Your class must also derive from ``HasEnvironment`` (or - ``EnvExperiment``). - - The created logger is called ``self.logger``.""" - level = self.get_argument("log_level", EnumerationValue( - ["DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL"])) - - if level is not None: - logging.basicConfig(level=getattr(logging, level)) - self.logger = logging.getLogger(self.__class__.__name__) diff --git a/artiq/master/worker.py b/artiq/master/worker.py index 59f3f9b4a..e6b42724f 100644 --- a/artiq/master/worker.py +++ b/artiq/master/worker.py @@ -21,10 +21,6 @@ class WorkerWatchdogTimeout(Exception): pass -class WorkerException(Exception): - pass - - class WorkerError(Exception): pass @@ -60,13 +56,14 @@ class Worker: else: return None - async def _create_process(self): + async def _create_process(self, log_level): await self.io_lock.acquire() try: if self.closed.is_set(): raise WorkerError("Attempting to create process after close") self.process = await asyncio.create_subprocess_exec( sys.executable, "-m", "artiq.master.worker_impl", + str(log_level), stdout=subprocess.PIPE, stdin=subprocess.PIPE) finally: self.io_lock.release() @@ -163,8 +160,6 @@ class Worker: return True elif action == "pause": return False - elif action == "exception": - raise WorkerException del obj["action"] if action == "create_watchdog": func = self.create_watchdog @@ -208,7 +203,7 @@ class Worker: async def build(self, rid, pipeline_name, wd, expid, priority, timeout=15.0): self.rid = rid - await self._create_process() + await self._create_process(expid["log_level"]) await self._worker_action( {"action": "build", "rid": rid, @@ -245,7 +240,7 @@ class Worker: timeout) async def examine(self, file, timeout=20.0): - await self._create_process() + await self._create_process(logging.WARNING) r = dict() def register(class_name, name, arguments): r[class_name] = {"name": name, "arguments": arguments} diff --git a/artiq/master/worker_impl.py b/artiq/master/worker_impl.py index be0c70315..40189e374 100644 --- a/artiq/master/worker_impl.py +++ b/artiq/master/worker_impl.py @@ -1,7 +1,7 @@ import sys import time import os -import traceback +import logging from artiq.protocols import pyon from artiq.tools import file_import @@ -160,6 +160,7 @@ def examine(device_mgr, dataset_mgr, file): def main(): sys.stdout = LogForwarder() sys.stderr = LogForwarder() + logging.basicConfig(level=int(sys.argv[1])) start_time = None rid = None @@ -217,10 +218,10 @@ def main(): elif action == "terminate": break except: - traceback.print_exc() - put_object({"action": "exception"}) + logging.error("Worker terminating with exception", exc_info=True) finally: device_mgr.close_devices() + if __name__ == "__main__": main() diff --git a/artiq/protocols/logging.py b/artiq/protocols/logging.py index e69f76223..47e646d47 100644 --- a/artiq/protocols/logging.py +++ b/artiq/protocols/logging.py @@ -61,7 +61,7 @@ class Server(AsyncioServer): return line = line[:-1] linesplit = line.split(":", 3) - if len(linesplit) != 4: + if len(linesplit) != 2: logger.warning("received improperly formatted message, " "dropping connection") return diff --git a/artiq/test/scheduler.py b/artiq/test/scheduler.py index c8d29759b..f6a029ee5 100644 --- a/artiq/test/scheduler.py +++ b/artiq/test/scheduler.py @@ -1,4 +1,5 @@ import unittest +import logging import asyncio import sys import os @@ -32,6 +33,7 @@ class BackgroundExperiment(EnvExperiment): def _get_expid(name): return { + "log_level": logging.WARNING, "file": sys.modules[__name__].__file__, "class_name": name, "arguments": dict() diff --git a/artiq/test/worker.py b/artiq/test/worker.py index 59847f6a5..c74b6f46a 100644 --- a/artiq/test/worker.py +++ b/artiq/test/worker.py @@ -1,4 +1,5 @@ import unittest +import logging import asyncio import sys import os @@ -64,6 +65,7 @@ async def _call_worker(worker, expid): def _run_experiment(class_name): expid = { + "log_level": logging.WARNING, "file": sys.modules[__name__].__file__, "class_name": class_name, "arguments": dict() @@ -85,7 +87,7 @@ class WorkerCase(unittest.TestCase): _run_experiment("SimpleExperiment") def test_exception(self): - with self.assertRaises(WorkerException): + with self.assertRaises(WorkerError): _run_experiment("ExceptionTermination") def test_watchdog_no_timeout(self): diff --git a/examples/master/dataset_db.pyon b/examples/master/dataset_db.pyon index 00f15d739..ccee85030 100644 --- a/examples/master/dataset_db.pyon +++ b/examples/master/dataset_db.pyon @@ -1 +1 @@ -{"flopping_freq": 1499.9876804260716} +{"flopping_freq": 1499.9977914479744} diff --git a/examples/master/repository/arguments_demo.py b/examples/master/repository/arguments_demo.py index efeb7977a..3bfcde254 100644 --- a/examples/master/repository/arguments_demo.py +++ b/examples/master/repository/arguments_demo.py @@ -1,3 +1,5 @@ +import logging + from artiq import * @@ -33,10 +35,8 @@ class SubComponent2(HasEnvironment): print(self.sc2_enum) -class ArgumentsDemo(EnvExperiment, LogExperiment): +class ArgumentsDemo(EnvExperiment): def build(self): - self.init_logger() - self.setattr_argument("free_value", FreeValue(None)) self.setattr_argument("number", NumberValue(42e-6, unit="us", scale=1e-6, @@ -53,10 +53,10 @@ class ArgumentsDemo(EnvExperiment, LogExperiment): self.sc2 = SubComponent2(parent=self) def run(self): - self.logger.error("logging test: error") - self.logger.warning("logging test: warning") - self.logger.info("logging test: info") - self.logger.debug("logging test: debug") + logging.error("logging test: error") + logging.warning("logging test: warning") + logging.info("logging test: info") + logging.debug("logging test: debug") print(self.free_value) print(self.boolean) From eb546bf4a05cd6763be63e381ab05b60c8923660 Mon Sep 17 00:00:00 2001 From: Sebastien Bourdeauducq Date: Tue, 20 Oct 2015 18:49:27 +0800 Subject: [PATCH 34/73] conda: make package noarch --- .travis.yml | 3 +++ conda/artiq/meta.yaml | 1 + 2 files changed, 4 insertions(+) diff --git a/.travis.yml b/.travis.yml index 7ed278574..3aa5f2648 100644 --- a/.travis.yml +++ b/.travis.yml @@ -19,6 +19,9 @@ before_install: - conda install -q pip coverage anaconda-client migen cython - pip install coveralls install: + # workaround for https://github.com/conda/conda-build/issues/466 + - mkdir -p /home/travis/miniconda/conda-bld/linux-64 + - conda index /home/travis/miniconda/conda-bld/linux-64 - conda build --python 3.5 conda/artiq - conda install -q artiq --use-local script: diff --git a/conda/artiq/meta.yaml b/conda/artiq/meta.yaml index e96b8d4bb..2edbdd7c1 100644 --- a/conda/artiq/meta.yaml +++ b/conda/artiq/meta.yaml @@ -7,6 +7,7 @@ source: git_tag: master build: + noarch_python: True number: {{ environ.get("GIT_DESCRIBE_NUMBER", 0) }} entry_points: - artiq_client = artiq.frontend.artiq_client:main From ec02bea054432bfb3a2e697da98a74b22c803697 Mon Sep 17 00:00:00 2001 From: Sebastien Bourdeauducq Date: Wed, 21 Oct 2015 09:17:39 +0800 Subject: [PATCH 35/73] controllers/thorlabs_tcube: accept any case for -P. Closes #150 --- artiq/frontend/thorlabs_tcube_controller.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/artiq/frontend/thorlabs_tcube_controller.py b/artiq/frontend/thorlabs_tcube_controller.py index 85ccc6ebc..c9b209070 100755 --- a/artiq/frontend/thorlabs_tcube_controller.py +++ b/artiq/frontend/thorlabs_tcube_controller.py @@ -12,7 +12,7 @@ def get_argparser(): parser = argparse.ArgumentParser() parser.add_argument("-P", "--product", required=True, help="type of the Thorlabs T-Cube device to control", - choices=["TDC001", "TPZ001"]) + choices=["tdc001", "tpz001"]) parser.add_argument("-d", "--device", default=None, help="serial device. See documentation for how to " "specify a USB Serial Number.") @@ -33,19 +33,20 @@ def main(): "argument. Use --help for more information.") sys.exit(1) + product = args.product.lower() if args.simulation: - if args.product == "TDC001": + if product == "tdc001": dev = TdcSim() - elif args.product == "TPZ001": + elif product == "tpz001": dev = TpzSim() else: - if args.product == "TDC001": + if product == "tdc001": dev = Tdc(args.device) - elif args.product == "TPZ001": + elif product == "tpz001": dev = Tpz(args.device) try: - simple_server_loop({args.product.lower(): dev}, args.bind, args.port) + simple_server_loop({product: dev}, args.bind, args.port) finally: dev.close() From a58e41684b831f31d078c1801a6c2d0e1d8c8d16 Mon Sep 17 00:00:00 2001 From: Sebastien Bourdeauducq Date: Wed, 21 Oct 2015 09:54:39 +0800 Subject: [PATCH 36/73] protocols/logging: revert debug hack that was incorrectly committed --- artiq/protocols/logging.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/artiq/protocols/logging.py b/artiq/protocols/logging.py index 47e646d47..e69f76223 100644 --- a/artiq/protocols/logging.py +++ b/artiq/protocols/logging.py @@ -61,7 +61,7 @@ class Server(AsyncioServer): return line = line[:-1] linesplit = line.split(":", 3) - if len(linesplit) != 2: + if len(linesplit) != 4: logger.warning("received improperly formatted message, " "dropping connection") return From 50809e02c2ac300190df362b6bba09d1ecd28671 Mon Sep 17 00:00:00 2001 From: Sebastien Bourdeauducq Date: Wed, 21 Oct 2015 11:13:46 +0800 Subject: [PATCH 37/73] tools/short_format: display shape of ndarrays --- artiq/tools.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/artiq/tools.py b/artiq/tools.py index 9e443fb6b..dfd9ef0bb 100644 --- a/artiq/tools.py +++ b/artiq/tools.py @@ -57,6 +57,8 @@ def short_format(v): r = t.__name__ if t is list or t is dict or t is set: r += " ({})".format(len(v)) + if t is np.ndarray: + r += " " + str(np.shape(v)) return r From e4165aecf226757e39568f885d0e21dc140cd908 Mon Sep 17 00:00:00 2001 From: Sebastien Bourdeauducq Date: Thu, 22 Oct 2015 11:00:11 +0800 Subject: [PATCH 38/73] gui/displays: do not test for empty list using bool conversion (breaks for numpy arrays). Closes #153 --- artiq/gui/displays.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/artiq/gui/displays.py b/artiq/gui/displays.py index a08aed041..5fb8069fc 100644 --- a/artiq/gui/displays.py +++ b/artiq/gui/displays.py @@ -137,7 +137,7 @@ class XYDisplay(dockarea.Dock): error = data.get(result_error, None) fit = data.get(result_fit, None) - if not y or len(y) != len(x): + if not len(y) or len(y) != len(x): return if error is not None and hasattr(error, "__len__"): if not len(error): @@ -201,7 +201,7 @@ class HistogramDisplay(dockarea.Dock): if x is None: x = list(range(len(y)+1)) - if y and len(x) == len(y) + 1: + if len(y) and len(x) == len(y) + 1: self.plot.clear() self.plot.plot(x, y, stepMode=True, fillLevel=0, brush=(0, 0, 255, 150)) From 7dea0972796d1f8da1129fbdd0e8bb0f1f9cd19b Mon Sep 17 00:00:00 2001 From: whitequark Date: Thu, 22 Oct 2015 12:58:06 +0300 Subject: [PATCH 39/73] conda: reformat. --- conda/artiq/meta.yaml | 109 +++++++++++++++++++++--------------------- 1 file changed, 54 insertions(+), 55 deletions(-) diff --git a/conda/artiq/meta.yaml b/conda/artiq/meta.yaml index 2edbdd7c1..ed342ca86 100644 --- a/conda/artiq/meta.yaml +++ b/conda/artiq/meta.yaml @@ -1,66 +1,65 @@ package: - name: artiq - version: {{ environ.get("GIT_DESCRIBE_TAG", "") }} + name: artiq + version: {{ environ.get("GIT_DESCRIBE_TAG", "") }} source: - git_url: https://github.com/m-labs/artiq - git_tag: master + git_url: https://github.com/m-labs/artiq + git_tag: master build: - noarch_python: True - number: {{ environ.get("GIT_DESCRIBE_NUMBER", 0) }} - entry_points: - - artiq_client = artiq.frontend.artiq_client:main - - artiq_compile = artiq.frontend.artiq_compile:main - - artiq_coretool = artiq.frontend.artiq_coretool:main - - artiq_ctlmgr = artiq.frontend.artiq_ctlmgr:main - - artiq_gui = artiq.frontend.artiq_gui:main - - artiq_influxdb = artiq.frontend.artiq_influxdb:main - - artiq_master = artiq.frontend.artiq_master:main - - artiq_mkfs = artiq.frontend.artiq_mkfs:main - - artiq_rpctool = artiq.frontend.artiq_rpctool:main - - artiq_run = artiq.frontend.artiq_run:main - - lda_controller = artiq.frontend.lda_controller:main - - novatech409b_controller = artiq.frontend.novatech409b_controller:main - - pdq2_client = artiq.frontend.pdq2_client:main - - pdq2_controller = artiq.frontend.pdq2_controller:main - - pxi6733_controller = artiq.frontend.pxi6733_controller:main - - thorlabs_tcube_controller = artiq.frontend.thorlabs_tcube_controller:main + noarch_python: True + number: {{ environ.get("GIT_DESCRIBE_NUMBER", 0) }} + entry_points: + - artiq_client = artiq.frontend.artiq_client:main + - artiq_compile = artiq.frontend.artiq_compile:main + - artiq_coretool = artiq.frontend.artiq_coretool:main + - artiq_ctlmgr = artiq.frontend.artiq_ctlmgr:main + - artiq_gui = artiq.frontend.artiq_gui:main + - artiq_influxdb = artiq.frontend.artiq_influxdb:main + - artiq_master = artiq.frontend.artiq_master:main + - artiq_mkfs = artiq.frontend.artiq_mkfs:main + - artiq_rpctool = artiq.frontend.artiq_rpctool:main + - artiq_run = artiq.frontend.artiq_run:main + - lda_controller = artiq.frontend.lda_controller:main + - novatech409b_controller = artiq.frontend.novatech409b_controller:main + - pdq2_client = artiq.frontend.pdq2_client:main + - pdq2_controller = artiq.frontend.pdq2_controller:main + - pxi6733_controller = artiq.frontend.pxi6733_controller:main + - thorlabs_tcube_controller = artiq.frontend.thorlabs_tcube_controller:main requirements: - build: - - python >=3.5.0 - - setuptools - - numpy - - migen 0.0 - - pyelftools - - binutils-or1k-linux - run: - - python >=3.5.0 - - llvmlite-artiq - - scipy - - numpy - - prettytable - - pyserial - - sphinx - - sphinx-argparse - - h5py - - dateutil - - pydaqmx - - pyelftools - - quamash - - pyqtgraph - - flterm # [linux] - - pygit2 - - aiohttp - - binutils-or1k-linux + build: + - python >=3.5.0 + - setuptools + - numpy + - migen 0.0 + - pyelftools + - binutils-or1k-linux + run: + - python >=3.5.0 + - llvmlite-artiq + - scipy + - numpy + - prettytable + - pyserial + - sphinx + - sphinx-argparse + - h5py + - dateutil + - pydaqmx + - pyelftools + - quamash + - pyqtgraph + - flterm # [linux] + - pygit2 + - aiohttp + - binutils-or1k-linux test: - imports: - - artiq - + imports: + - artiq about: - home: http://m-labs.hk/artiq - license: 3-clause BSD - summary: 'ARTIQ (Advanced Real-Time Infrastructure for Quantum physics) is a next-generation control system for quantum information experiments. It is being developed in partnership with the Ion Storage Group at NIST, and its applicability reaches beyond ion trapping.' + home: http://m-labs.hk/artiq + license: 3-clause BSD + summary: 'ARTIQ (Advanced Real-Time Infrastructure for Quantum physics) is a next-generation control system for quantum information experiments. It is being developed in partnership with the Ion Storage Group at NIST, and its applicability reaches beyond ion trapping.' From 237959b1f2426d3dca31045eaed4e9d6f94be339 Mon Sep 17 00:00:00 2001 From: whitequark Date: Thu, 22 Oct 2015 12:59:00 +0300 Subject: [PATCH 40/73] conda: include git hash in build string. --- conda/artiq/meta.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/conda/artiq/meta.yaml b/conda/artiq/meta.yaml index ed342ca86..50c37d14a 100644 --- a/conda/artiq/meta.yaml +++ b/conda/artiq/meta.yaml @@ -9,6 +9,7 @@ source: build: noarch_python: True number: {{ environ.get("GIT_DESCRIBE_NUMBER", 0) }} + string: py_{{ environ.get("GIT_DESCRIBE_NUMBER", 0) }}+git{{ environ.get("GIT_DESCRIBE_HASH", "")[1:] }} entry_points: - artiq_client = artiq.frontend.artiq_client:main - artiq_compile = artiq.frontend.artiq_compile:main From 8b78fe492a076ce105fbbc912c898c6318a6f242 Mon Sep 17 00:00:00 2001 From: whitequark Date: Thu, 22 Oct 2015 12:59:18 +0300 Subject: [PATCH 41/73] conda: build noarch package without bitstreams. --- .travis.yml | 36 ++++++++++++------------- conda/artiq/build.sh | 61 ------------------------------------------- conda/artiq/meta.yaml | 2 +- 3 files changed, 19 insertions(+), 80 deletions(-) diff --git a/.travis.yml b/.travis.yml index 3aa5f2648..559a48fec 100644 --- a/.travis.yml +++ b/.travis.yml @@ -6,32 +6,32 @@ env: global: - secure: "DUk/Ihg8KbbzEgPF0qrHqlxU8e8eET9i/BtzNvFddIGX4HP/P2qz0nk3cVkmjuWhqJXSbC22RdKME9qqPzw6fJwJ6dpJ3OR6dDmSd7rewavq+niwxu52PVa+yK8mL4yf1terM7QQ5tIRf+yUL9qGKrZ2xyvEuRit6d4cFep43Ws=" matrix: - - BUILD_SOC=0 - - BUILD_SOC=1 -before_install: - - mkdir -p $HOME/.mlabs - - if [ $TRAVIS_PULL_REQUEST != false ]; then BUILD_SOC=0; fi - - if [ $BUILD_SOC -ne 0 ]; then ./.travis/get-xilinx.sh; fi - - . ./.travis/get-toolchain.sh - - . ./.travis/get-anaconda.sh - - echo "BUILD_SOC=$BUILD_SOC" >> $HOME/.mlabs/build_settings.sh - - source $HOME/miniconda/bin/activate py35 - - conda install -q pip coverage anaconda-client migen cython - - pip install coveralls + - BUILD_SOC=none install: + - mkdir -p $HOME/.mlabs + - if [ $TRAVIS_PULL_REQUEST != false ]; then BUILD_SOC=none; fi + - if [ $BUILD_SOC != none ]; then ./.travis/get-xilinx.sh; ./.travis/get-toolchain.sh; fi + - . ./.travis/get-anaconda.sh + - source $HOME/miniconda/bin/activate py35 + - conda install -q pip coverage anaconda-client migen=0.0 cython + - pip install coveralls # workaround for https://github.com/conda/conda-build/issues/466 - mkdir -p /home/travis/miniconda/conda-bld/linux-64 - conda index /home/travis/miniconda/conda-bld/linux-64 - - conda build --python 3.5 conda/artiq - - conda install -q artiq --use-local script: - - coverage run --source=artiq setup.py test - - make -C doc/manual html + - conda build --python 3.5 conda/artiq + - conda install -q --use-local artiq + - | + if [ $BUILD_SOC == none ]; then + PACKAGES="$(conda build --output --python 3.5 conda/artiq) $PACKAGES" + coverage run --source=artiq setup.py test + make -C doc/manual html + fi after_success: - | - if [ "$TRAVIS_BRANCH" == "master" -a $BUILD_SOC -eq 1 ]; then + if [ "$TRAVIS_BRANCH" == "master" -a "$PACKAGES" != "" ]; then anaconda -q login --hostname $(hostname) --username $binstar_login --password $binstar_password - anaconda -q upload --user $binstar_login --channel dev --force $HOME/miniconda/conda-bld/linux-64/artiq-*.tar.bz2 + anaconda -q upload --user $binstar_login --channel dev $PACKAGES anaconda -q logout fi - coveralls diff --git a/conda/artiq/build.sh b/conda/artiq/build.sh index 4d574de48..3786386a7 100755 --- a/conda/artiq/build.sh +++ b/conda/artiq/build.sh @@ -1,64 +1,3 @@ #!/bin/bash -BUILD_SETTINGS_FILE=$HOME/.mlabs/build_settings.sh - -if [ -f $BUILD_SETTINGS_FILE ] -then - source $BUILD_SETTINGS_FILE -fi - $PYTHON setup.py install --single-version-externally-managed --record=record.txt -git clone --recursive https://github.com/m-labs/misoc -export MSCDIR=$SRC_DIR/misoc - -ARTIQ_PREFIX=$PREFIX/lib/python3.5/site-packages/artiq -BIN_PREFIX=$ARTIQ_PREFIX/binaries/ -mkdir -p $ARTIQ_PREFIX/misc -mkdir -p $BIN_PREFIX/kc705 $BIN_PREFIX/pipistrello - -# build for KC705 NIST_QC1 - -cd $SRC_DIR/misoc; $PYTHON make.py -X ../soc -t artiq_kc705 build-headers build-bios; cd - -make -C soc/runtime clean runtime.fbi -[ "$BUILD_SOC" != "0" ] && (cd $SRC_DIR/misoc; $PYTHON make.py -X ../soc -t artiq_kc705 $MISOC_EXTRA_VIVADO_CMDLINE build-bitstream) - -# install KC705 NIST_QC1 binaries - -mkdir -p $BIN_PREFIX/kc705/nist_qc1 -cp soc/runtime/runtime.fbi $BIN_PREFIX/kc705/nist_qc1/ -cp $SRC_DIR/misoc/software/bios/bios.bin $BIN_PREFIX/kc705/ -[ "$BUILD_SOC" != "0" ] && cp $SRC_DIR/misoc/build/artiq_kc705-nist_qc1-kc705.bit $BIN_PREFIX/kc705/ -wget http://sionneau.net/artiq/binaries/kc705/flash_proxy/bscan_spi_kc705.bit -mv bscan_spi_kc705.bit $BIN_PREFIX/kc705/ - -# build for Pipistrello - -cd $SRC_DIR/misoc; $PYTHON make.py -X ../soc -t artiq_pipistrello build-headers build-bios; cd - -make -C soc/runtime clean runtime.fbi -[ "$BUILD_SOC" != "0" ] && (cd $SRC_DIR/misoc; $PYTHON make.py -X ../soc -t artiq_pipistrello $MISOC_EXTRA_ISE_CMDLINE build-bitstream) - -# install Pipistrello binaries - -cp soc/runtime/runtime.fbi $BIN_PREFIX/pipistrello/ -cp $SRC_DIR/misoc/software/bios/bios.bin $BIN_PREFIX/pipistrello/ -[ "$BUILD_SOC" != "0" ] && cp $SRC_DIR/misoc/build/artiq_pipistrello-nist_qc1-pipistrello.bit $BIN_PREFIX/pipistrello/ -wget https://people.phys.ethz.ch/~robertjo/bscan_spi_lx45_csg324.bit -mv bscan_spi_lx45_csg324.bit $BIN_PREFIX/pipistrello/ - -# build for KC705 NIST_QC2 - -cd $SRC_DIR/misoc; $PYTHON make.py -X ../soc -t artiq_kc705 -s NIST_QC2 build-headers; cd - -make -C soc/runtime clean runtime.fbi -[ "$BUILD_SOC" != "0" ] && (cd $SRC_DIR/misoc; $PYTHON make.py -X ../soc -t artiq_kc705 -s NIST_QC2 $MISOC_EXTRA_VIVADO_CMDLINE build-bitstream) - -# install KC705 NIST_QC2 binaries - -mkdir -p $BIN_PREFIX/kc705/nist_qc2 -cp soc/runtime/runtime.fbi $BIN_PREFIX/kc705/nist_qc2/ -[ "$BUILD_SOC" != "0" ] && cp $SRC_DIR/misoc/build/artiq_kc705-nist_qc2-kc705.bit $BIN_PREFIX/kc705/ - -cp artiq/frontend/artiq_flash.sh $PREFIX/bin - -# misc -cp misc/99-papilio.rules $ARTIQ_PREFIX/misc/ -cp misc/99-kc705.rules $ARTIQ_PREFIX/misc/ diff --git a/conda/artiq/meta.yaml b/conda/artiq/meta.yaml index 50c37d14a..f6f24e2f3 100644 --- a/conda/artiq/meta.yaml +++ b/conda/artiq/meta.yaml @@ -7,7 +7,7 @@ source: git_tag: master build: - noarch_python: True + noarch_python: true number: {{ environ.get("GIT_DESCRIBE_NUMBER", 0) }} string: py_{{ environ.get("GIT_DESCRIBE_NUMBER", 0) }}+git{{ environ.get("GIT_DESCRIBE_HASH", "")[1:] }} entry_points: From 8374e8d33e288a9bde2ad9624d81d64c10731551 Mon Sep 17 00:00:00 2001 From: whitequark Date: Thu, 22 Oct 2015 15:08:40 +0300 Subject: [PATCH 42/73] conda: add artiq-pipistrello-nist_qc1. --- .travis.yml | 12 +++++++--- .travis/get-misoc.sh | 4 ++++ .travis/get-toolchain.sh | 18 ++------------- .travis/get-xilinx.sh | 2 +- conda/artiq-pipistrello-nist_qc1/build.sh | 26 +++++++++++++++++++++ conda/artiq-pipistrello-nist_qc1/meta.yaml | 27 ++++++++++++++++++++++ 6 files changed, 69 insertions(+), 20 deletions(-) create mode 100755 .travis/get-misoc.sh create mode 100644 conda/artiq-pipistrello-nist_qc1/build.sh create mode 100644 conda/artiq-pipistrello-nist_qc1/meta.yaml diff --git a/.travis.yml b/.travis.yml index 559a48fec..044f31114 100644 --- a/.travis.yml +++ b/.travis.yml @@ -7,13 +7,16 @@ env: - secure: "DUk/Ihg8KbbzEgPF0qrHqlxU8e8eET9i/BtzNvFddIGX4HP/P2qz0nk3cVkmjuWhqJXSbC22RdKME9qqPzw6fJwJ6dpJ3OR6dDmSd7rewavq+niwxu52PVa+yK8mL4yf1terM7QQ5tIRf+yUL9qGKrZ2xyvEuRit6d4cFep43Ws=" matrix: - BUILD_SOC=none + - BUILD_SOC=pipistrello-nist_qc1 install: - - mkdir -p $HOME/.mlabs + - mkdir -p $HOME/.m-labs - if [ $TRAVIS_PULL_REQUEST != false ]; then BUILD_SOC=none; fi - - if [ $BUILD_SOC != none ]; then ./.travis/get-xilinx.sh; ./.travis/get-toolchain.sh; fi + - if [ $BUILD_SOC != none ]; then ./.travis/get-xilinx.sh; fi + - if [ $BUILD_SOC != none ]; then ./.travis/get-toolchain.sh; fi + - if [ $BUILD_SOC != none ]; then ./.travis/get-misoc.sh; fi - . ./.travis/get-anaconda.sh - source $HOME/miniconda/bin/activate py35 - - conda install -q pip coverage anaconda-client migen=0.0 cython + - conda install -q pip coverage anaconda-client cython - pip install coveralls # workaround for https://github.com/conda/conda-build/issues/466 - mkdir -p /home/travis/miniconda/conda-bld/linux-64 @@ -26,6 +29,9 @@ script: PACKAGES="$(conda build --output --python 3.5 conda/artiq) $PACKAGES" coverage run --source=artiq setup.py test make -C doc/manual html + else + PACKAGES="$(conda build --output --python 3.5 conda/artiq-$BUILD_SOC) $PACKAGES" + conda build --python 3.5 conda/artiq-$BUILD_SOC fi after_success: - | diff --git a/.travis/get-misoc.sh b/.travis/get-misoc.sh new file mode 100755 index 000000000..355e8ffa5 --- /dev/null +++ b/.travis/get-misoc.sh @@ -0,0 +1,4 @@ +#!/bin/sh + +git clone --recursive https://github.com/m-labs/misoc $HOME/misoc +echo "export MSCDIR=$HOME/misoc" >> $HOME/.m-labs/build_settings.sh diff --git a/.travis/get-toolchain.sh b/.travis/get-toolchain.sh index 73c268d0a..6dd5f94ff 100755 --- a/.travis/get-toolchain.sh +++ b/.travis/get-toolchain.sh @@ -1,7 +1,6 @@ #!/bin/sh packages="http://us.archive.ubuntu.com/ubuntu/pool/universe/i/iverilog/iverilog_0.9.7-1_amd64.deb" -archives="http://fehu.whitequark.org/files/llvm-or1k.tbz2" mkdir -p packages @@ -12,18 +11,5 @@ do dpkg -x $pkg_name packages done -for a in $archives -do - wget $a - (cd packages && tar xf ../$(basename $a)) -done - -export PATH=$PWD/packages/usr/local/llvm-or1k/bin:$PWD/packages/usr/local/bin:$PWD/packages/usr/bin:$PATH -export LD_LIBRARY_PATH=$PWD/packages/usr/lib/x86_64-linux-gnu:$PWD/packages/usr/local/x86_64-unknown-linux-gnu/or1k-elf/lib:$LD_LIBRARY_PATH - -echo "export LD_LIBRARY_PATH=$PWD/packages/usr/lib/x86_64-linux-gnu:$PWD/packages/usr/local/x86_64-unknown-linux-gnu/or1k-elf/lib:\$LD_LIBRARY_PATH" >> $HOME/.mlabs/build_settings.sh -echo "export PATH=$PWD/packages/usr/local/llvm-or1k/bin:$PWD/packages/usr/local/bin:$PWD/packages/usr/bin:\$PATH" >> $HOME/.mlabs/build_settings.sh - -or1k-linux-as --version -llc --version -clang --version +echo "export LD_LIBRARY_PATH=$PWD/packages/usr/lib/x86_64-linux-gnu" >> $HOME/.m-labs/build_settings.sh +echo "export PATH=$PWD/packages/usr/bin:\$PATH" >> $HOME/.m-labs/build_settings.sh diff --git a/.travis/get-xilinx.sh b/.travis/get-xilinx.sh index 0f37893b3..3d49fd693 100755 --- a/.travis/get-xilinx.sh +++ b/.travis/get-xilinx.sh @@ -30,7 +30,7 @@ git clone https://github.com/fallen/impersonate_macaddress make -C impersonate_macaddress # Tell mibuild where Xilinx toolchains are installed # and feed it the mac address corresponding to the license -cat >> $HOME/.mlabs/build_settings.sh << EOF +cat >> $HOME/.m-labs/build_settings.sh << EOF MISOC_EXTRA_VIVADO_CMDLINE="-Ob vivado_path $HOME/Xilinx/Vivado" MISOC_EXTRA_ISE_CMDLINE="-Ob ise_path $HOME/opt/Xilinx/" export MACADDR=$macaddress diff --git a/conda/artiq-pipistrello-nist_qc1/build.sh b/conda/artiq-pipistrello-nist_qc1/build.sh new file mode 100644 index 000000000..f35b22a39 --- /dev/null +++ b/conda/artiq-pipistrello-nist_qc1/build.sh @@ -0,0 +1,26 @@ +#!/bin/bash + +BUILD_SETTINGS_FILE=$HOME/.m-labs/build_settings.sh +[ -f $BUILD_SETTINGS_FILE ] && . $BUILD_SETTINGS_FILE + +SOC_PREFIX=$PREFIX/lib/python3.5/site-packages/artiq/binaries/pipistrello +mkdir -p $SOC_PREFIX + +SOC_ROOT=$PWD/soc + +# build bitstream + +(cd $MSCDIR; $PYTHON make.py -X $SOC_ROOT -t artiq_pipistrello $MISOC_EXTRA_ISE_CMDLINE build-bitstream) +cp $MSCDIR/build/artiq_pipistrello-nist_qc1-pipistrello.bit $SOC_PREFIX/ +wget https://people.phys.ethz.ch/~robertjo/bscan_spi_lx45_csg324.bit +mv bscan_spi_lx45_csg324.bit $SOC_PREFIX/ + +# build BIOS + +(cd $MSCDIR; $PYTHON make.py -X $SOC_ROOT -t artiq_pipistrello build-headers build-bios) +cp $MSCDIR/software/bios/bios.bin $SOC_PREFIX/ + +# build runtime + +make -C soc/runtime clean runtime.fbi +cp soc/runtime/runtime.fbi $SOC_PREFIX/ diff --git a/conda/artiq-pipistrello-nist_qc1/meta.yaml b/conda/artiq-pipistrello-nist_qc1/meta.yaml new file mode 100644 index 000000000..254361e67 --- /dev/null +++ b/conda/artiq-pipistrello-nist_qc1/meta.yaml @@ -0,0 +1,27 @@ +package: + name: artiq-pipistrello-nist_qc1 + version: {{ environ.get("GIT_DESCRIBE_TAG", "") }} + +source: + git_url: https://github.com/m-labs/artiq + git_tag: master + +build: + noarch_python: true + number: {{ environ.get("GIT_DESCRIBE_NUMBER", 0) }} + string: py_{{ environ.get("GIT_DESCRIBE_NUMBER", 0) }}+git{{ environ.get("GIT_DESCRIBE_HASH", "")[1:] }} + +requirements: + build: + # We don't get meaningful GIT_DESCRIBE_* values until before conda installs build dependencies. + - artiq 0.0 + - migen 0.0 + - llvm-or1k + - binutils-or1k-linux + run: + - artiq {{ "{tag} py_{number}+git{hash}".format(tag=environ.get("GIT_DESCRIBE_TAG"), number=environ.get("GIT_DESCRIBE_NUMBER"), hash=environ.get("GIT_DESCRIBE_HASH")[1:]) if "GIT_DESCRIBE_TAG" in environ else "" }} + +about: + home: http://m-labs.hk/artiq + license: 3-clause BSD + summary: 'Bitstream, BIOS and runtime for NIST_QC1 on the Pipistrello board' From 026504e5782b5b49b4130db445b6393fe076ff05 Mon Sep 17 00:00:00 2001 From: whitequark Date: Thu, 22 Oct 2015 16:00:31 +0300 Subject: [PATCH 43/73] conda: add artiq-kc705-nist_qc[12]. --- .travis.yml | 2 ++ conda/artiq-kc705-nist_qc1/build.sh | 26 ++++++++++++++++++++++++++ conda/artiq-kc705-nist_qc1/meta.yaml | 27 +++++++++++++++++++++++++++ conda/artiq-kc705-nist_qc2/build.sh | 26 ++++++++++++++++++++++++++ conda/artiq-kc705-nist_qc2/meta.yaml | 27 +++++++++++++++++++++++++++ 5 files changed, 108 insertions(+) create mode 100644 conda/artiq-kc705-nist_qc1/build.sh create mode 100644 conda/artiq-kc705-nist_qc1/meta.yaml create mode 100644 conda/artiq-kc705-nist_qc2/build.sh create mode 100644 conda/artiq-kc705-nist_qc2/meta.yaml diff --git a/.travis.yml b/.travis.yml index 044f31114..e05589de2 100644 --- a/.travis.yml +++ b/.travis.yml @@ -8,6 +8,8 @@ env: matrix: - BUILD_SOC=none - BUILD_SOC=pipistrello-nist_qc1 + - BUILD_SOC=kc705-nist_qc1 + - BUILD_SOC=kc705-nist_qc2 install: - mkdir -p $HOME/.m-labs - if [ $TRAVIS_PULL_REQUEST != false ]; then BUILD_SOC=none; fi diff --git a/conda/artiq-kc705-nist_qc1/build.sh b/conda/artiq-kc705-nist_qc1/build.sh new file mode 100644 index 000000000..495a0e74c --- /dev/null +++ b/conda/artiq-kc705-nist_qc1/build.sh @@ -0,0 +1,26 @@ +#!/bin/bash + +BUILD_SETTINGS_FILE=$HOME/.m-labs/build_settings.sh +[ -f $BUILD_SETTINGS_FILE ] && . $BUILD_SETTINGS_FILE + +SOC_PREFIX=$PREFIX/lib/python3.5/site-packages/artiq/binaries/kc705 +mkdir -p $SOC_PREFIX/nist_qc1 + +SOC_ROOT=$PWD/soc + +# build bitstream + +(cd $MSCDIR; $PYTHON make.py -X $SOC_ROOT -t artiq_kc705 $MISOC_EXTRA_VIVADO_CMDLINE build-bitstream) +cp $MSCDIR/build/artiq_kc705-nist_qc1-kc705.bit $SOC_PREFIX/ +wget http://sionneau.net/artiq/binaries/kc705/flash_proxy/bscan_spi_kc705.bit +mv bscan_spi_kc705.bit $SOC_PREFIX/ + +# build BIOS + +(cd $MSCDIR; $PYTHON make.py -X $SOC_ROOT -t artiq_kc705 build-headers build-bios) +cp $MSCDIR/software/bios/bios.bin $SOC_PREFIX/ + +# build runtime + +make -C soc/runtime clean runtime.fbi +cp soc/runtime/runtime.fbi $SOC_PREFIX/nist_qc1/ diff --git a/conda/artiq-kc705-nist_qc1/meta.yaml b/conda/artiq-kc705-nist_qc1/meta.yaml new file mode 100644 index 000000000..bee5c234b --- /dev/null +++ b/conda/artiq-kc705-nist_qc1/meta.yaml @@ -0,0 +1,27 @@ +package: + name: artiq-kc705-nist_qc1 + version: {{ environ.get("GIT_DESCRIBE_TAG", "") }} + +source: + git_url: https://github.com/m-labs/artiq + git_tag: master + +build: + noarch_python: true + number: {{ environ.get("GIT_DESCRIBE_NUMBER", 0) }} + string: py_{{ environ.get("GIT_DESCRIBE_NUMBER", 0) }}+git{{ environ.get("GIT_DESCRIBE_HASH", "")[1:] }} + +requirements: + build: + # We don't get meaningful GIT_DESCRIBE_* values until before conda installs build dependencies. + - artiq 0.0 + - migen 0.0 + - llvm-or1k + - binutils-or1k-linux + run: + - artiq {{ "{tag} py_{number}+git{hash}".format(tag=environ.get("GIT_DESCRIBE_TAG"), number=environ.get("GIT_DESCRIBE_NUMBER"), hash=environ.get("GIT_DESCRIBE_HASH")[1:]) if "GIT_DESCRIBE_TAG" in environ else "" }} + +about: + home: http://m-labs.hk/artiq + license: 3-clause BSD + summary: 'Bitstream, BIOS and runtime for NIST_QC1 on the KC705 board' diff --git a/conda/artiq-kc705-nist_qc2/build.sh b/conda/artiq-kc705-nist_qc2/build.sh new file mode 100644 index 000000000..a65294b85 --- /dev/null +++ b/conda/artiq-kc705-nist_qc2/build.sh @@ -0,0 +1,26 @@ +#!/bin/bash + +BUILD_SETTINGS_FILE=$HOME/.m-labs/build_settings.sh +[ -f $BUILD_SETTINGS_FILE ] && . $BUILD_SETTINGS_FILE + +SOC_PREFIX=$PREFIX/lib/python3.5/site-packages/artiq/binaries/kc705 +mkdir -p $SOC_PREFIX/nist_qc2 + +SOC_ROOT=$PWD/soc + +# build bitstream + +(cd $MSCDIR; $PYTHON make.py -X $SOC_ROOT -t artiq_kc705 -s NIST_QC2 $MISOC_EXTRA_VIVADO_CMDLINE build-bitstream) +cp $MSCDIR/build/artiq_kc705-nist_qc2-kc705.bit $SOC_PREFIX/ +wget http://sionneau.net/artiq/binaries/kc705/flash_proxy/bscan_spi_kc705.bit +mv bscan_spi_kc705.bit $SOC_PREFIX/ + +# build BIOS + +(cd $MSCDIR; $PYTHON make.py -X $SOC_ROOT -t artiq_kc705 -s NIST_QC2 build-headers build-bios) +cp $MSCDIR/software/bios/bios.bin $SOC_PREFIX/ + +# build runtime + +make -C soc/runtime clean runtime.fbi +cp soc/runtime/runtime.fbi $SOC_PREFIX/nist_qc2/ diff --git a/conda/artiq-kc705-nist_qc2/meta.yaml b/conda/artiq-kc705-nist_qc2/meta.yaml new file mode 100644 index 000000000..1cbb86f2e --- /dev/null +++ b/conda/artiq-kc705-nist_qc2/meta.yaml @@ -0,0 +1,27 @@ +package: + name: artiq-kc705-nist_qc2 + version: {{ environ.get("GIT_DESCRIBE_TAG", "") }} + +source: + git_url: https://github.com/m-labs/artiq + git_tag: master + +build: + noarch_python: true + number: {{ environ.get("GIT_DESCRIBE_NUMBER", 0) }} + string: py_{{ environ.get("GIT_DESCRIBE_NUMBER", 0) }}+git{{ environ.get("GIT_DESCRIBE_HASH", "")[1:] }} + +requirements: + build: + # We don't get meaningful GIT_DESCRIBE_* values until before conda installs build dependencies. + - artiq 0.0 + - migen 0.0 + - llvm-or1k + - binutils-or1k-linux + run: + - artiq {{ "{tag} py_{number}+git{hash}".format(tag=environ.get("GIT_DESCRIBE_TAG"), number=environ.get("GIT_DESCRIBE_NUMBER"), hash=environ.get("GIT_DESCRIBE_HASH")[1:]) if "GIT_DESCRIBE_TAG" in environ else "" }} + +about: + home: http://m-labs.hk/artiq + license: 3-clause BSD + summary: 'Bitstream, BIOS and runtime for NIST_QC2 on the KC705 board' From 30842a71743f9fdfe0c7c5aa997c994bbb724326 Mon Sep 17 00:00:00 2001 From: whitequark Date: Thu, 22 Oct 2015 17:18:59 +0300 Subject: [PATCH 44/73] conda: include udev rules and artiq_flash.sh in artiq package. --- artiq/frontend/artiq_flash.sh | 16 ++++++++++++++-- conda/artiq/build.sh | 12 ++++++++++++ 2 files changed, 26 insertions(+), 2 deletions(-) diff --git a/artiq/frontend/artiq_flash.sh b/artiq/frontend/artiq_flash.sh index 881e5a616..155fcb422 100755 --- a/artiq/frontend/artiq_flash.sh +++ b/artiq/frontend/artiq_flash.sh @@ -1,5 +1,16 @@ -#!/bin/bash +#!/usr/bin/env python +# conda-build requires all scripts to have a python shebang. +# see https://github.com/conda/conda-build/blob/6921f067a/conda_build/noarch_python.py#L36-L38 +def run(script): + import sys, tempfile, subprocess + file = tempfile.NamedTemporaryFile(mode='w+t', suffix='sh') + file.write(script) + file.flush() + subprocess.run(["/bin/bash", file.name] + sys.argv[1:]) + file.close() + +run(""" # exit on error set -e # print commands @@ -72,7 +83,7 @@ do echo "" echo "To flash everything, do not use any of the -b|-B|-r option." echo "" - echo "usage: $0 [-b] [-B] [-r] [-h] [-m nist_qc1|nist_qc2] [-t kc705|pipistrello] [-d path] [-f path]" + echo "usage: artiq_flash.sh [-b] [-B] [-r] [-h] [-m nist_qc1|nist_qc2] [-t kc705|pipistrello] [-d path] [-f path]" echo "-b Flash bitstream" echo "-B Flash BIOS" echo "-r Flash ARTIQ runtime" @@ -193,3 +204,4 @@ then fi echo "Done." xc3sprog -v -c $CABLE -R > /dev/null 2>&1 +""") diff --git a/conda/artiq/build.sh b/conda/artiq/build.sh index 3786386a7..1bbc544de 100755 --- a/conda/artiq/build.sh +++ b/conda/artiq/build.sh @@ -1,3 +1,15 @@ #!/bin/bash +ARTIQ_PREFIX=$PREFIX/lib/python3.5/site-packages/artiq + $PYTHON setup.py install --single-version-externally-managed --record=record.txt + +# install scripts + +cp artiq/frontend/artiq_flash.sh $PREFIX/bin + +# install udev rules + +mkdir -p $ARTIQ_PREFIX/misc +cp misc/99-papilio.rules $ARTIQ_PREFIX/misc/ +cp misc/99-kc705.rules $ARTIQ_PREFIX/misc/ From fd910faa38bf2eb4d05c0df65dec0a2ac4627060 Mon Sep 17 00:00:00 2001 From: Sebastien Bourdeauducq Date: Fri, 23 Oct 2015 09:09:45 +0800 Subject: [PATCH 45/73] change license to GPL in scripts --- conda/artiq-kc705-nist_qc1/meta.yaml | 2 +- conda/artiq-kc705-nist_qc2/meta.yaml | 2 +- conda/artiq-pipistrello-nist_qc1/meta.yaml | 2 +- conda/artiq/meta.yaml | 2 +- setup.py | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/conda/artiq-kc705-nist_qc1/meta.yaml b/conda/artiq-kc705-nist_qc1/meta.yaml index bee5c234b..0b19cbb9d 100644 --- a/conda/artiq-kc705-nist_qc1/meta.yaml +++ b/conda/artiq-kc705-nist_qc1/meta.yaml @@ -23,5 +23,5 @@ requirements: about: home: http://m-labs.hk/artiq - license: 3-clause BSD + license: GPL summary: 'Bitstream, BIOS and runtime for NIST_QC1 on the KC705 board' diff --git a/conda/artiq-kc705-nist_qc2/meta.yaml b/conda/artiq-kc705-nist_qc2/meta.yaml index 1cbb86f2e..6ab132bba 100644 --- a/conda/artiq-kc705-nist_qc2/meta.yaml +++ b/conda/artiq-kc705-nist_qc2/meta.yaml @@ -23,5 +23,5 @@ requirements: about: home: http://m-labs.hk/artiq - license: 3-clause BSD + license: GPL summary: 'Bitstream, BIOS and runtime for NIST_QC2 on the KC705 board' diff --git a/conda/artiq-pipistrello-nist_qc1/meta.yaml b/conda/artiq-pipistrello-nist_qc1/meta.yaml index 254361e67..64988647d 100644 --- a/conda/artiq-pipistrello-nist_qc1/meta.yaml +++ b/conda/artiq-pipistrello-nist_qc1/meta.yaml @@ -23,5 +23,5 @@ requirements: about: home: http://m-labs.hk/artiq - license: 3-clause BSD + license: GPL summary: 'Bitstream, BIOS and runtime for NIST_QC1 on the Pipistrello board' diff --git a/conda/artiq/meta.yaml b/conda/artiq/meta.yaml index f6f24e2f3..9fa02839d 100644 --- a/conda/artiq/meta.yaml +++ b/conda/artiq/meta.yaml @@ -62,5 +62,5 @@ test: about: home: http://m-labs.hk/artiq - license: 3-clause BSD + license: GPL summary: 'ARTIQ (Advanced Real-Time Infrastructure for Quantum physics) is a next-generation control system for quantum information experiments. It is being developed in partnership with the Ion Storage Group at NIST, and its applicability reaches beyond ion trapping.' diff --git a/setup.py b/setup.py index 9ca327bde..68b7b9b30 100755 --- a/setup.py +++ b/setup.py @@ -56,7 +56,7 @@ setup( url="http://m-labs.hk/artiq", description="A control system for trapped-ion experiments", long_description=open("README.rst").read(), - license="BSD", + license="GPL", install_requires=requirements, extras_require={}, dependency_links=[ From b0ef0d205a9e39a7d3f609627b82b865f19cd734 Mon Sep 17 00:00:00 2001 From: Sebastien Bourdeauducq Date: Sat, 24 Oct 2015 09:57:42 +0800 Subject: [PATCH 46/73] gui/datasets: do not crash when a display data is unavailable --- artiq/gui/datasets.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/artiq/gui/datasets.py b/artiq/gui/datasets.py index 4af8a59ac..1c51dc387 100644 --- a/artiq/gui/datasets.py +++ b/artiq/gui/datasets.py @@ -100,8 +100,10 @@ class DatasetsDock(dockarea.Dock): return self.table_model def update_display_data(self, dsp): - dsp.update_data({k: self.table_model.backing_store[k][1] - for k in dsp.data_sources()}) + filtered_data = {k: self.table_model.backing_store[k][1] + for k in dsp.data_sources() + if k in self.table_model.backing_store} + dsp.update_data(filtered_data) def on_mod(self, mod): if mod["action"] == "init": From 3044a053e1880282711a6f4017cae8139eaf2f7c Mon Sep 17 00:00:00 2001 From: Sebastien Bourdeauducq Date: Sat, 24 Oct 2015 10:15:20 +0800 Subject: [PATCH 47/73] databases: assume empty dataset_db if file not present. Closes #156 --- .gitignore | 2 ++ artiq/master/databases.py | 5 ++++- doc/manual/getting_started_core.rst | 2 +- doc/manual/getting_started_mgmt.rst | 2 +- examples/master/dataset_db.pyon | 1 - examples/sim/dataset_db.pyon | 1 - 6 files changed, 8 insertions(+), 5 deletions(-) delete mode 100644 examples/master/dataset_db.pyon delete mode 100644 examples/sim/dataset_db.pyon diff --git a/.gitignore b/.gitignore index ce877f17f..50894b653 100644 --- a/.gitignore +++ b/.gitignore @@ -13,3 +13,5 @@ doc/manual/_build /.coverage artiq/test/results examples/master/results +examples/master/dataset_db.pyon +examples/sim/dataset_db.pyon diff --git a/artiq/master/databases.py b/artiq/master/databases.py index 40549683c..5e2c7d004 100644 --- a/artiq/master/databases.py +++ b/artiq/master/databases.py @@ -32,7 +32,10 @@ class DatasetDB(TaskObject): self.persist_file = persist_file self.autosave_period = autosave_period - file_data = pyon.load_file(self.persist_file) + try: + file_data = pyon.load_file(self.persist_file) + except FileNotFoundError: + file_data = dict() self.data = Notifier({k: (True, v) for k, v in file_data.items()}) def save(self): diff --git a/doc/manual/getting_started_core.rst b/doc/manual/getting_started_core.rst index a9e78518c..1f42d152d 100644 --- a/doc/manual/getting_started_core.rst +++ b/doc/manual/getting_started_core.rst @@ -22,7 +22,7 @@ As a very first step, we will turn on a LED on the core device. Create a file `` The central part of our code is our ``LED`` class, that derives from :class:`artiq.language.environment.EnvExperiment`. Among other features, ``EnvExperiment`` calls our ``build`` method and provides the ``setattr_device`` method that interfaces to the device database to create the appropriate device drivers and make those drivers accessible as ``self.core`` and ``self.led``. The ``@kernel`` decorator tells the system that the ``run`` method must be executed on the core device (instead of the host). The decorator uses ``self.core`` internally, which is why we request the core device using ``setattr_device`` like any other. -Copy the files ``device_db.pyon`` and ``dataset_db.pyon`` (containing the device and dataset databases) from the ``examples/master`` folder of ARTIQ into the same directory as ``led.py`` (alternatively, you can use the ``--device-db`` and ``--dataset-db`` options of ``artiq_run``). You can open the database files using a text editor - their contents are in a human-readable format. You will probably want to set the IP address of the core device in ``device_db.pyon`` so that the computer can connect to it (it is the ``host`` parameter of the ``comm`` entry). See :ref:`device-db` for more information. The example device database is designed for the NIST QC1 hardware on the KC705; see :ref:`board-ports` for RTIO channel assignments if you need to adapt the device database to a different hardware platform. +Copy the file ``device_db.pyon`` (containing the device database) from the ``examples/master`` folder of ARTIQ into the same directory as ``led.py`` (alternatively, you can use the ``--device-db`` option of ``artiq_run``). You can open PYON database files using a text editor - their contents are in a human-readable format. You will probably want to set the IP address of the core device in ``device_db.pyon`` so that the computer can connect to it (it is the ``host`` parameter of the ``comm`` entry). See :ref:`device-db` for more information. The example device database is designed for the NIST QC1 hardware on the KC705; see :ref:`board-ports` for RTIO channel assignments if you need to adapt the device database to a different hardware platform. .. note:: If the ``led`` device is a bidirectional TTL (i.e. ``TTLInOut`` instead of ``TTLOut``), you need to put it in output (driving) mode. Add the following at the beginning of ``run``: :: diff --git a/doc/manual/getting_started_mgmt.rst b/doc/manual/getting_started_mgmt.rst index 9ddf82f1f..c030ad69f 100644 --- a/doc/manual/getting_started_mgmt.rst +++ b/doc/manual/getting_started_mgmt.rst @@ -10,7 +10,7 @@ Starting your first experiment with the master In the previous tutorial, we used the ``artiq_run`` utility to execute our experiments, which is a simple stand-alone tool that bypasses the ARTIQ management system. We will now see how to run an experiment using the master (the central program in the management system that schedules and executes experiments) and the GUI client (that connects to the master and controls it). -First, create a folder ``~/artiq-master`` and copy the ``device_db.pyon`` and ``dataset_db.pyon`` (containing the device and dataset databases) found in the ``examples/master`` directory from the ARTIQ sources. The master uses those files in the same way as ``artiq_run``. +First, create a folder ``~/artiq-master`` and copy the file ``device_db.pyon`` (containing the device database) found in the ``examples/master`` directory from the ARTIQ sources. The master uses those files in the same way as ``artiq_run``. Then create a ``~/artiq-master/repository`` sub-folder to contain experiments. The master scans this ``repository`` folder to determine what experiments are available (the name of the folder can be changed using ``-r``). diff --git a/examples/master/dataset_db.pyon b/examples/master/dataset_db.pyon deleted file mode 100644 index ccee85030..000000000 --- a/examples/master/dataset_db.pyon +++ /dev/null @@ -1 +0,0 @@ -{"flopping_freq": 1499.9977914479744} diff --git a/examples/sim/dataset_db.pyon b/examples/sim/dataset_db.pyon deleted file mode 100644 index 0967ef424..000000000 --- a/examples/sim/dataset_db.pyon +++ /dev/null @@ -1 +0,0 @@ -{} From fcd29492f92405c1c184267d25a469e93582adc4 Mon Sep 17 00:00:00 2001 From: Sebastien Bourdeauducq Date: Sat, 24 Oct 2015 10:54:59 +0800 Subject: [PATCH 48/73] worker_db: support more types in HDF5 output. Closes #144. Closes #121 --- .gitignore | 1 + artiq/master/worker_db.py | 34 ++++++++++++++++++++++++++-------- artiq/test/h5types.py | 33 +++++++++++++++++++++++++++++++++ 3 files changed, 60 insertions(+), 8 deletions(-) create mode 100644 artiq/test/h5types.py diff --git a/.gitignore b/.gitignore index 50894b653..2cc1c622f 100644 --- a/.gitignore +++ b/.gitignore @@ -12,6 +12,7 @@ doc/manual/_build /*.egg-info /.coverage artiq/test/results +artiq/test/h5types.h5 examples/master/results examples/master/dataset_db.pyon examples/sim/dataset_db.pyon diff --git a/artiq/master/worker_db.py b/artiq/master/worker_db.py index 950ec147f..17dee5a1b 100644 --- a/artiq/master/worker_db.py +++ b/artiq/master/worker_db.py @@ -5,7 +5,7 @@ import os import time import re -import numpy +import numpy as np import h5py from artiq.protocols.sync_struct import Notifier @@ -119,7 +119,21 @@ def get_last_rid(): _type_to_hdf5 = { int: h5py.h5t.STD_I64BE, - float: h5py.h5t.IEEE_F64BE + float: h5py.h5t.IEEE_F64BE, + + np.int8: h5py.h5t.STD_I8BE, + np.int16: h5py.h5t.STD_I16BE, + np.int32: h5py.h5t.STD_I32BE, + np.int64: h5py.h5t.STD_I64BE, + + np.uint8: h5py.h5t.STD_U8BE, + np.uint16: h5py.h5t.STD_U16BE, + np.uint32: h5py.h5t.STD_U32BE, + np.uint64: h5py.h5t.STD_U64BE, + + np.float16: h5py.h5t.IEEE_F16BE, + np.float32: h5py.h5t.IEEE_F32BE, + np.float64: h5py.h5t.IEEE_F64BE } def result_dict_to_hdf5(f, rd): @@ -137,15 +151,19 @@ def result_dict_to_hdf5(f, rd): " HDF5 output".format(el_ty)) dataset = f.create_dataset(name, (len(data), ), el_ty_h5) dataset[:] = data - elif isinstance(data, numpy.ndarray): + elif isinstance(data, np.ndarray): f.create_dataset(name, data=data) else: ty = type(data) - try: - ty_h5 = _type_to_hdf5[ty] - except KeyError: - raise TypeError("Type {} is not supported for HDF5 output" - .format(ty)) + if ty is str: + ty_h5 = "S{}".format(len(data)) + data = data.encode() + else: + try: + ty_h5 = _type_to_hdf5[ty] + except KeyError: + raise TypeError("Type {} is not supported for HDF5 output" + .format(ty)) dataset = f.create_dataset(name, (), ty_h5) dataset[()] = data diff --git a/artiq/test/h5types.py b/artiq/test/h5types.py new file mode 100644 index 000000000..dd204aa66 --- /dev/null +++ b/artiq/test/h5types.py @@ -0,0 +1,33 @@ +import unittest + +import h5py +import numpy as np + +from artiq.master.worker_db import result_dict_to_hdf5 + + +class TypesCase(unittest.TestCase): + def test_types(self): + d = { + "int": 42, + "float": 42.0, + "string": "abcdef", + + "intlist": [1, 2, 3], + "floatlist": [1.0, 2.0, 3.0] + } + + for size in 8, 16, 32, 64: + signed = getattr(np, "int" + str(size)) + unsigned = getattr(np, "uint" + str(size)) + d["i"+str(size)] = signed(42) + d["u"+str(size)] = unsigned(42) + d["i{}list".format(size)] = [signed(x) for x in range(3)] + d["u{}list".format(size)] = [unsigned(x) for x in range(3)] + for size in 16, 32, 64: + ty = getattr(np, "float" + str(size)) + d["f"+str(size)] = ty(42) + d["f{}list".format(size)] = [ty(x) for x in range(3)] + + with h5py.File("h5types.h5", "w") as f: + result_dict_to_hdf5(f, d) From 53a9d7eb9afa34d41edf3fa61db372a90da99777 Mon Sep 17 00:00:00 2001 From: Sebastien Bourdeauducq Date: Sat, 24 Oct 2015 16:00:20 +0800 Subject: [PATCH 49/73] gui/explorer: more compact layout --- artiq/gui/explorer.py | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/artiq/gui/explorer.py b/artiq/gui/explorer.py index babf60833..d0518ac4b 100644 --- a/artiq/gui/explorer.py +++ b/artiq/gui/explorer.py @@ -238,32 +238,32 @@ class ExplorerDock(dockarea.Dock): self.datetime.setDate(QtCore.QDate.currentDate()) self.datetime.dateTimeChanged.connect(self.enable_duedate) self.datetime_en = QtGui.QCheckBox("Due date:") - grid.addWidget(self.datetime_en, 1, 0) - grid.addWidget(self.datetime, 1, 1) - - self.priority = QtGui.QSpinBox() - self.priority.setRange(-99, 99) - grid.addWidget(QtGui.QLabel("Priority:"), 1, 2) - grid.addWidget(self.priority, 1, 3) + grid.addWidget(self.datetime_en, 1, 0, colspan=2) + grid.addWidget(self.datetime, 1, 2, colspan=2) self.pipeline = QtGui.QLineEdit() self.pipeline.setText("main") - grid.addWidget(QtGui.QLabel("Pipeline:"), 2, 0) - grid.addWidget(self.pipeline, 2, 1) + grid.addWidget(QtGui.QLabel("Pipeline:"), 2, 0, colspan=2) + grid.addWidget(self.pipeline, 2, 2, colspan=2) + + self.priority = QtGui.QSpinBox() + self.priority.setRange(-99, 99) + grid.addWidget(QtGui.QLabel("Priority:"), 3, 0) + grid.addWidget(self.priority, 3, 1) self.flush = QtGui.QCheckBox("Flush") self.flush.setToolTip("Flush the pipeline before starting the experiment") - grid.addWidget(self.flush, 2, 2) + grid.addWidget(self.flush, 3, 2) self.log_level = QtGui.QComboBox() for item in "DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL": self.log_level.addItem(item) self.log_level.setCurrentIndex(1) self.log_level.setToolTip("Minimum level for log entry production") - grid.addWidget(self.log_level, 2, 3) + grid.addWidget(self.log_level, 3, 3) submit = QtGui.QPushButton("Submit") - grid.addWidget(submit, 3, 0, colspan=4) + grid.addWidget(submit, 4, 0, colspan=4) submit.clicked.connect(self.submit_clicked) self.argeditor = _ArgumentEditor(self.dialog_parent) From 5ae9bbc0cab48c030c08abab3e029683f3b9443a Mon Sep 17 00:00:00 2001 From: Sebastien Bourdeauducq Date: Sat, 24 Oct 2015 19:19:12 +0800 Subject: [PATCH 50/73] gui/explorer: add CTRL+S shortcut to submit --- artiq/gui/explorer.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/artiq/gui/explorer.py b/artiq/gui/explorer.py index d0518ac4b..eefacd999 100644 --- a/artiq/gui/explorer.py +++ b/artiq/gui/explorer.py @@ -263,6 +263,8 @@ class ExplorerDock(dockarea.Dock): grid.addWidget(self.log_level, 3, 3) submit = QtGui.QPushButton("Submit") + submit.setShortcut("CTRL+S") + submit.setToolTip("Schedule the selected experiment (CTRL+S)") grid.addWidget(submit, 4, 0, colspan=4) submit.clicked.connect(self.submit_clicked) From 62bcadfa44779b44e94aab6844a27ce5324565ca Mon Sep 17 00:00:00 2001 From: Sebastien Bourdeauducq Date: Sun, 25 Oct 2015 23:21:27 +0800 Subject: [PATCH 51/73] gui/explorer: change submit shortcut to CTRL+ENTER --- artiq/gui/explorer.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/artiq/gui/explorer.py b/artiq/gui/explorer.py index eefacd999..ab216fd64 100644 --- a/artiq/gui/explorer.py +++ b/artiq/gui/explorer.py @@ -263,8 +263,8 @@ class ExplorerDock(dockarea.Dock): grid.addWidget(self.log_level, 3, 3) submit = QtGui.QPushButton("Submit") - submit.setShortcut("CTRL+S") - submit.setToolTip("Schedule the selected experiment (CTRL+S)") + submit.setShortcut("CTRL+RETURN") + submit.setToolTip("Schedule the selected experiment (CTRL+ENTER)") grid.addWidget(submit, 4, 0, colspan=4) submit.clicked.connect(self.submit_clicked) From fe809f81abbf42a66220ebc906559376c2c7e6f2 Mon Sep 17 00:00:00 2001 From: Sebastien Bourdeauducq Date: Mon, 26 Oct 2015 00:21:51 +0800 Subject: [PATCH 52/73] protocols/pyon: make received numpy arrays writable --- artiq/protocols/pyon.py | 1 + 1 file changed, 1 insertion(+) diff --git a/artiq/protocols/pyon.py b/artiq/protocols/pyon.py index 471025b43..5c25e882c 100644 --- a/artiq/protocols/pyon.py +++ b/artiq/protocols/pyon.py @@ -145,6 +145,7 @@ def encode(x, pretty=False): def _nparray(shape, dtype, data): a = numpy.frombuffer(base64.b64decode(data), dtype=dtype) + a = a.copy() return a.reshape(shape) From a6d38b8a97f0a6a8387981cb637d6933a907d425 Mon Sep 17 00:00:00 2001 From: Sebastien Bourdeauducq Date: Mon, 26 Oct 2015 00:32:49 +0800 Subject: [PATCH 53/73] gui/datasets: fix source decoding --- artiq/gui/datasets.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/artiq/gui/datasets.py b/artiq/gui/datasets.py index 1c51dc387..fcaa36ac1 100644 --- a/artiq/gui/datasets.py +++ b/artiq/gui/datasets.py @@ -111,10 +111,10 @@ class DatasetsDock(dockarea.Dock): display.update_data(self.table_model.backing_store) return - if mod["action"] == "setitem": - source = mod["key"] - elif mod["path"]: + if mod["path"]: source = mod["path"][0] + elif mod["action"] == "setitem": + source = mod["key"] else: return From 24e728f075114186d947cd56edcf9e08ccaf5c86 Mon Sep 17 00:00:00 2001 From: Sebastien Bourdeauducq Date: Mon, 26 Oct 2015 00:33:11 +0800 Subject: [PATCH 54/73] examples/flopping_f_simulation: numpy outputs --- .../repository/flopping_f_simulation.py | 20 +++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/examples/master/repository/flopping_f_simulation.py b/examples/master/repository/flopping_f_simulation.py index 3624502c9..a2f0f04c9 100644 --- a/examples/master/repository/flopping_f_simulation.py +++ b/examples/master/repository/flopping_f_simulation.py @@ -37,16 +37,20 @@ class FloppingF(EnvExperiment): self.setattr_device("scheduler") def run(self): - frequency = self.set_dataset("flopping_f_frequency", [], + l = len(self.frequency_scan) + frequency = self.set_dataset("flopping_f_frequency", + np.full(l, np.nan), broadcast=True, save=False) - brightness = self.set_dataset("flopping_f_brightness", [], + brightness = self.set_dataset("flopping_f_brightness", + np.full(l, np.nan), broadcast=True) - self.set_dataset("flopping_f_fit", [], broadcast=True, save=False) + self.set_dataset("flopping_f_fit", np.full(l, np.nan), + broadcast=True, save=False) - for f in self.frequency_scan: + for i, f in enumerate(self.frequency_scan): m_brightness = model(f, self.F0) + self.noise_amplitude*random.random() - frequency.append(f) - brightness.append(m_brightness) + frequency[i] = f + brightness[i] = m_brightness time.sleep(0.1) self.scheduler.submit(self.scheduler.pipeline_name, self.scheduler.expid, self.scheduler.priority, time.time() + 20, False) @@ -57,11 +61,11 @@ class FloppingF(EnvExperiment): brightness = self.get_dataset("flopping_f_brightness") popt, pcov = curve_fit(model_numpy, frequency, brightness, - p0=[self.get_dataset("flopping_freq")]) + p0=[self.get_dataset("flopping_freq", 1500.0)]) perr = np.sqrt(np.diag(pcov)) if perr < 0.1: F0 = float(popt) self.set_dataset("flopping_freq", F0, persist=True, save=False) self.set_dataset("flopping_f_fit", - [model(x, F0) for x in frequency], + np.array([model(x, F0) for x in frequency]), broadcast=True, save=False) From f384142869279fabf39dca134e1f4ffc8c677870 Mon Sep 17 00:00:00 2001 From: whitequark Date: Mon, 26 Oct 2015 14:27:16 +0300 Subject: [PATCH 55/73] conda: work around conda/conda@1740. --- conda/artiq-kc705-nist_qc1/meta.yaml | 1 + conda/artiq-kc705-nist_qc2/meta.yaml | 1 + conda/artiq-pipistrello-nist_qc1/meta.yaml | 1 + 3 files changed, 3 insertions(+) diff --git a/conda/artiq-kc705-nist_qc1/meta.yaml b/conda/artiq-kc705-nist_qc1/meta.yaml index 0b19cbb9d..9a4bc6d45 100644 --- a/conda/artiq-kc705-nist_qc1/meta.yaml +++ b/conda/artiq-kc705-nist_qc1/meta.yaml @@ -19,6 +19,7 @@ requirements: - llvm-or1k - binutils-or1k-linux run: + - artiq - artiq {{ "{tag} py_{number}+git{hash}".format(tag=environ.get("GIT_DESCRIBE_TAG"), number=environ.get("GIT_DESCRIBE_NUMBER"), hash=environ.get("GIT_DESCRIBE_HASH")[1:]) if "GIT_DESCRIBE_TAG" in environ else "" }} about: diff --git a/conda/artiq-kc705-nist_qc2/meta.yaml b/conda/artiq-kc705-nist_qc2/meta.yaml index 6ab132bba..23af79a0c 100644 --- a/conda/artiq-kc705-nist_qc2/meta.yaml +++ b/conda/artiq-kc705-nist_qc2/meta.yaml @@ -19,6 +19,7 @@ requirements: - llvm-or1k - binutils-or1k-linux run: + - artiq - artiq {{ "{tag} py_{number}+git{hash}".format(tag=environ.get("GIT_DESCRIBE_TAG"), number=environ.get("GIT_DESCRIBE_NUMBER"), hash=environ.get("GIT_DESCRIBE_HASH")[1:]) if "GIT_DESCRIBE_TAG" in environ else "" }} about: diff --git a/conda/artiq-pipistrello-nist_qc1/meta.yaml b/conda/artiq-pipistrello-nist_qc1/meta.yaml index 64988647d..ad31d31cc 100644 --- a/conda/artiq-pipistrello-nist_qc1/meta.yaml +++ b/conda/artiq-pipistrello-nist_qc1/meta.yaml @@ -19,6 +19,7 @@ requirements: - llvm-or1k - binutils-or1k-linux run: + - artiq - artiq {{ "{tag} py_{number}+git{hash}".format(tag=environ.get("GIT_DESCRIBE_TAG"), number=environ.get("GIT_DESCRIBE_NUMBER"), hash=environ.get("GIT_DESCRIBE_HASH")[1:]) if "GIT_DESCRIBE_TAG" in environ else "" }} about: From 372bc906c1f2bb8f37d9c1452a746c6afa4fc653 Mon Sep 17 00:00:00 2001 From: whitequark Date: Mon, 26 Oct 2015 15:10:46 +0300 Subject: [PATCH 56/73] Revert "conda: work around conda/conda#1740." This just made conda recurse forever. This reverts commit f384142869279fabf39dca134e1f4ffc8c677870. --- conda/artiq-kc705-nist_qc1/meta.yaml | 1 - conda/artiq-kc705-nist_qc2/meta.yaml | 1 - conda/artiq-pipistrello-nist_qc1/meta.yaml | 1 - 3 files changed, 3 deletions(-) diff --git a/conda/artiq-kc705-nist_qc1/meta.yaml b/conda/artiq-kc705-nist_qc1/meta.yaml index 9a4bc6d45..0b19cbb9d 100644 --- a/conda/artiq-kc705-nist_qc1/meta.yaml +++ b/conda/artiq-kc705-nist_qc1/meta.yaml @@ -19,7 +19,6 @@ requirements: - llvm-or1k - binutils-or1k-linux run: - - artiq - artiq {{ "{tag} py_{number}+git{hash}".format(tag=environ.get("GIT_DESCRIBE_TAG"), number=environ.get("GIT_DESCRIBE_NUMBER"), hash=environ.get("GIT_DESCRIBE_HASH")[1:]) if "GIT_DESCRIBE_TAG" in environ else "" }} about: diff --git a/conda/artiq-kc705-nist_qc2/meta.yaml b/conda/artiq-kc705-nist_qc2/meta.yaml index 23af79a0c..6ab132bba 100644 --- a/conda/artiq-kc705-nist_qc2/meta.yaml +++ b/conda/artiq-kc705-nist_qc2/meta.yaml @@ -19,7 +19,6 @@ requirements: - llvm-or1k - binutils-or1k-linux run: - - artiq - artiq {{ "{tag} py_{number}+git{hash}".format(tag=environ.get("GIT_DESCRIBE_TAG"), number=environ.get("GIT_DESCRIBE_NUMBER"), hash=environ.get("GIT_DESCRIBE_HASH")[1:]) if "GIT_DESCRIBE_TAG" in environ else "" }} about: diff --git a/conda/artiq-pipistrello-nist_qc1/meta.yaml b/conda/artiq-pipistrello-nist_qc1/meta.yaml index ad31d31cc..64988647d 100644 --- a/conda/artiq-pipistrello-nist_qc1/meta.yaml +++ b/conda/artiq-pipistrello-nist_qc1/meta.yaml @@ -19,7 +19,6 @@ requirements: - llvm-or1k - binutils-or1k-linux run: - - artiq - artiq {{ "{tag} py_{number}+git{hash}".format(tag=environ.get("GIT_DESCRIBE_TAG"), number=environ.get("GIT_DESCRIBE_NUMBER"), hash=environ.get("GIT_DESCRIBE_HASH")[1:]) if "GIT_DESCRIBE_TAG" in environ else "" }} about: From 9f2ff32948bd978405194488bdfe9420b978f0ce Mon Sep 17 00:00:00 2001 From: Sebastien Bourdeauducq Date: Mon, 26 Oct 2015 23:41:05 +0800 Subject: [PATCH 57/73] test/sync_struct: nparray mutation --- artiq/test/sync_struct.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/artiq/test/sync_struct.py b/artiq/test/sync_struct.py index ece90360b..e5e950535 100644 --- a/artiq/test/sync_struct.py +++ b/artiq/test/sync_struct.py @@ -17,15 +17,17 @@ def write_test_data(test_dict): for key, value in enumerate(test_values): test_dict[key] = value test_dict[1.5] = 1.5 - test_dict["array"] = [] - test_dict["array"].append(42) - test_dict["array"].insert(1, 1) + test_dict["list"] = [] + test_dict["list"].append(42) + test_dict["list"].insert(1, 1) test_dict[100] = 0 test_dict[100] = 1 test_dict[101] = 1 test_dict.pop(101) test_dict[102] = 1 del test_dict[102] + test_dict["array"] = np.zeros(1) + test_dict["array"][0] = 10 test_dict["finished"] = True From 967d4eda634d9fc75e7d5c7888775872c5ab4bc4 Mon Sep 17 00:00:00 2001 From: Sebastien Bourdeauducq Date: Tue, 27 Oct 2015 17:59:34 +0800 Subject: [PATCH 58/73] gui: shortcut support --- artiq/gui/explorer.py | 90 +++++++++++++++++++++++++++----------- artiq/gui/shortcuts.py | 98 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 162 insertions(+), 26 deletions(-) create mode 100644 artiq/gui/shortcuts.py diff --git a/artiq/gui/explorer.py b/artiq/gui/explorer.py index ab216fd64..06dd9c3f6 100644 --- a/artiq/gui/explorer.py +++ b/artiq/gui/explorer.py @@ -9,6 +9,7 @@ from artiq.protocols.sync_struct import Subscriber from artiq.protocols import pyon from artiq.gui.tools import DictSyncModel from artiq.gui.scan import ScanController +from artiq.gui.shortcuts import ShortcutManager class _ExplistModel(DictSyncModel): @@ -122,14 +123,14 @@ _procty_to_entry = { class _ArgumentEditor(QtGui.QTreeWidget): - def __init__(self, dialog_parent): + def __init__(self, main_window): QtGui.QTreeWidget.__init__(self) self.setColumnCount(2) self.header().setResizeMode(QtGui.QHeaderView.ResizeToContents) self.header().setVisible(False) self.setSelectionMode(QtGui.QAbstractItemView.NoSelection) - self.dialog_parent = dialog_parent + self.main_window = main_window self._groups = dict() self.set_arguments([]) @@ -176,7 +177,7 @@ class _ArgumentEditor(QtGui.QTreeWidget): r[arg] = entry.get_argument_value() except Exception as e: if show_error_message: - msgbox = QtGui.QMessageBox(self.dialog_parent) + msgbox = QtGui.QMessageBox(self.main_window) msgbox.setWindowTitle("Error") msgbox.setText("Failed to obtain value for argument '{}':\n{}" .format(arg, str(e))) @@ -215,10 +216,10 @@ class _ArgumentEditor(QtGui.QTreeWidget): class ExplorerDock(dockarea.Dock): - def __init__(self, dialog_parent, status_bar, schedule_ctl): + def __init__(self, main_window, status_bar, schedule_ctl): dockarea.Dock.__init__(self, "Explorer", size=(1500, 500)) - self.dialog_parent = dialog_parent + self.main_window = main_window self.status_bar = status_bar self.schedule_ctl = schedule_ctl @@ -268,20 +269,27 @@ class ExplorerDock(dockarea.Dock): grid.addWidget(submit, 4, 0, colspan=4) submit.clicked.connect(self.submit_clicked) - self.argeditor = _ArgumentEditor(self.dialog_parent) + self.argeditor = _ArgumentEditor(self.main_window) self.splitter.addWidget(self.argeditor) self.splitter.setSizes([grid.minimumSizeHint().width(), 1000]) - self.state = dict() + self.argeditor_states = dict() + + self.shortcuts = ShortcutManager(self.main_window, self) + + self.el.setContextMenuPolicy(QtCore.Qt.ActionsContextMenu) + edit_shortcuts_action = QtGui.QAction("Edit shortcuts", self.el) + edit_shortcuts_action.triggered.connect(self.edit_shortcuts) + self.el.addAction(edit_shortcuts_action) def update_selection(self, selected, deselected): if deselected: - self.state[deselected] = self.argeditor.save_state() + self.argeditor_states[deselected] = self.argeditor.save_state() if selected: expinfo = self.explist_model.backing_store[selected] self.argeditor.set_arguments(expinfo["arguments"]) - if selected in self.state: - self.argeditor.restore_state(self.state[selected]) + if selected in self.argeditor_states: + self.argeditor.restore_state(self.argeditor_states[selected]) self.splitter.insertWidget(1, self.argeditor) self.selected_key = selected @@ -302,11 +310,20 @@ class ExplorerDock(dockarea.Dock): if idx: row = idx[0].row() key = self.explist_model.row_to_key[row] - self.state[key] = self.argeditor.save_state() - return self.state + self.argeditor_states[key] = self.argeditor.save_state() + return { + "argeditor": self.argeditor_states, + "shortcuts": self.shortcuts.save_state() + } def restore_state(self, state): - self.state = state + try: + argeditor_states = state["argeditor"] + shortcuts_state = state["shortcuts"] + except KeyError: + return + self.argeditor_states = argeditor_states + self.shortcuts.restore_state(shortcuts_state) def enable_duedate(self): self.datetime_en.setChecked(True) @@ -324,8 +341,8 @@ class ExplorerDock(dockarea.Dock): self.el.setModel(self.explist_model) return self.explist_model - async def submit(self, pipeline_name, file, class_name, arguments, - priority, due_date, flush): + async def submit_task(self, pipeline_name, file, class_name, arguments, + priority, due_date, flush): expid = { "log_level": getattr(logging, self.log_level.currentText()), "repo_rev": None, @@ -337,20 +354,41 @@ class ExplorerDock(dockarea.Dock): priority, due_date, flush) self.status_bar.showMessage("Submitted RID {}".format(rid)) + def submit(self, pipeline, key, priority, due_date, flush): + # TODO: refactor explorer and cleanup. + # Argument editors should immediately modify the global state. + expinfo = self.explist_model.backing_store[key] + if key == self.selected_key: + arguments = self.argeditor.get_argument_values(True) + if arguments is None: + # There has been an error. Displaying the error message box + # was done by argeditor. + return + else: + try: + arguments = self.argeditor_states[key]["argument_values"] + except KeyError: + arguments = dict() + asyncio.ensure_future(self.submit_task(self.pipeline.text(), + expinfo["file"], + expinfo["class_name"], + arguments, + priority, + due_date, + flush)) + def submit_clicked(self): if self.selected_key is not None: - expinfo = self.explist_model.backing_store[self.selected_key] if self.datetime_en.isChecked(): due_date = self.datetime.dateTime().toMSecsSinceEpoch()/1000 else: due_date = None - arguments = self.argeditor.get_argument_values(True) - if arguments is None: - return - asyncio.ensure_future(self.submit(self.pipeline.text(), - expinfo["file"], - expinfo["class_name"], - arguments, - self.priority.value(), - due_date, - self.flush.isChecked())) + self.submit(self.pipeline.text(), + self.selected_key, + self.priority.value(), + due_date, + self.flush.isChecked()) + + def edit_shortcuts(self): + experiments = sorted(self.explist_model.backing_store.keys()) + self.shortcuts.edit(experiments) diff --git a/artiq/gui/shortcuts.py b/artiq/gui/shortcuts.py new file mode 100644 index 000000000..82308f0be --- /dev/null +++ b/artiq/gui/shortcuts.py @@ -0,0 +1,98 @@ +from functools import partial + +from quamash import QtGui +try: + from quamash import QtWidgets + QShortcut = QtWidgets.QShortcut +except: + QShortcut = QtGui.QShortcut + + +class _ShortcutEditor(QtGui.QDialog): + def __init__(self, parent, experiments, shortcuts): + QtGui.QDialog.__init__(self, parent=parent) + self.setWindowTitle("Shortcuts") + + self.shortcuts = shortcuts + self.edit_widgets = dict() + + grid = QtGui.QGridLayout() + self.setLayout(grid) + + for n, title in enumerate(["Key", "Experiment", "Priority", "Pipeline"]): + label = QtGui.QLabel("" + title + "") + experiment.addItems(experiments) + experiment.setEditable(True) + experiment.setEditText( + existing_shortcut.get("experiment", "")) + + priority = QtGui.QSpinBox() + grid.addWidget(priority, row, 2) + priority.setRange(-99, 99) + priority.setValue(existing_shortcut.get("priority", 0)) + + pipeline = QtGui.QLineEdit() + grid.addWidget(pipeline, row, 3) + pipeline.setText(existing_shortcut.get("pipeline", "main")) + + self.edit_widgets[i] = { + "experiment": experiment, + "priority": priority, + "pipeline": pipeline + } + + buttons = QtGui.QDialogButtonBox( + QtGui.QDialogButtonBox.Ok | QtGui.QDialogButtonBox.Cancel) + grid.addWidget(buttons, 14, 0, 1, 4) + buttons.accepted.connect(self.accept) + buttons.rejected.connect(self.reject) + self.accepted.connect(self.on_accept) + + def on_accept(self): + for n, widgets in self.edit_widgets.items(): + self.shortcuts[n] = { + "experiment": widgets["experiment"].currentText(), + "priority": widgets["priority"].value(), + "pipeline": widgets["pipeline"].text() + } + + +class ShortcutManager: + def __init__(self, main_window, explorer): + for i in range(12): + shortcut = QShortcut("F" + str(i+1), main_window) + shortcut.activated.connect(partial(self._activated, i)) + self.main_window = main_window + self.explorer = explorer + self.shortcuts = dict() + + def edit(self, experiments): + dlg = _ShortcutEditor(self.main_window, experiments, self.shortcuts) + dlg.open() + + def _activated(self, nr): + info = self.shortcuts.get(nr, dict()) + experiment = info.get("experiment", "") + if experiment and experiment != "": + self.explorer.submit(info["pipeline"], experiment, + info["priority"], None, False) + + def save_state(self): + return self.shortcuts + + def restore_state(self, state): + self.shortcuts = state From 138ba7e2fab3bd474459271217d10903107276d4 Mon Sep 17 00:00:00 2001 From: Sebastien Bourdeauducq Date: Tue, 27 Oct 2015 18:10:31 +0800 Subject: [PATCH 59/73] gui: add addItems to populate log level comboboxes --- artiq/gui/explorer.py | 3 +-- artiq/gui/log.py | 3 +-- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/artiq/gui/explorer.py b/artiq/gui/explorer.py index 06dd9c3f6..7ee3e311e 100644 --- a/artiq/gui/explorer.py +++ b/artiq/gui/explorer.py @@ -257,8 +257,7 @@ class ExplorerDock(dockarea.Dock): grid.addWidget(self.flush, 3, 2) self.log_level = QtGui.QComboBox() - for item in "DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL": - self.log_level.addItem(item) + self.log_level.addItems(["DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL"]) self.log_level.setCurrentIndex(1) self.log_level.setToolTip("Minimum level for log entry production") grid.addWidget(self.log_level, 3, 3) diff --git a/artiq/gui/log.py b/artiq/gui/log.py index 3c4b1bd99..f5b4aa15e 100644 --- a/artiq/gui/log.py +++ b/artiq/gui/log.py @@ -100,8 +100,7 @@ class LogDock(dockarea.Dock): grid.layout.setColumnStretch(1, 0) grid.layout.setColumnStretch(2, 1) self.filterbox = QtGui.QComboBox() - for item in "DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL": - self.filterbox.addItem(item) + self.filterbox.addItems(["DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL"]) self.filterbox.setToolTip("Display entries at or above this level") grid.addWidget(self.filterbox, 0, 1) self.filterbox.currentIndexChanged.connect(self.filter_changed) From f872d92d2260cefc810c16b8d65a5994da1bbe1c Mon Sep 17 00:00:00 2001 From: Sebastien Bourdeauducq Date: Tue, 27 Oct 2015 18:20:11 +0800 Subject: [PATCH 60/73] gui/schedule: improve delete status bar message --- artiq/gui/schedule.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/artiq/gui/schedule.py b/artiq/gui/schedule.py index 1439dfa6d..5505bf5bf 100644 --- a/artiq/gui/schedule.py +++ b/artiq/gui/schedule.py @@ -104,5 +104,9 @@ class ScheduleDock(dockarea.Dock): if idx: row = idx[0].row() rid = self.table_model.row_to_key[row] - self.status_bar.showMessage("Deleted RID {}".format(rid)) + if graceful: + msg = "Requested termination of RID {}".format(rid) + else: + msg = "Deleted RID {}".format(rid) + self.status_bar.showMessage(msg) asyncio.ensure_future(self.delete(rid, graceful)) From 179416dac75de701a6e49b244eb9078c1be24db0 Mon Sep 17 00:00:00 2001 From: Sebastien Bourdeauducq Date: Tue, 27 Oct 2015 18:20:25 +0800 Subject: [PATCH 61/73] gui: delete shortcuts --- artiq/gui/schedule.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/artiq/gui/schedule.py b/artiq/gui/schedule.py index 5505bf5bf..42ae01a1d 100644 --- a/artiq/gui/schedule.py +++ b/artiq/gui/schedule.py @@ -75,9 +75,11 @@ class ScheduleDock(dockarea.Dock): self.table.setContextMenuPolicy(QtCore.Qt.ActionsContextMenu) request_termination_action = QtGui.QAction("Request termination", self.table) request_termination_action.triggered.connect(partial(self.delete_clicked, True)) + request_termination_action.setShortcut("DELETE") self.table.addAction(request_termination_action) delete_action = QtGui.QAction("Delete", self.table) delete_action.triggered.connect(partial(self.delete_clicked, False)) + delete_action.setShortcut("SHIFT+DELETE") self.table.addAction(delete_action) From d708002fb9df2cef3860af2cfc4cb4a32d9f78ca Mon Sep 17 00:00:00 2001 From: Sebastien Bourdeauducq Date: Wed, 28 Oct 2015 09:46:17 +0800 Subject: [PATCH 62/73] devices/pdq2: add RPC getter functions --- artiq/devices/pdq2/driver.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/artiq/devices/pdq2/driver.py b/artiq/devices/pdq2/driver.py index a8637cc5b..d51a14992 100644 --- a/artiq/devices/pdq2/driver.py +++ b/artiq/devices/pdq2/driver.py @@ -139,6 +139,12 @@ class Pdq2: self.num_channels = self.num_dacs * self.num_boards self.channels = [Channel() for i in range(self.num_channels)] + def get_num_boards(self): + return self.num_boards + + def get_num_channels(self): + return self.num_channels + def close(self): self.dev.close() del self.dev From 3789273e29bf0ab86c35cc5e42f22183dec39968 Mon Sep 17 00:00:00 2001 From: Sebastien Bourdeauducq Date: Wed, 28 Oct 2015 09:47:01 +0800 Subject: [PATCH 63/73] pdq2_client: remove init call --- artiq/frontend/pdq2_client.py | 1 - 1 file changed, 1 deletion(-) diff --git a/artiq/frontend/pdq2_client.py b/artiq/frontend/pdq2_client.py index 2f3c3b787..c0bef9aa5 100755 --- a/artiq/frontend/pdq2_client.py +++ b/artiq/frontend/pdq2_client.py @@ -55,7 +55,6 @@ def main(): args = get_argparser().parse_args() init_logger(args) dev = Client(args.server, args.port, "pdq2") - dev.init() if args.reset: dev.write(b"\x00\x00") # flush any escape From 4e35a247d1faacfe99310f10f8b65d49aad01e04 Mon Sep 17 00:00:00 2001 From: Sebastien Bourdeauducq Date: Wed, 28 Oct 2015 09:48:14 +0800 Subject: [PATCH 64/73] pdq2_client: remove unnecessary calls --- artiq/frontend/pdq2_client.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/artiq/frontend/pdq2_client.py b/artiq/frontend/pdq2_client.py index c0bef9aa5..d8c7955ed 100755 --- a/artiq/frontend/pdq2_client.py +++ b/artiq/frontend/pdq2_client.py @@ -65,8 +65,6 @@ def main(): dev.cmd("DCM", args.dcm) freq = 100e6 if args.dcm else 50e6 dev.set_freq(freq) - num_channels = dev.get_num_channels() - num_frames = dev.get_num_frames() times = eval(args.times, globals(), {}) voltages = eval(args.voltages, globals(), dict(t=times)) From 1ada15ae5daf65eabfc4acaab530ee03a941cec2 Mon Sep 17 00:00:00 2001 From: Sebastien Bourdeauducq Date: Wed, 28 Oct 2015 17:35:57 +0800 Subject: [PATCH 65/73] master: simplify worker/parent RPC --- artiq/master/worker.py | 5 ++--- artiq/master/worker_impl.py | 43 ++++++++++++++++--------------------- 2 files changed, 21 insertions(+), 27 deletions(-) diff --git a/artiq/master/worker.py b/artiq/master/worker.py index e6b42724f..7d732bfe9 100644 --- a/artiq/master/worker.py +++ b/artiq/master/worker.py @@ -160,8 +160,7 @@ class Worker: return True elif action == "pause": return False - del obj["action"] - if action == "create_watchdog": + elif action == "create_watchdog": func = self.create_watchdog elif action == "delete_watchdog": func = self.delete_watchdog @@ -172,7 +171,7 @@ class Worker: if getattr(func, "worker_pass_rid", False): func = partial(func, self.rid) try: - data = func(**obj) + data = func(*obj["args"], **obj["kwargs"]) reply = {"status": "ok", "data": data} except: reply = {"status": "failed", diff --git a/artiq/master/worker_impl.py b/artiq/master/worker_impl.py index 40189e374..722fcb75c 100644 --- a/artiq/master/worker_impl.py +++ b/artiq/master/worker_impl.py @@ -26,12 +26,9 @@ class ParentActionError(Exception): pass -def make_parent_action(action, argnames, exception=ParentActionError): - argnames = argnames.split() - def parent_action(*args): - request = {"action": action} - for argname, arg in zip(argnames, args): - request[argname] = arg +def make_parent_action(action, exception=ParentActionError): + def parent_action(*args, **kwargs): + request = {"action": action, "args": args, "kwargs": kwargs} put_object(request) reply = get_object() if "action" in reply: @@ -50,7 +47,7 @@ class LogForwarder: def __init__(self): self.buffer = "" - to_parent = staticmethod(make_parent_action("log", "message")) + to_parent = staticmethod(make_parent_action("log")) def write(self, data): self.buffer += data @@ -64,18 +61,18 @@ class LogForwarder: class ParentDeviceDB: - get_device_db = make_parent_action("get_device_db", "") - get = make_parent_action("get_device", "key", KeyError) + get_device_db = make_parent_action("get_device_db") + get = make_parent_action("get_device", KeyError) class ParentDatasetDB: - get = make_parent_action("get_dataset", "key", KeyError) - update = make_parent_action("update_dataset", "mod") + get = make_parent_action("get_dataset", KeyError) + update = make_parent_action("update_dataset") class Watchdog: - _create = make_parent_action("create_watchdog", "t") - _delete = make_parent_action("delete_watchdog", "wid") + _create = make_parent_action("create_watchdog") + _delete = make_parent_action("delete_watchdog") def __init__(self, t): self.t = t @@ -91,15 +88,14 @@ set_watchdog_factory(Watchdog) class Scheduler: - pause_noexc = staticmethod(make_parent_action("pause", "")) + pause_noexc = staticmethod(make_parent_action("pause")) def pause(self): if self.pause_noexc(): raise TerminationRequested - submit = staticmethod(make_parent_action("scheduler_submit", - "pipeline_name expid priority due_date flush")) - cancel = staticmethod(make_parent_action("scheduler_cancel", "rid")) + submit = staticmethod(make_parent_action("scheduler_submit")) + cancel = staticmethod(make_parent_action("scheduler_cancel")) def set_run_info(self, pipeline_name, expid, priority): self.pipeline_name = pipeline_name @@ -120,22 +116,21 @@ def get_exp(file, class_name): return getattr(module, class_name) -register_experiment = make_parent_action("register_experiment", - "class_name name arguments") +register_experiment = make_parent_action("register_experiment") class ExamineDeviceMgr: - get_device_db = make_parent_action("get_device_db", "") + get_device_db = make_parent_action("get_device_db") - def get(self, name): + def get(name): return None class DummyDatasetMgr: - def set(self, key, value, broadcast=False, persist=False, save=True): + def set(key, value, broadcast=False, persist=False, save=True): return None - def get(self, key): + def get(key): pass @@ -213,7 +208,7 @@ def main(): f.close() put_object({"action": "completed"}) elif action == "examine": - examine(ExamineDeviceMgr(), DummyDatasetMgr(), obj["file"]) + examine(ExamineDeviceMgr, DummyDatasetMgr, obj["file"]) put_object({"action": "completed"}) elif action == "terminate": break From 828b48ad89133ca108e8490d054c1e6cda1ea413 Mon Sep 17 00:00:00 2001 From: Sebastien Bourdeauducq Date: Wed, 28 Oct 2015 17:48:50 +0800 Subject: [PATCH 66/73] master/scheduler: reduce logging severity of worker exception backtraces to debug --- artiq/master/scheduler.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/artiq/master/scheduler.py b/artiq/master/scheduler.py index de4f80af6..11eb6384e 100644 --- a/artiq/master/scheduler.py +++ b/artiq/master/scheduler.py @@ -229,8 +229,8 @@ class PrepareStage(TaskObject): await run.prepare() except: logger.error("got worker exception in prepare stage, " - "deleting RID %d", - run.rid, exc_info=True) + "deleting RID %d", run.rid) + logger.debug("worker exception details", exc_info=True) self.delete_cb(run.rid) else: run.status = RunStatus.prepare_done @@ -279,8 +279,8 @@ class RunStage(TaskObject): completed = await run.run() except: logger.error("got worker exception in run stage, " - "deleting RID %d", - run.rid, exc_info=True) + "deleting RID %d", run.rid) + logger.debug("worker exception details", exc_info=True) self.delete_cb(run.rid) else: if completed: @@ -317,8 +317,8 @@ class AnalyzeStage(TaskObject): await run.write_results() except: logger.error("got worker exception in analyze stage, " - "deleting RID %d", - run.rid, exc_info=True) + "deleting RID %d", run.rid) + logger.debug("worker exception details", exc_info=True) self.delete_cb(run.rid) else: self.delete_cb(run.rid) From ec328cf5e18afdce104f47a9746a8ddd0835b317 Mon Sep 17 00:00:00 2001 From: Sebastien Bourdeauducq Date: Wed, 28 Oct 2015 17:50:48 +0800 Subject: [PATCH 67/73] master/worker_db: reduce exception backtrace clutter of get_dataset. Closes #168 --- artiq/master/worker_db.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/artiq/master/worker_db.py b/artiq/master/worker_db.py index 17dee5a1b..c0cbbe73d 100644 --- a/artiq/master/worker_db.py +++ b/artiq/master/worker_db.py @@ -191,7 +191,8 @@ class DatasetManager: try: return self.local[key] except KeyError: - return self.ddb.get(key) + pass + return self.ddb.get(key) def write_hdf5(self, f): result_dict_to_hdf5(f, self.local) From 40b4129c65645c37a88cf0d6be96117150a0f870 Mon Sep 17 00:00:00 2001 From: Sebastien Bourdeauducq Date: Wed, 28 Oct 2015 18:33:42 +0800 Subject: [PATCH 68/73] Only support scalars and numpy arrays in HDF5 output. Update documentation. Closes #145 --- artiq/language/environment.py | 10 ++++++++-- artiq/master/worker_db.py | 31 +++++++++++++++---------------- artiq/test/h5types.py | 16 ++++------------ 3 files changed, 27 insertions(+), 30 deletions(-) diff --git a/artiq/language/environment.py b/artiq/language/environment.py index bd7cea725..8628da8d5 100644 --- a/artiq/language/environment.py +++ b/artiq/language/environment.py @@ -208,9 +208,15 @@ class HasEnvironment: broadcast=False, persist=False, save=True): """Sets the contents and handling modes of a dataset. + If the dataset is broadcasted, it must be PYON-serializable. + If the dataset is saved, it must be a scalar (``bool``, ``int``, + ``float`` or NumPy scalar) or a NumPy array. + :param broadcast: the data is sent in real-time to the master, which - dispatches it. Returns a Notifier that can be used to mutate the dataset. - :param persist: the master should store the data on-disk. Implies broadcast. + dispatches it. Returns a Notifier that can be used to mutate the + dataset. + :param persist: the master should store the data on-disk. Implies + broadcast. :param save: the data is saved into the local storage of the current run (archived as a HDF5 file). """ diff --git a/artiq/master/worker_db.py b/artiq/master/worker_db.py index c0cbbe73d..e490c3396 100644 --- a/artiq/master/worker_db.py +++ b/artiq/master/worker_db.py @@ -138,21 +138,17 @@ _type_to_hdf5 = { def result_dict_to_hdf5(f, rd): for name, data in rd.items(): - if isinstance(data, list): - el_ty = type(data[0]) - for d in data: - if type(d) != el_ty: - raise TypeError("All list elements must have the same" - " type for HDF5 output") - try: - el_ty_h5 = _type_to_hdf5[el_ty] - except KeyError: - raise TypeError("List element type {} is not supported for" - " HDF5 output".format(el_ty)) - dataset = f.create_dataset(name, (len(data), ), el_ty_h5) - dataset[:] = data - elif isinstance(data, np.ndarray): - f.create_dataset(name, data=data) + flag = None + # beware: isinstance(True/False, int) == True + if isinstance(data, bool): + data = np.int8(data) + flag = "py_bool" + elif isinstance(data, int): + data = np.int64(data) + flag = "py_int" + + if isinstance(data, np.ndarray): + dataset = f.create_dataset(name, data=data) else: ty = type(data) if ty is str: @@ -163,10 +159,13 @@ def result_dict_to_hdf5(f, rd): ty_h5 = _type_to_hdf5[ty] except KeyError: raise TypeError("Type {} is not supported for HDF5 output" - .format(ty)) + .format(ty)) from None dataset = f.create_dataset(name, (), ty_h5) dataset[()] = data + if flag is not None: + dataset.attrs[flag] = np.int8(1) + class DatasetManager: def __init__(self, ddb): diff --git a/artiq/test/h5types.py b/artiq/test/h5types.py index dd204aa66..4a3b1ca19 100644 --- a/artiq/test/h5types.py +++ b/artiq/test/h5types.py @@ -9,25 +9,17 @@ from artiq.master.worker_db import result_dict_to_hdf5 class TypesCase(unittest.TestCase): def test_types(self): d = { + "bool": True, "int": 42, "float": 42.0, "string": "abcdef", - - "intlist": [1, 2, 3], - "floatlist": [1.0, 2.0, 3.0] } for size in 8, 16, 32, 64: - signed = getattr(np, "int" + str(size)) - unsigned = getattr(np, "uint" + str(size)) - d["i"+str(size)] = signed(42) - d["u"+str(size)] = unsigned(42) - d["i{}list".format(size)] = [signed(x) for x in range(3)] - d["u{}list".format(size)] = [unsigned(x) for x in range(3)] + d["i"+str(size)] = getattr(np, "int" + str(size))(42) + d["u"+str(size)] = getattr(np, "uint" + str(size))(42) for size in 16, 32, 64: - ty = getattr(np, "float" + str(size)) - d["f"+str(size)] = ty(42) - d["f{}list".format(size)] = [ty(x) for x in range(3)] + d["f"+str(size)] = getattr(np, "float" + str(size))(42) with h5py.File("h5types.h5", "w") as f: result_dict_to_hdf5(f, d) From 0d53f7ab0d904d941bd6f2fb7841372aae94e5ea Mon Sep 17 00:00:00 2001 From: Sebastien Bourdeauducq Date: Wed, 28 Oct 2015 20:57:28 +0800 Subject: [PATCH 69/73] ignore ProcessLookupError when killing subprocesses. Closes #167 --- artiq/frontend/artiq_ctlmgr.py | 10 ++++++++-- artiq/master/worker.py | 10 ++++++++-- 2 files changed, 16 insertions(+), 4 deletions(-) diff --git a/artiq/frontend/artiq_ctlmgr.py b/artiq/frontend/artiq_ctlmgr.py index 6e2827d35..ad9071eda 100755 --- a/artiq/frontend/artiq_ctlmgr.py +++ b/artiq/frontend/artiq_ctlmgr.py @@ -134,14 +134,20 @@ class Controller: except: logger.warning("Controller %s did not respond to terminate " "command, killing", self.name) - self.process.kill() + try: + self.process.kill() + except ProcessLookupError: + pass try: await asyncio.wait_for(self.process.wait(), self.term_timeout) except: logger.warning("Controller %s failed to exit, killing", self.name) - self.process.kill() + try: + self.process.kill() + except ProcessLookupError: + pass await self.process.wait() logger.debug("Controller %s terminated", self.name) diff --git a/artiq/master/worker.py b/artiq/master/worker.py index 7d732bfe9..9ff40be7d 100644 --- a/artiq/master/worker.py +++ b/artiq/master/worker.py @@ -94,14 +94,20 @@ class Worker: except: logger.warning("failed to send terminate command to worker" " (RID %s), killing", self.rid, exc_info=True) - self.process.kill() + try: + self.process.kill() + except ProcessLookupError: + pass await self.process.wait() return try: await asyncio.wait_for(self.process.wait(), term_timeout) except asyncio.TimeoutError: logger.warning("worker did not exit (RID %s), killing", self.rid) - self.process.kill() + try: + self.process.kill() + except ProcessLookupError: + pass await self.process.wait() else: logger.debug("worker exited gracefully (RID %s)", self.rid) From c0e040c4b90562a23fbc053449862cce1379191f Mon Sep 17 00:00:00 2001 From: whitequark Date: Thu, 29 Oct 2015 01:24:32 +0300 Subject: [PATCH 70/73] conda: give up on build strings in dependencies. --- conda/artiq-kc705-nist_qc1/meta.yaml | 2 +- conda/artiq-kc705-nist_qc2/meta.yaml | 2 +- conda/artiq-pipistrello-nist_qc1/meta.yaml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/conda/artiq-kc705-nist_qc1/meta.yaml b/conda/artiq-kc705-nist_qc1/meta.yaml index 0b19cbb9d..765cb2c96 100644 --- a/conda/artiq-kc705-nist_qc1/meta.yaml +++ b/conda/artiq-kc705-nist_qc1/meta.yaml @@ -19,7 +19,7 @@ requirements: - llvm-or1k - binutils-or1k-linux run: - - artiq {{ "{tag} py_{number}+git{hash}".format(tag=environ.get("GIT_DESCRIBE_TAG"), number=environ.get("GIT_DESCRIBE_NUMBER"), hash=environ.get("GIT_DESCRIBE_HASH")[1:]) if "GIT_DESCRIBE_TAG" in environ else "" }} + - artiq 0.0 about: home: http://m-labs.hk/artiq diff --git a/conda/artiq-kc705-nist_qc2/meta.yaml b/conda/artiq-kc705-nist_qc2/meta.yaml index 6ab132bba..166163a1c 100644 --- a/conda/artiq-kc705-nist_qc2/meta.yaml +++ b/conda/artiq-kc705-nist_qc2/meta.yaml @@ -19,7 +19,7 @@ requirements: - llvm-or1k - binutils-or1k-linux run: - - artiq {{ "{tag} py_{number}+git{hash}".format(tag=environ.get("GIT_DESCRIBE_TAG"), number=environ.get("GIT_DESCRIBE_NUMBER"), hash=environ.get("GIT_DESCRIBE_HASH")[1:]) if "GIT_DESCRIBE_TAG" in environ else "" }} + - artiq 0.0 about: home: http://m-labs.hk/artiq diff --git a/conda/artiq-pipistrello-nist_qc1/meta.yaml b/conda/artiq-pipistrello-nist_qc1/meta.yaml index 64988647d..11a62058a 100644 --- a/conda/artiq-pipistrello-nist_qc1/meta.yaml +++ b/conda/artiq-pipistrello-nist_qc1/meta.yaml @@ -19,7 +19,7 @@ requirements: - llvm-or1k - binutils-or1k-linux run: - - artiq {{ "{tag} py_{number}+git{hash}".format(tag=environ.get("GIT_DESCRIBE_TAG"), number=environ.get("GIT_DESCRIBE_NUMBER"), hash=environ.get("GIT_DESCRIBE_HASH")[1:]) if "GIT_DESCRIBE_TAG" in environ else "" }} + - artiq 0.0 about: home: http://m-labs.hk/artiq From 32c95f24d0f83b6e4cff310e787727460c3c07eb Mon Sep 17 00:00:00 2001 From: Sebastien Bourdeauducq Date: Thu, 29 Oct 2015 09:34:41 +0800 Subject: [PATCH 71/73] worker: reduce some logging levels --- artiq/master/worker.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/artiq/master/worker.py b/artiq/master/worker.py index 9ff40be7d..f08679239 100644 --- a/artiq/master/worker.py +++ b/artiq/master/worker.py @@ -92,8 +92,8 @@ class Worker: try: await self._send(obj, cancellable=False) except: - logger.warning("failed to send terminate command to worker" - " (RID %s), killing", self.rid, exc_info=True) + logger.debug("failed to send terminate command to worker" + " (RID %s), killing", self.rid, exc_info=True) try: self.process.kill() except ProcessLookupError: @@ -103,14 +103,15 @@ class Worker: try: await asyncio.wait_for(self.process.wait(), term_timeout) except asyncio.TimeoutError: - logger.warning("worker did not exit (RID %s), killing", self.rid) + logger.debug("worker did not exit by itself (RID %s), killing", + self.rid) try: self.process.kill() except ProcessLookupError: pass await self.process.wait() else: - logger.debug("worker exited gracefully (RID %s)", self.rid) + logger.debug("worker exited by itself (RID %s)", self.rid) finally: self.io_lock.release() From f0eed1182ac4d30c7311c94eb1c76357d5ec1543 Mon Sep 17 00:00:00 2001 From: Sebastien Bourdeauducq Date: Thu, 29 Oct 2015 09:42:45 +0800 Subject: [PATCH 72/73] protocols/pyon: improve error reporting of non-serializable type --- artiq/protocols/pyon.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/artiq/protocols/pyon.py b/artiq/protocols/pyon.py index 5c25e882c..ee50af99f 100644 --- a/artiq/protocols/pyon.py +++ b/artiq/protocols/pyon.py @@ -132,7 +132,10 @@ class _Encoder: return r def encode(self, x): - return getattr(self, "encode_" + _encode_map[type(x)])(x) + ty = _encode_map.get(type(x), None) + if ty is None: + raise TypeError(repr(x) + " is not PYON serializable") + return getattr(self, "encode_" + ty)(x) def encode(x, pretty=False): From 62419c0952d1e10151197eeedd7a5c0f06d08af1 Mon Sep 17 00:00:00 2001 From: whitequark Date: Fri, 30 Oct 2015 04:18:42 +0300 Subject: [PATCH 73/73] manual: update installing.rst. --- doc/manual/installing.rst | 31 +++++++++++++++++++++++++++---- 1 file changed, 27 insertions(+), 4 deletions(-) diff --git a/doc/manual/installing.rst b/doc/manual/installing.rst index 6d2498121..263f5fceb 100644 --- a/doc/manual/installing.rst +++ b/doc/manual/installing.rst @@ -29,7 +29,7 @@ If it prints the help of the ``conda`` command, your install is OK. If not, then make sure your ``$PATH`` environment variable contains the path to anaconda3/bin (or miniconda3/bin):: $ echo $PATH - /home/..../miniconda3/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/usr/games:/usr/local/games + /home/.../miniconda3/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin If your ``$PATH`` misses reference the miniconda3/bin or anaconda3/bin you can fix this by typing:: @@ -38,14 +38,37 @@ If your ``$PATH`` misses reference the miniconda3/bin or anaconda3/bin you can f Installing the host side software ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -For this, you need to add our binstar repository to your conda configuration:: +For this, you need to add our Anaconda repository to your conda configuration:: $ conda config --add channels http://conda.anaconda.org/m-labs/channel/main $ conda config --add channels http://conda.anaconda.org/m-labs/channel/dev -Then you can install the ARTIQ package, it will pull all the necessary dependencies:: +Then you can install the ARTIQ package, it will pull all the necessary dependencies. - $ conda install artiq +* For the Pipistrello board:: + + $ ENV=$(date +artiq-%Y-%m-%d); conda create -n $ENV artiq-pipistrello-nist_qc1; \ + echo "Created environment $ENV for ARTIQ" + +* For the KC705 board:: + + $ ENV=$(date +artiq-%Y-%m-%d); conda create -n $ENV artiq-kc705-nist_qc1 artiq-kc705-nist_qc2; \ + echo "Created environment $ENV for ARTIQ" + +This creates a new Conda "environment" (i.e. an isolated installation) and prints its name. +If you ever need to upgrade ARTIQ, it is advised to install it again +in a new environment so that you can roll back to a version that is known to +work correctly. + +After this, add the newly created environment to your ``$PATH``. This can be easily +done using the following command:: + + $ source activate artiq-[date] + +You will need to invoke this command in every new shell. When in doubt, you can list +the existing environments using:: + + $ conda env list Preparing the core device FPGA board ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^