From 34b9f7e90119b9d1c04d9b58f11cbe0129d99663 Mon Sep 17 00:00:00 2001 From: Sebastien Bourdeauducq Date: Sat, 3 Oct 2015 13:55:57 +0800 Subject: [PATCH 01/53] examples/speed_benchmark: remove decimals on nruns --- examples/master/repository/speed_benchmark.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/master/repository/speed_benchmark.py b/examples/master/repository/speed_benchmark.py index 6face8d55..df2d3c630 100644 --- a/examples/master/repository/speed_benchmark.py +++ b/examples/master/repository/speed_benchmark.py @@ -78,7 +78,7 @@ class SpeedBenchmark(EnvExperiment): "CoreSend100Ints", "CoreSend1MB", "CorePrimes"])) - self.attr_argument("nruns", NumberValue(10, min=1, max=1000)) + self.attr_argument("nruns", NumberValue(10, min=1, max=1000, ndecimals=0)) self.attr_device("core") self.attr_device("scheduler") From cd3107ba750d42eab97260a0f2dc08cd164838bb Mon Sep 17 00:00:00 2001 From: Sebastien Bourdeauducq Date: Sat, 3 Oct 2015 13:59:18 +0800 Subject: [PATCH 02/53] do not use deprecated asyncio.JoinableQueue --- artiq/master/scheduler.py | 2 +- conda/artiq/meta.yaml | 4 ++-- doc/manual/installing.rst | 2 +- setup.py | 4 ++-- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/artiq/master/scheduler.py b/artiq/master/scheduler.py index 0e1b0f1b6..d7b931f42 100644 --- a/artiq/master/scheduler.py +++ b/artiq/master/scheduler.py @@ -348,7 +348,7 @@ class Pipeline: class Deleter(TaskObject): def __init__(self, pipelines): self._pipelines = pipelines - self._queue = asyncio.JoinableQueue() + self._queue = asyncio.Queue() def delete(self, rid): logger.debug("delete request for RID %d", rid) diff --git a/conda/artiq/meta.yaml b/conda/artiq/meta.yaml index 708aea045..c24da2c4b 100644 --- a/conda/artiq/meta.yaml +++ b/conda/artiq/meta.yaml @@ -28,14 +28,14 @@ build: requirements: build: - - python >=3.4.3 + - python >=3.4.4 - setuptools - numpy - migen - pyelftools - binutils-or1k-linux run: - - python >=3.4.3 + - python >=3.4.4 - llvmlite-artiq - scipy - numpy diff --git a/doc/manual/installing.rst b/doc/manual/installing.rst index 0c7310a9f..312e68a6b 100644 --- a/doc/manual/installing.rst +++ b/doc/manual/installing.rst @@ -369,7 +369,7 @@ This command installs all the required packages: :: $ sudo apt-get install build-essential autotools-dev file git patch perl xutils-dev python3-pip texinfo flex bison libmpc-dev python3-serial python3-dateutil python3-prettytable python3-setuptools python3-numpy python3-scipy python3-sphinx python3-h5py python3-dev python-dev subversion cmake libusb-dev libftdi-dev pkg-config libffi-dev libgit2-dev -Note that ARTIQ requires Python 3.4.3 or above. +Note that ARTIQ requires Python 3.4.4 or above. To set user permissions on the JTAG and serial ports of the Pipistrello, create a ``/etc/udev/rules.d/30-usb-papilio.rules`` file containing the following: :: diff --git a/setup.py b/setup.py index ec1603748..34e112884 100755 --- a/setup.py +++ b/setup.py @@ -5,8 +5,8 @@ import sys import os -if sys.version_info[:3] < (3, 4, 3): - raise Exception("You need at least Python 3.4.3 to run ARTIQ") +if sys.version_info[:3] < (3, 4, 4): + raise Exception("You need Python 3.4.4+") class PushDocCommand(Command): From 4b2a99b0909a4d2353742484ddb8a7b8692b9cd1 Mon Sep 17 00:00:00 2001 From: Sebastien Bourdeauducq Date: Sat, 3 Oct 2015 14:00:48 +0800 Subject: [PATCH 03/53] indent --- examples/master/repository/speed_benchmark.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/examples/master/repository/speed_benchmark.py b/examples/master/repository/speed_benchmark.py index df2d3c630..1105d7f4c 100644 --- a/examples/master/repository/speed_benchmark.py +++ b/examples/master/repository/speed_benchmark.py @@ -71,8 +71,8 @@ class SpeedBenchmark(EnvExperiment): """Speed benchmark""" def build(self): self.attr_argument("mode", EnumerationValue(["Single experiment", - "With pause", - "With scheduler"])) + "With pause", + "With scheduler"])) self.attr_argument("payload", EnumerationValue(["NOP", "CoreNOP", "CoreSend100Ints", From 125503139e62ba7f1b33925ee04dacc9797cea45 Mon Sep 17 00:00:00 2001 From: Sebastien Bourdeauducq Date: Sat, 3 Oct 2015 14:33:18 +0800 Subject: [PATCH 04/53] remove workaround for Python bug in asyncio process.wait(). Requires Python 3.5. Closes #58 --- artiq/frontend/artiq_ctlmgr.py | 10 +++++----- artiq/master/worker.py | 10 ++++------ artiq/tools.py | 23 ----------------------- conda/artiq/meta.yaml | 4 ++-- doc/manual/installing.rst | 8 ++++---- setup.py | 4 ++-- 6 files changed, 17 insertions(+), 42 deletions(-) diff --git a/artiq/frontend/artiq_ctlmgr.py b/artiq/frontend/artiq_ctlmgr.py index 878d6e649..273cae9af 100755 --- a/artiq/frontend/artiq_ctlmgr.py +++ b/artiq/frontend/artiq_ctlmgr.py @@ -11,7 +11,7 @@ import socket from artiq.protocols.sync_struct import Subscriber from artiq.protocols.pc_rpc import AsyncioClient, Server from artiq.tools import verbosity_args, init_logger -from artiq.tools import TaskObject, asyncio_process_wait_timeout, Condition +from artiq.tools import TaskObject, Condition logger = logging.getLogger(__name__) @@ -88,8 +88,8 @@ class Controller: def _wait_and_ping(self): while True: try: - yield from asyncio_process_wait_timeout(self.process, - self.ping_timer) + yield from asyncio.wait_for(self.process.wait(), + self.ping_timer) except asyncio.TimeoutError: logger.debug("pinging controller %s", self.name) ok = yield from self._ping() @@ -137,8 +137,8 @@ class Controller: "command, killing", self.name) self.process.kill() try: - yield from asyncio_process_wait_timeout(self.process, - self.term_timeout) + yield from asyncio.wait_for(self.process.wait(), + self.term_timeout) except: logger.warning("Controller %s failed to exit, killing", self.name) diff --git a/artiq/master/worker.py b/artiq/master/worker.py index 100b4e4ee..1bb36f94d 100644 --- a/artiq/master/worker.py +++ b/artiq/master/worker.py @@ -7,8 +7,7 @@ import time from functools import partial from artiq.protocols import pyon -from artiq.tools import (asyncio_process_wait_timeout, asyncio_process_wait, - asyncio_wait_or_cancel) +from artiq.tools import asyncio_wait_or_cancel logger = logging.getLogger(__name__) @@ -97,15 +96,14 @@ class Worker: logger.warning("failed to send terminate command to worker" " (RID %s), killing", self.rid, exc_info=True) self.process.kill() - yield from asyncio_process_wait(self.process) + yield from self.process.wait() return try: - yield from asyncio_process_wait_timeout(self.process, - term_timeout) + yield from asyncio.wait_for(self.process.wait(), term_timeout) except asyncio.TimeoutError: logger.warning("worker did not exit (RID %s), killing", self.rid) self.process.kill() - yield from asyncio_process_wait(self.process) + yield from self.process.wait() else: logger.debug("worker exited gracefully (RID %s)", self.rid) finally: diff --git a/artiq/tools.py b/artiq/tools.py index 499341d04..321c87bfe 100644 --- a/artiq/tools.py +++ b/artiq/tools.py @@ -88,29 +88,6 @@ def exc_to_warning(coro): exc_info=True) -@asyncio.coroutine -def asyncio_process_wait_timeout(process, timeout): - # In Python < 3.5, asyncio.wait_for(process.wait(), ... - # causes a futures.InvalidStateError inside asyncio if and when the - # process terminates after the timeout. - # Work around this problem. - @asyncio.coroutine - def process_wait_returncode_timeout(): - while True: - if process.returncode is not None: - break - yield from asyncio.sleep(0.1) - yield from asyncio.wait_for(process_wait_returncode_timeout(), - timeout=timeout) - -@asyncio.coroutine -def asyncio_process_wait(process): - r = True - while r: - f, p = yield from asyncio.wait([process.stdout.read(1024)]) - r = f.pop().result() - - @asyncio.coroutine def asyncio_wait_or_cancel(fs, **kwargs): fs = [asyncio.async(f) for f in fs] diff --git a/conda/artiq/meta.yaml b/conda/artiq/meta.yaml index c24da2c4b..6dacc0293 100644 --- a/conda/artiq/meta.yaml +++ b/conda/artiq/meta.yaml @@ -28,14 +28,14 @@ build: requirements: build: - - python >=3.4.4 + - python >=3.5.0 - setuptools - numpy - migen - pyelftools - binutils-or1k-linux run: - - python >=3.4.4 + - python >=3.5.0 - llvmlite-artiq - scipy - numpy diff --git a/doc/manual/installing.rst b/doc/manual/installing.rst index 312e68a6b..8cb1727a2 100644 --- a/doc/manual/installing.rst +++ b/doc/manual/installing.rst @@ -13,9 +13,9 @@ Installing using conda Installing Anaconda or Miniconda ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -* You can either install Anaconda (chose Python 3.4) from https://store.continuum.io/cshop/anaconda/ +* You can either install Anaconda (chose Python 3.5) from https://store.continuum.io/cshop/anaconda/ -* Or install the more minimalistic Miniconda (chose Python 3.4) from http://conda.pydata.org/miniconda.html +* Or install the more minimalistic Miniconda (chose Python 3.5) from http://conda.pydata.org/miniconda.html .. warning:: If you are installing on Windows, chose the Windows 32-bit version regardless of whether you have @@ -148,7 +148,7 @@ These steps are required to generate bitstream (``.bit``) files, build the MiSoC $ python3 setup.py develop --user .. note:: - The options ``develop`` and ``--user`` are for setup.py to install Migen in ``~/.local/lib/python3.4``. + The options ``develop`` and ``--user`` are for setup.py to install Migen in ``~/.local/lib/python3.5``. .. _install-xc3sprog: @@ -369,7 +369,7 @@ This command installs all the required packages: :: $ sudo apt-get install build-essential autotools-dev file git patch perl xutils-dev python3-pip texinfo flex bison libmpc-dev python3-serial python3-dateutil python3-prettytable python3-setuptools python3-numpy python3-scipy python3-sphinx python3-h5py python3-dev python-dev subversion cmake libusb-dev libftdi-dev pkg-config libffi-dev libgit2-dev -Note that ARTIQ requires Python 3.4.4 or above. +Note that ARTIQ requires Python 3.5.0 or above. To set user permissions on the JTAG and serial ports of the Pipistrello, create a ``/etc/udev/rules.d/30-usb-papilio.rules`` file containing the following: :: diff --git a/setup.py b/setup.py index 34e112884..9ca327bde 100755 --- a/setup.py +++ b/setup.py @@ -5,8 +5,8 @@ import sys import os -if sys.version_info[:3] < (3, 4, 4): - raise Exception("You need Python 3.4.4+") +if sys.version_info[:3] < (3, 5, 0): + raise Exception("You need Python 3.5.0+") class PushDocCommand(Command): From b117b9320d0d8117f5d5de703dd7af30a3de3b7b Mon Sep 17 00:00:00 2001 From: Sebastien Bourdeauducq Date: Sat, 3 Oct 2015 14:37:02 +0800 Subject: [PATCH 05/53] asyncio.async -> asyncio.ensure_future --- artiq/frontend/artiq_gui.py | 2 +- artiq/gui/explorer.py | 11 +++++++---- artiq/gui/schedule.py | 2 +- artiq/master/repository.py | 2 +- artiq/test/sync_struct.py | 4 ++-- artiq/tools.py | 4 ++-- 6 files changed, 14 insertions(+), 11 deletions(-) diff --git a/artiq/frontend/artiq_gui.py b/artiq/frontend/artiq_gui.py index f6fbf7ebb..f6f9db57c 100755 --- a/artiq/frontend/artiq_gui.py +++ b/artiq/frontend/artiq_gui.py @@ -132,7 +132,7 @@ def main(): args.server, args.port_control, "master_pdb")) atexit.register(lambda: pdb.close_rpc()) def _get_parameter(k, v): - asyncio.async(pdb.set(k, v)) + asyncio.ensure_future(pdb.set(k, v)) d_console = ConsoleDock( d_params.get_parameter, _get_parameter, diff --git a/artiq/gui/explorer.py b/artiq/gui/explorer.py index bed58ae39..bf2fc3e69 100644 --- a/artiq/gui/explorer.py +++ b/artiq/gui/explorer.py @@ -338,7 +338,10 @@ class ExplorerDock(dockarea.Dock): arguments = self.argeditor.get_argument_values(True) if arguments is None: return - asyncio.async(self.submit(self.pipeline.text(), - expinfo["file"], expinfo["class_name"], - arguments, self.priority.value(), - due_date, self.flush.isChecked())) + asyncio.ensure_future(self.submit(self.pipeline.text(), + expinfo["file"], + expinfo["class_name"], + arguments, + self.priority.value(), + due_date, + self.flush.isChecked())) diff --git a/artiq/gui/schedule.py b/artiq/gui/schedule.py index ab11714c1..21ab02b88 100644 --- a/artiq/gui/schedule.py +++ b/artiq/gui/schedule.py @@ -99,4 +99,4 @@ class ScheduleDock(dockarea.Dock): row = idx[0].row() rid = self.table_model.row_to_key[row] self.status_bar.showMessage("Deleted RID {}".format(rid)) - asyncio.async(self.delete(rid)) + asyncio.ensure_future(self.delete(rid)) diff --git a/artiq/master/repository.py b/artiq/master/repository.py index 556232014..6478570c8 100644 --- a/artiq/master/repository.py +++ b/artiq/master/repository.py @@ -86,7 +86,7 @@ class Repository: self._scanning = False def scan_async(self, new_cur_rev=None): - asyncio.async(exc_to_warning(self.scan(new_cur_rev))) + asyncio.ensure_future(exc_to_warning(self.scan(new_cur_rev))) class FilesystemBackend: diff --git a/artiq/test/sync_struct.py b/artiq/test/sync_struct.py index d8b229d7e..3f30062ff 100644 --- a/artiq/test/sync_struct.py +++ b/artiq/test/sync_struct.py @@ -59,7 +59,7 @@ class SyncStructCase(unittest.TestCase): self.receiving_done = asyncio.Event() publisher = asyncio.Future() test_dict = asyncio.Future() - asyncio.async(start_server(publisher, test_dict)) + asyncio.ensure_future(start_server(publisher, test_dict)) loop.run_until_complete(publisher) loop.run_until_complete(test_dict) @@ -68,7 +68,7 @@ class SyncStructCase(unittest.TestCase): test_vector = dict() loop.run_until_complete(write_test_data(test_vector)) - asyncio.async(write_test_data(test_dict)) + asyncio.ensure_future(write_test_data(test_dict)) self.subscriber = sync_struct.Subscriber("test", self.init_test_dict, self.notify) loop.run_until_complete(self.subscriber.connect(test_address, diff --git a/artiq/tools.py b/artiq/tools.py index 321c87bfe..e89910e20 100644 --- a/artiq/tools.py +++ b/artiq/tools.py @@ -90,7 +90,7 @@ def exc_to_warning(coro): @asyncio.coroutine def asyncio_wait_or_cancel(fs, **kwargs): - fs = [asyncio.async(f) for f in fs] + fs = [asyncio.ensure_future(f) for f in fs] try: d, p = yield from asyncio.wait(fs, **kwargs) except: @@ -105,7 +105,7 @@ def asyncio_wait_or_cancel(fs, **kwargs): class TaskObject: def start(self): - self.task = asyncio.async(self._do()) + self.task = asyncio.ensure_future(self._do()) @asyncio.coroutine def stop(self): From f552d62b6987fa33068bcfe7c73a914eee013b2c Mon Sep 17 00:00:00 2001 From: Sebastien Bourdeauducq Date: Sat, 3 Oct 2015 19:28:57 +0800 Subject: [PATCH 06/53] use Python 3.5 coroutines --- artiq/frontend/artiq_ctlmgr.py | 85 +++++++++++++-------------- artiq/frontend/artiq_influxdb.py | 20 +++---- artiq/gui/explorer.py | 17 +++--- artiq/gui/log.py | 10 ++-- artiq/gui/moninj.py | 21 +++---- artiq/gui/parameters.py | 10 ++-- artiq/gui/results.py | 10 ++-- artiq/gui/schedule.py | 15 ++--- artiq/gui/state.py | 5 +- artiq/master/repository.py | 12 ++-- artiq/master/scheduler.py | 95 +++++++++++++------------------ artiq/master/worker.py | 93 +++++++++++++----------------- artiq/protocols/asyncio_server.py | 14 ++--- artiq/protocols/pc_rpc.py | 38 ++++++------- artiq/protocols/sync_struct.py | 26 ++++----- artiq/test/pc_rpc.py | 13 ++--- artiq/test/sync_struct.py | 10 ++-- artiq/test/worker.py | 13 ++--- artiq/tools.py | 25 ++++---- 19 files changed, 228 insertions(+), 304 deletions(-) diff --git a/artiq/frontend/artiq_ctlmgr.py b/artiq/frontend/artiq_ctlmgr.py index 273cae9af..fb25519b1 100755 --- a/artiq/frontend/artiq_ctlmgr.py +++ b/artiq/frontend/artiq_ctlmgr.py @@ -56,60 +56,55 @@ class Controller: self.process = None self.launch_task = asyncio.Task(self.launcher()) - @asyncio.coroutine - def end(self): + async def end(self): self.launch_task.cancel() - yield from asyncio.wait_for(self.launch_task, None) + await asyncio.wait_for(self.launch_task, None) - @asyncio.coroutine - def _call_controller(self, method): + async def _call_controller(self, method): remote = AsyncioClient() - yield from remote.connect_rpc(self.host, self.port, None) + await remote.connect_rpc(self.host, self.port, None) try: targets, _ = remote.get_rpc_id() remote.select_rpc_target(targets[0]) - r = yield from getattr(remote, method)() + r = await getattr(remote, method)() finally: remote.close_rpc() return r - @asyncio.coroutine - def _ping(self): + async def _ping(self): try: - ok = yield from asyncio.wait_for(self._call_controller("ping"), - self.ping_timeout) + ok = await asyncio.wait_for(self._call_controller("ping"), + self.ping_timeout) if ok: self.retry_timer_cur = self.retry_timer return ok except: return False - @asyncio.coroutine - def _wait_and_ping(self): + async def _wait_and_ping(self): while True: try: - yield from asyncio.wait_for(self.process.wait(), - self.ping_timer) + await asyncio.wait_for(self.process.wait(), + self.ping_timer) except asyncio.TimeoutError: logger.debug("pinging controller %s", self.name) - ok = yield from self._ping() + ok = await self._ping() if not ok: logger.warning("Controller %s ping failed", self.name) - yield from self._terminate() + await self._terminate() return else: break - @asyncio.coroutine - def launcher(self): + async def launcher(self): try: while True: logger.info("Starting controller %s with command: %s", self.name, self.command) try: - self.process = yield from asyncio.create_subprocess_exec( + self.process = await asyncio.create_subprocess_exec( *shlex.split(self.command)) - yield from self._wait_and_ping() + await self._wait_and_ping() except FileNotFoundError: logger.warning("Controller %s failed to start", self.name) else: @@ -117,33 +112,32 @@ class Controller: logger.warning("Restarting in %.1f seconds", self.retry_timer_cur) try: - yield from asyncio.wait_for(self.retry_now.wait(), - self.retry_timer_cur) + await asyncio.wait_for(self.retry_now.wait(), + self.retry_timer_cur) except asyncio.TimeoutError: pass self.retry_timer_cur *= self.retry_timer_backoff except asyncio.CancelledError: - yield from self._terminate() + await self._terminate() - @asyncio.coroutine - def _terminate(self): + async def _terminate(self): logger.info("Terminating controller %s", self.name) if self.process is not None and self.process.returncode is None: try: - yield from asyncio.wait_for(self._call_controller("terminate"), - self.term_timeout) + await asyncio.wait_for(self._call_controller("terminate"), + self.term_timeout) except: logger.warning("Controller %s did not respond to terminate " "command, killing", self.name) self.process.kill() try: - yield from asyncio.wait_for(self.process.wait(), - self.term_timeout) + await asyncio.wait_for(self.process.wait(), + self.term_timeout) except: logger.warning("Controller %s failed to exit, killing", self.name) self.process.kill() - yield from self.process.wait() + await self.process.wait() logger.debug("Controller %s terminated", self.name) @@ -163,17 +157,16 @@ class Controllers: self.active = dict() self.process_task = asyncio.Task(self._process()) - @asyncio.coroutine - def _process(self): + async def _process(self): while True: - action, param = yield from self.queue.get() + action, param = await self.queue.get() if action == "set": k, ddb_entry = param if k in self.active: - yield from self.active[k].end() + await self.active[k].end() self.active[k] = Controller(k, ddb_entry) elif action == "del": - yield from self.active[param].end() + await self.active[param].end() del self.active[param] else: raise ValueError @@ -196,11 +189,10 @@ class Controllers: for name in set(self.active_or_queued): del self[name] - @asyncio.coroutine - def shutdown(self): + async def shutdown(self): self.process_task.cancel() for c in self.active.values(): - yield from c.end() + await c.end() class ControllerDB: @@ -225,8 +217,7 @@ class ControllerManager(TaskObject): self.retry_master = retry_master self.controller_db = ControllerDB() - @asyncio.coroutine - def _do(self): + async def _do(self): try: subscriber = Subscriber("devices", self.controller_db.sync_struct_init) @@ -236,12 +227,12 @@ class ControllerManager(TaskObject): s = subscriber.writer.get_extra_info("socket") localhost = s.getsockname()[0] self.controller_db.set_host_filter(localhost) - yield from subscriber.connect(self.server, self.port, - set_host_filter) + await subscriber.connect(self.server, self.port, + set_host_filter) try: - yield from asyncio.wait_for(subscriber.receive_task, None) + await asyncio.wait_for(subscriber.receive_task, None) finally: - yield from subscriber.close() + await subscriber.close() except (ConnectionAbortedError, ConnectionError, ConnectionRefusedError, ConnectionResetError) as e: logger.warning("Connection to master failed (%s: %s)", @@ -249,11 +240,11 @@ class ControllerManager(TaskObject): else: logger.warning("Connection to master lost") logger.warning("Retrying in %.1f seconds", self.retry_master) - yield from asyncio.sleep(self.retry_master) + await asyncio.sleep(self.retry_master) except asyncio.CancelledError: pass finally: - yield from self.controller_db.current_controllers.shutdown() + await self.controller_db.current_controllers.shutdown() def retry_now(self, k): """If a controller is disabled and pending retry, perform that retry diff --git a/artiq/frontend/artiq_influxdb.py b/artiq/frontend/artiq_influxdb.py index 299695030..f69da9b0e 100755 --- a/artiq/frontend/artiq_influxdb.py +++ b/artiq/frontend/artiq_influxdb.py @@ -96,24 +96,23 @@ class DBWriter(TaskObject): logger.warning("failed to update parameter '%s': " "too many pending updates", k) - @asyncio.coroutine - def _do(self): + async def _do(self): while True: - k, v = yield from self._queue.get() + k, v = await self._queue.get() url = self.base_url + "/write" params = {"u": self.user, "p": self.password, "db": self.database, "consistency": "any", "precision": "n"} fmt_ty, fmt_v = format_influxdb(v) data = "{},parameter={} {}={}".format(self.table, k, fmt_ty, fmt_v) try: - response = yield from aiohttp.request( + response = await aiohttp.request( "POST", url, params=params, data=data) except: logger.warning("got exception trying to update '%s'", k, exc_info=True) else: if response.status not in (200, 204): - content = (yield from response.content.read()).decode() + content = (await response.content.read()).decode() if content: content = content[:-1] # drop \n logger.warning("got HTTP status %d " @@ -144,18 +143,17 @@ class MasterReader(TaskObject): self.filter_function = filter_function self.writer = writer - @asyncio.coroutine - def _do(self): + async def _do(self): subscriber = Subscriber( "parameters", partial(Parameters, self.filter_function, self.writer)) while True: try: - yield from subscriber.connect(self.server, self.port) + await subscriber.connect(self.server, self.port) try: - yield from asyncio.wait_for(subscriber.receive_task, None) + await asyncio.wait_for(subscriber.receive_task, None) finally: - yield from subscriber.close() + await subscriber.close() except (ConnectionAbortedError, ConnectionError, ConnectionRefusedError, ConnectionResetError) as e: logger.warning("Connection to master failed (%s: %s)", @@ -163,7 +161,7 @@ class MasterReader(TaskObject): else: logger.warning("Connection to master lost") logger.warning("Retrying in %.1f seconds", self.retry) - yield from asyncio.sleep(self.retry) + await asyncio.sleep(self.retry) class Filter: diff --git a/artiq/gui/explorer.py b/artiq/gui/explorer.py index bf2fc3e69..fe6cb58cb 100644 --- a/artiq/gui/explorer.py +++ b/artiq/gui/explorer.py @@ -300,23 +300,20 @@ class ExplorerDock(dockarea.Dock): def enable_duedate(self): self.datetime_en.setChecked(True) - @asyncio.coroutine - def sub_connect(self, host, port): + async def sub_connect(self, host, port): self.explist_subscriber = Subscriber("explist", self.init_explist_model) - yield from self.explist_subscriber.connect(host, port) + await self.explist_subscriber.connect(host, port) - @asyncio.coroutine - def sub_close(self): - yield from self.explist_subscriber.close() + async def sub_close(self): + await self.explist_subscriber.close() def init_explist_model(self, init): self.explist_model = _ExplistModel(self, self.el, init) self.el.setModel(self.explist_model) return self.explist_model - @asyncio.coroutine - def submit(self, pipeline_name, file, class_name, arguments, + async def submit(self, pipeline_name, file, class_name, arguments, priority, due_date, flush): expid = { "repo_rev": None, @@ -324,8 +321,8 @@ class ExplorerDock(dockarea.Dock): "class_name": class_name, "arguments": arguments, } - rid = yield from self.schedule_ctl.submit(pipeline_name, expid, - priority, due_date, flush) + rid = await self.schedule_ctl.submit(pipeline_name, expid, + priority, due_date, flush) self.status_bar.showMessage("Submitted RID {}".format(rid)) def submit_clicked(self): diff --git a/artiq/gui/log.py b/artiq/gui/log.py index 8c7a99c24..4c49673fa 100644 --- a/artiq/gui/log.py +++ b/artiq/gui/log.py @@ -41,14 +41,12 @@ class LogDock(dockarea.Dock): self.addWidget(self.log) self.scroll_at_bottom = False - @asyncio.coroutine - def sub_connect(self, host, port): + async def sub_connect(self, host, port): self.subscriber = Subscriber("log", self.init_log_model) - yield from self.subscriber.connect(host, port) + await self.subscriber.connect(host, port) - @asyncio.coroutine - def sub_close(self): - yield from self.subscriber.close() + async def sub_close(self): + await self.subscriber.close() def rows_inserted_before(self): scrollbar = self.log.verticalScrollBar() diff --git a/artiq/gui/moninj.py b/artiq/gui/moninj.py index bd7e94214..835fb58ab 100644 --- a/artiq/gui/moninj.py +++ b/artiq/gui/moninj.py @@ -232,26 +232,24 @@ class MonInj(TaskObject): self.dm = _DeviceManager(self.send_to_device, dict()) self.transport = None - @asyncio.coroutine - def start(self, server, port): + async def start(self, server, port): loop = asyncio.get_event_loop() - yield from loop.create_datagram_endpoint(lambda: self, + await loop.create_datagram_endpoint(lambda: self, family=socket.AF_INET) try: - yield from self.subscriber.connect(server, port) + await self.subscriber.connect(server, port) try: TaskObject.start(self) except: - yield from self.subscriber.close() + await self.subscriber.close() raise except: self.transport.close() raise - @asyncio.coroutine - def stop(self): - yield from TaskObject.stop(self) - yield from self.subscriber.close() + async def stop(self): + await TaskObject.stop(self) + await self.subscriber.close() if self.transport is not None: self.transport.close() self.transport = None @@ -295,10 +293,9 @@ class MonInj(TaskObject): else: self.transport.sendto(data, (ca, 3250)) - @asyncio.coroutine - def _do(self): + async def _do(self): while True: - yield from asyncio.sleep(0.2) + await asyncio.sleep(0.2) # MONINJ_REQ_MONITOR self.send_to_device(b"\x01") diff --git a/artiq/gui/parameters.py b/artiq/gui/parameters.py index 4bc53b927..21b91a499 100644 --- a/artiq/gui/parameters.py +++ b/artiq/gui/parameters.py @@ -59,14 +59,12 @@ class ParametersDock(dockarea.Dock): else: self.table.hideRow(row) - @asyncio.coroutine - def sub_connect(self, host, port): + async def sub_connect(self, host, port): self.subscriber = Subscriber("parameters", self.init_parameters_model) - yield from self.subscriber.connect(host, port) + await self.subscriber.connect(host, port) - @asyncio.coroutine - def sub_close(self): - yield from self.subscriber.close() + async def sub_close(self): + await self.subscriber.close() def init_parameters_model(self, init): self.table_model = ParametersModel(self.table, init) diff --git a/artiq/gui/results.py b/artiq/gui/results.py index 0ed872a6a..edd40e2ad 100644 --- a/artiq/gui/results.py +++ b/artiq/gui/results.py @@ -68,15 +68,13 @@ class ResultsDock(dockarea.Dock): def get_result(self, key): return self.table_model.backing_store[key] - @asyncio.coroutine - def sub_connect(self, host, port): + async def sub_connect(self, host, port): self.subscriber = Subscriber("rt_results", self.init_results_model, self.on_mod) - yield from self.subscriber.connect(host, port) + await self.subscriber.connect(host, port) - @asyncio.coroutine - def sub_close(self): - yield from self.subscriber.close() + async def sub_close(self): + await self.subscriber.close() def init_results_model(self, init): self.table_model = ResultsModel(self.table, init) diff --git a/artiq/gui/schedule.py b/artiq/gui/schedule.py index 21ab02b88..f1606caf9 100644 --- a/artiq/gui/schedule.py +++ b/artiq/gui/schedule.py @@ -75,23 +75,20 @@ class ScheduleDock(dockarea.Dock): delete_action.triggered.connect(self.delete_clicked) self.table.addAction(delete_action) - @asyncio.coroutine - def sub_connect(self, host, port): + async def sub_connect(self, host, port): self.subscriber = Subscriber("schedule", self.init_schedule_model) - yield from self.subscriber.connect(host, port) + await self.subscriber.connect(host, port) - @asyncio.coroutine - def sub_close(self): - yield from self.subscriber.close() + async def sub_close(self): + await self.subscriber.close() def init_schedule_model(self, init): self.table_model = _ScheduleModel(self.table, init) self.table.setModel(self.table_model) return self.table_model - @asyncio.coroutine - def delete(self, rid): - yield from self.schedule_ctl.delete(rid) + async def delete(self, rid): + await self.schedule_ctl.delete(rid) def delete_clicked(self): idx = self.table.selectedIndexes() diff --git a/artiq/gui/state.py b/artiq/gui/state.py index 9088da4e6..8cead13f5 100644 --- a/artiq/gui/state.py +++ b/artiq/gui/state.py @@ -69,11 +69,10 @@ class StateManager(TaskObject): exc_info=True) pyon.store_file(self.filename, data) - @asyncio.coroutine - def _do(self): + async def _do(self): try: while True: - yield from asyncio.sleep(self.autosave_period) + await asyncio.sleep(self.autosave_period) self.save() finally: self.save() diff --git a/artiq/master/repository.py b/artiq/master/repository.py index 6478570c8..7560cd240 100644 --- a/artiq/master/repository.py +++ b/artiq/master/repository.py @@ -12,17 +12,16 @@ from artiq.tools import exc_to_warning logger = logging.getLogger(__name__) -@asyncio.coroutine -def _scan_experiments(wd, log): +async def _scan_experiments(wd, log): r = dict() for f in os.listdir(wd): if f.endswith(".py"): try: worker = Worker({"log": lambda message: log("scan", message)}) try: - description = yield from worker.examine(os.path.join(wd, f)) + description = await worker.examine(os.path.join(wd, f)) finally: - yield from worker.close() + await worker.close() for class_name, class_desc in description.items(): name = class_desc["name"] arguments = class_desc["arguments"] @@ -68,8 +67,7 @@ class Repository: # The object cannot be used anymore after calling this method. self.backend.release_rev(self.cur_rev) - @asyncio.coroutine - def scan(self, new_cur_rev=None): + async def scan(self, new_cur_rev=None): if self._scanning: return self._scanning = True @@ -79,7 +77,7 @@ class Repository: wd, _ = self.backend.request_rev(new_cur_rev) self.backend.release_rev(self.cur_rev) self.cur_rev = new_cur_rev - new_explist = yield from _scan_experiments(wd, self.log_fn) + new_explist = await _scan_experiments(wd, self.log_fn) _sync_explist(self.explist, new_explist) finally: diff --git a/artiq/master/scheduler.py b/artiq/master/scheduler.py index d7b931f42..fdd925287 100644 --- a/artiq/master/scheduler.py +++ b/artiq/master/scheduler.py @@ -24,13 +24,12 @@ class RunStatus(Enum): def _mk_worker_method(name): - @asyncio.coroutine - def worker_method(self, *args, **kwargs): + async def worker_method(self, *args, **kwargs): if self.worker.closed.is_set(): return True m = getattr(self.worker, name) try: - return (yield from m(*args, **kwargs)) + return await m(*args, **kwargs) except Exception as e: if isinstance(e, asyncio.CancelledError): raise @@ -97,19 +96,17 @@ class Run: runnable = 1 return (runnable, self.priority, due_date_k, -self.rid) - @asyncio.coroutine - def close(self): + async def close(self): # called through pool - yield from self.worker.close() + await self.worker.close() del self._notifier[self.rid] _build = _mk_worker_method("build") - @asyncio.coroutine - def build(self): - yield from self._build(self.rid, self.pipeline_name, - self.wd, self.expid, - self.priority) + async def build(self): + await self._build(self.rid, self.pipeline_name, + self.wd, self.expid, + self.priority) prepare = _mk_worker_method("prepare") run = _mk_worker_method("run") @@ -154,13 +151,12 @@ class RunPool: self.state_changed.notify() return rid - @asyncio.coroutine - def delete(self, rid): + async def delete(self, rid): # called through deleter if rid not in self.runs: return run = self.runs[rid] - yield from run.close() + await run.close() if "repo_rev" in run.expid: self.repo_backend.release_rev(run.expid["repo_rev"]) del self.runs[rid] @@ -203,14 +199,13 @@ class PrepareStage(TaskObject): else: return candidate.due_date - now - @asyncio.coroutine - def _do(self): + async def _do(self): while True: run = self._get_run() if run is None: - yield from self.pool.state_changed.wait() + await self.pool.state_changed.wait() elif isinstance(run, float): - yield from asyncio_wait_or_cancel([self.pool.state_changed.wait()], + await asyncio_wait_or_cancel([self.pool.state_changed.wait()], timeout=run) else: if run.flush: @@ -221,7 +216,7 @@ class PrepareStage(TaskObject): for r in self.pool.runs.values()): ev = [self.pool.state_changed.wait(), run.worker.closed.wait()] - yield from asyncio_wait_or_cancel( + await asyncio_wait_or_cancel( ev, return_when=asyncio.FIRST_COMPLETED) if run.worker.closed.is_set(): break @@ -229,8 +224,8 @@ class PrepareStage(TaskObject): continue run.status = RunStatus.preparing try: - yield from run.build() - yield from run.prepare() + await run.build() + await run.prepare() except: logger.warning("got worker exception in prepare stage, " "deleting RID %d", @@ -255,8 +250,7 @@ class RunStage(TaskObject): r = None return r - @asyncio.coroutine - def _do(self): + async def _do(self): stack = [] while True: @@ -265,7 +259,7 @@ class RunStage(TaskObject): next_irun is not None and next_irun.priority_key() > stack[-1].priority_key()): while next_irun is None: - yield from self.pool.state_changed.wait() + await self.pool.state_changed.wait() next_irun = self._get_run() stack.append(next_irun) @@ -273,10 +267,10 @@ class RunStage(TaskObject): try: if run.status == RunStatus.paused: run.status = RunStatus.running - completed = yield from run.resume() + completed = await run.resume() else: run.status = RunStatus.running - completed = yield from run.run() + completed = await run.run() except: logger.warning("got worker exception in run stage, " "deleting RID %d", @@ -305,17 +299,16 @@ class AnalyzeStage(TaskObject): r = None return r - @asyncio.coroutine - def _do(self): + async def _do(self): while True: run = self._get_run() while run is None: - yield from self.pool.state_changed.wait() + await self.pool.state_changed.wait() run = self._get_run() run.status = RunStatus.analyzing try: - yield from run.analyze() - yield from run.write_results() + await run.analyze() + await run.write_results() except: logger.warning("got worker exception in analyze stage, " "deleting RID %d", @@ -337,12 +330,11 @@ class Pipeline: self._run.start() self._analyze.start() - @asyncio.coroutine - def stop(self): + async def stop(self): # NB: restart of a stopped pipeline is not supported - yield from self._analyze.stop() - yield from self._run.stop() - yield from self._prepare.stop() + await self._analyze.stop() + await self._run.stop() + await self._prepare.stop() class Deleter(TaskObject): @@ -358,36 +350,32 @@ class Deleter(TaskObject): break self._queue.put_nowait(rid) - @asyncio.coroutine - def join(self): - yield from self._queue.join() + async def join(self): + await self._queue.join() - @asyncio.coroutine - def _delete(self, rid): + async def _delete(self, rid): for pipeline in self._pipelines.values(): if rid in pipeline.pool.runs: logger.debug("deleting RID %d...", rid) - yield from pipeline.pool.delete(rid) + await pipeline.pool.delete(rid) logger.debug("deletion of RID %d completed", rid) break - @asyncio.coroutine - def _gc_pipelines(self): + async def _gc_pipelines(self): pipeline_names = list(self._pipelines.keys()) for name in pipeline_names: if not self._pipelines[name].pool.runs: logger.debug("garbage-collecting pipeline '%s'...", name) - yield from self._pipelines[name].stop() + await self._pipelines[name].stop() del self._pipelines[name] logger.debug("garbage-collection of pipeline '%s' completed", name) - @asyncio.coroutine - def _do(self): + async def _do(self): while True: - rid = yield from self._queue.get() - yield from self._delete(rid) - yield from self._gc_pipelines() + rid = await self._queue.get() + await self._delete(rid) + await self._gc_pipelines() self._queue.task_done() @@ -406,15 +394,14 @@ class Scheduler: def start(self): self._deleter.start() - @asyncio.coroutine - def stop(self): + async def stop(self): # NB: restart of a stopped scheduler is not supported self._terminated = True # prevent further runs from being created for pipeline in self._pipelines.values(): for rid in pipeline.pool.runs.keys(): self._deleter.delete(rid) - yield from self._deleter.join() - yield from self._deleter.stop() + await self._deleter.join() + await self._deleter.stop() if self._pipelines: logger.warning("some pipelines were not garbage-collected") diff --git a/artiq/master/worker.py b/artiq/master/worker.py index 1bb36f94d..104cd8a15 100644 --- a/artiq/master/worker.py +++ b/artiq/master/worker.py @@ -56,27 +56,25 @@ class Worker: else: return None - @asyncio.coroutine - def _create_process(self): - yield from self.io_lock.acquire() + async def _create_process(self): + await self.io_lock.acquire() try: if self.closed.is_set(): raise WorkerError("Attempting to create process after close") - self.process = yield from asyncio.create_subprocess_exec( + self.process = await asyncio.create_subprocess_exec( sys.executable, "-m", "artiq.master.worker_impl", stdout=subprocess.PIPE, stdin=subprocess.PIPE) finally: self.io_lock.release() - @asyncio.coroutine - def close(self, term_timeout=1.0): + async def close(self, term_timeout=1.0): """Interrupts any I/O with the worker process and terminates the worker process. This method should always be called by the user to clean up, even if build() or examine() raises an exception.""" self.closed.set() - yield from self.io_lock.acquire() + await self.io_lock.acquire() try: if self.process is None: # Note the %s - self.rid can be None @@ -91,26 +89,25 @@ class Worker: return obj = {"action": "terminate"} try: - yield from self._send(obj, cancellable=False) + await self._send(obj, cancellable=False) except: logger.warning("failed to send terminate command to worker" " (RID %s), killing", self.rid, exc_info=True) self.process.kill() - yield from self.process.wait() + await self.process.wait() return try: - yield from asyncio.wait_for(self.process.wait(), term_timeout) + await asyncio.wait_for(self.process.wait(), term_timeout) except asyncio.TimeoutError: logger.warning("worker did not exit (RID %s), killing", self.rid) self.process.kill() - yield from self.process.wait() + await self.process.wait() else: logger.debug("worker exited gracefully (RID %s)", self.rid) finally: self.io_lock.release() - @asyncio.coroutine - def _send(self, obj, cancellable=True): + async def _send(self, obj, cancellable=True): assert self.io_lock.locked() line = pyon.encode(obj) self.process.stdin.write(line.encode()) @@ -118,7 +115,7 @@ class Worker: ifs = [self.process.stdin.drain()] if cancellable: ifs.append(self.closed.wait()) - fs = yield from asyncio_wait_or_cancel( + fs = await asyncio_wait_or_cancel( ifs, timeout=self.send_timeout, return_when=asyncio.FIRST_COMPLETED) if all(f.cancelled() for f in fs): @@ -129,10 +126,9 @@ class Worker: if cancellable and self.closed.is_set(): raise WorkerError("Data transmission to worker cancelled") - @asyncio.coroutine - def _recv(self, timeout): + async def _recv(self, timeout): assert self.io_lock.locked() - fs = yield from asyncio_wait_or_cancel( + fs = await asyncio_wait_or_cancel( [self.process.stdout.readline(), self.closed.wait()], timeout=timeout, return_when=asyncio.FIRST_COMPLETED) if all(f.cancelled() for f in fs): @@ -148,13 +144,12 @@ class Worker: raise WorkerError("Worker sent invalid PYON data") return obj - @asyncio.coroutine - def _handle_worker_requests(self): + async def _handle_worker_requests(self): while True: try: - yield from self.io_lock.acquire() + await self.io_lock.acquire() try: - obj = yield from self._recv(self.watchdog_time()) + obj = await self._recv(self.watchdog_time()) finally: self.io_lock.release() except WorkerTimeout: @@ -181,24 +176,23 @@ class Worker: except: reply = {"status": "failed", "message": traceback.format_exc()} - yield from self.io_lock.acquire() + await self.io_lock.acquire() try: - yield from self._send(reply) + await self._send(reply) finally: self.io_lock.release() - @asyncio.coroutine - def _worker_action(self, obj, timeout=None): + async def _worker_action(self, obj, timeout=None): if timeout is not None: self.watchdogs[-1] = time.monotonic() + timeout try: - yield from self.io_lock.acquire() + await self.io_lock.acquire() try: - yield from self._send(obj) + await self._send(obj) finally: self.io_lock.release() try: - completed = yield from self._handle_worker_requests() + completed = await self._handle_worker_requests() except WorkerTimeout: raise WorkerWatchdogTimeout finally: @@ -206,11 +200,10 @@ class Worker: del self.watchdogs[-1] return completed - @asyncio.coroutine - def build(self, rid, pipeline_name, wd, expid, priority, timeout=15.0): + async def build(self, rid, pipeline_name, wd, expid, priority, timeout=15.0): self.rid = rid - yield from self._create_process() - yield from self._worker_action( + await self._create_process() + await self._worker_action( {"action": "build", "rid": rid, "pipeline_name": pipeline_name, @@ -219,45 +212,39 @@ class Worker: "priority": priority}, timeout) - @asyncio.coroutine - def prepare(self): - yield from self._worker_action({"action": "prepare"}) + async def prepare(self): + await self._worker_action({"action": "prepare"}) - @asyncio.coroutine - def run(self): - completed = yield from self._worker_action({"action": "run"}) + async def run(self): + completed = await self._worker_action({"action": "run"}) if not completed: self.yield_time = time.monotonic() return completed - @asyncio.coroutine - def resume(self): + async def resume(self): stop_duration = time.monotonic() - self.yield_time for wid, expiry in self.watchdogs: self.watchdogs[wid] += stop_duration - completed = yield from self._worker_action({"status": "ok", - "data": None}) + completed = await self._worker_action({"status": "ok", + "data": None}) if not completed: self.yield_time = time.monotonic() return completed - @asyncio.coroutine - def analyze(self): - yield from self._worker_action({"action": "analyze"}) + async def analyze(self): + await self._worker_action({"action": "analyze"}) - @asyncio.coroutine - def write_results(self, timeout=15.0): - yield from self._worker_action({"action": "write_results"}, - timeout) + async def write_results(self, timeout=15.0): + await self._worker_action({"action": "write_results"}, + timeout) - @asyncio.coroutine - def examine(self, file, timeout=20.0): - yield from self._create_process() + async def examine(self, file, timeout=20.0): + await self._create_process() r = dict() def register(class_name, name, arguments): r[class_name] = {"name": name, "arguments": arguments} self.register_experiment = register - yield from self._worker_action({"action": "examine", + await self._worker_action({"action": "examine", "file": file}, timeout) del self.register_experiment return r diff --git a/artiq/protocols/asyncio_server.py b/artiq/protocols/asyncio_server.py index 451bb272f..a2425137f 100644 --- a/artiq/protocols/asyncio_server.py +++ b/artiq/protocols/asyncio_server.py @@ -12,8 +12,7 @@ class AsyncioServer: def __init__(self): self._client_tasks = set() - @asyncio.coroutine - def start(self, host, port): + async def start(self, host, port): """Starts the server. The user must call ``stop`` to free resources properly after this @@ -26,11 +25,10 @@ class AsyncioServer: :param port: TCP port to bind to. """ - self.server = yield from asyncio.start_server(self._handle_connection, - host, port) + self.server = await asyncio.start_server(self._handle_connection, + host, port) - @asyncio.coroutine - def stop(self): + async def stop(self): """Stops the server. """ @@ -39,11 +37,11 @@ class AsyncioServer: task.cancel() for task in wait_for: try: - yield from asyncio.wait_for(task, None) + await asyncio.wait_for(task, None) except asyncio.CancelledError: pass self.server.close() - yield from self.server.wait_closed() + await self.server.wait_closed() del self.server def _client_done(self, task): diff --git a/artiq/protocols/pc_rpc.py b/artiq/protocols/pc_rpc.py index f001d3a26..beab88224 100644 --- a/artiq/protocols/pc_rpc.py +++ b/artiq/protocols/pc_rpc.py @@ -159,16 +159,15 @@ class AsyncioClient: self.__target_names = None self.__description = None - @asyncio.coroutine - def connect_rpc(self, host, port, target_name): + async def connect_rpc(self, host, port, target_name): """Connects to the server. This cannot be done in __init__ because this method is a coroutine. See ``Client`` for a description of the parameters.""" self.__reader, self.__writer = \ - yield from asyncio.open_connection(host, port) + await asyncio.open_connection(host, port) try: self.__writer.write(_init_string) - server_identification = yield from self.__recv() + server_identification = await self.__recv() self.__target_names = server_identification["targets"] self.__description = server_identification["description"] if target_name is not None: @@ -205,20 +204,18 @@ class AsyncioClient: line = pyon.encode(obj) + "\n" self.__writer.write(line.encode()) - @asyncio.coroutine - def __recv(self): - line = yield from self.__reader.readline() + async def __recv(self): + line = await self.__reader.readline() return pyon.decode(line.decode()) - @asyncio.coroutine - def __do_rpc(self, name, args, kwargs): - yield from self.__lock.acquire() + async def __do_rpc(self, name, args, kwargs): + await self.__lock.acquire() try: obj = {"action": "call", "name": name, "args": args, "kwargs": kwargs} self.__send(obj) - obj = yield from self.__recv() + obj = await self.__recv() if obj["status"] == "ok": return obj["ret"] elif obj["status"] == "failed": @@ -229,9 +226,8 @@ class AsyncioClient: self.__lock.release() def __getattr__(self, name): - @asyncio.coroutine - def proxy(*args, **kwargs): - res = yield from self.__do_rpc(name, args, kwargs) + async def proxy(*args, **kwargs): + res = await self.__do_rpc(name, args, kwargs) return res return proxy @@ -413,10 +409,9 @@ class Server(_AsyncioServer): if builtin_terminate: self._terminate_request = asyncio.Event() - @asyncio.coroutine - def _handle_connection_cr(self, reader, writer): + async def _handle_connection_cr(self, reader, writer): try: - line = yield from reader.readline() + line = await reader.readline() if line != _init_string: return @@ -426,7 +421,7 @@ class Server(_AsyncioServer): } line = pyon.encode(obj) + "\n" writer.write(line.encode()) - line = yield from reader.readline() + line = await reader.readline() if not line: return target_name = line.decode()[:-1] @@ -436,7 +431,7 @@ class Server(_AsyncioServer): return while True: - line = yield from reader.readline() + line = await reader.readline() if not line: break obj = pyon.decode(line.decode()) @@ -486,9 +481,8 @@ class Server(_AsyncioServer): finally: writer.close() - @asyncio.coroutine - def wait_terminate(self): - yield from self._terminate_request.wait() + async def wait_terminate(self): + await self._terminate_request.wait() def simple_server_loop(targets, host, port, description=None): diff --git a/artiq/protocols/sync_struct.py b/artiq/protocols/sync_struct.py index e8fd2c69d..e2c1021ad 100644 --- a/artiq/protocols/sync_struct.py +++ b/artiq/protocols/sync_struct.py @@ -61,10 +61,9 @@ class Subscriber: self.target_builders = [target_builder] self.notify_cb = notify_cb - @asyncio.coroutine - def connect(self, host, port, before_receive_cb=None): + async def connect(self, host, port, before_receive_cb=None): self.reader, self.writer = \ - yield from asyncio.open_connection(host, port) + await asyncio.open_connection(host, port) try: if before_receive_cb is not None: before_receive_cb() @@ -77,12 +76,11 @@ class Subscriber: del self.writer raise - @asyncio.coroutine - def close(self): + async def close(self): try: self.receive_task.cancel() try: - yield from asyncio.wait_for(self.receive_task, None) + await asyncio.wait_for(self.receive_task, None) except asyncio.CancelledError: pass finally: @@ -90,11 +88,10 @@ class Subscriber: del self.reader del self.writer - @asyncio.coroutine - def _receive_cr(self): + async def _receive_cr(self): targets = [] while True: - line = yield from self.reader.readline() + line = await self.reader.readline() if not line: return mod = pyon.decode(line.decode()) @@ -209,14 +206,13 @@ class Publisher(AsyncioServer): for notifier in notifiers.values(): notifier.publish = partial(self.publish, notifier) - @asyncio.coroutine - def _handle_connection_cr(self, reader, writer): + async def _handle_connection_cr(self, reader, writer): try: - line = yield from reader.readline() + line = await reader.readline() if line != _init_string: return - line = yield from reader.readline() + line = await reader.readline() if not line: return notifier_name = line.decode()[:-1] @@ -234,10 +230,10 @@ class Publisher(AsyncioServer): self._recipients[notifier_name].add(queue) try: while True: - line = yield from queue.get() + line = await queue.get() writer.write(line) # raise exception on connection error - yield from writer.drain() + await writer.drain() finally: self._recipients[notifier_name].remove(queue) except ConnectionResetError: diff --git a/artiq/test/pc_rpc.py b/artiq/test/pc_rpc.py index 5bd0a64cd..19000c659 100644 --- a/artiq/test/pc_rpc.py +++ b/artiq/test/pc_rpc.py @@ -52,23 +52,22 @@ class RPCCase(unittest.TestCase): def test_blocking_echo(self): self._run_server_and_test(self._blocking_echo) - @asyncio.coroutine - def _asyncio_echo(self): + async def _asyncio_echo(self): remote = pc_rpc.AsyncioClient() for attempt in range(100): - yield from asyncio.sleep(.2) + await asyncio.sleep(.2) try: - yield from remote.connect_rpc(test_address, test_port, "test") + await remote.connect_rpc(test_address, test_port, "test") except ConnectionRefusedError: pass else: break try: - test_object_back = yield from remote.echo(test_object) + test_object_back = await remote.echo(test_object) self.assertEqual(test_object, test_object_back) with self.assertRaises(pc_rpc.RemoteError): - yield from remote.non_existing_method() - yield from remote.terminate() + await remote.non_existing_method() + await remote.terminate() finally: remote.close_rpc() diff --git a/artiq/test/sync_struct.py b/artiq/test/sync_struct.py index 3f30062ff..00e4af878 100644 --- a/artiq/test/sync_struct.py +++ b/artiq/test/sync_struct.py @@ -8,7 +8,6 @@ test_address = "::1" test_port = 7777 -@asyncio.coroutine def write_test_data(test_dict): test_values = [5, 2.1, None, True, False, {"a": 5, 2: np.linspace(0, 10, 1)}, @@ -30,12 +29,11 @@ def write_test_data(test_dict): test_dict["finished"] = True -@asyncio.coroutine -def start_server(publisher_future, test_dict_future): +async def start_server(publisher_future, test_dict_future): test_dict = sync_struct.Notifier(dict()) publisher = sync_struct.Publisher( {"test": test_dict}) - yield from publisher.start(test_address, test_port) + await publisher.start(test_address, test_port) publisher_future.set_result(publisher) test_dict_future.set_result(test_dict) @@ -66,9 +64,9 @@ class SyncStructCase(unittest.TestCase): self.publisher = publisher.result() test_dict = test_dict.result() test_vector = dict() - loop.run_until_complete(write_test_data(test_vector)) + write_test_data(test_vector) - asyncio.ensure_future(write_test_data(test_dict)) + write_test_data(test_dict) self.subscriber = sync_struct.Subscriber("test", self.init_test_dict, self.notify) loop.run_until_complete(self.subscriber.connect(test_address, diff --git a/artiq/test/worker.py b/artiq/test/worker.py index b00660188..7be4304a0 100644 --- a/artiq/test/worker.py +++ b/artiq/test/worker.py @@ -36,15 +36,14 @@ class WatchdogTimeoutInBuild(EnvExperiment): pass -@asyncio.coroutine -def _call_worker(worker, expid): +async def _call_worker(worker, expid): try: - yield from worker.build(0, "main", None, expid, 0) - yield from worker.prepare() - yield from worker.run() - yield from worker.analyze() + await worker.build(0, "main", None, expid, 0) + await worker.prepare() + await worker.run() + await worker.analyze() finally: - yield from worker.close() + await worker.close() def _run_experiment(class_name): diff --git a/artiq/tools.py b/artiq/tools.py index e89910e20..56ae0fda1 100644 --- a/artiq/tools.py +++ b/artiq/tools.py @@ -79,27 +79,25 @@ def init_logger(args): logging.basicConfig(level=logging.WARNING + args.quiet*10 - args.verbose*10) -@asyncio.coroutine -def exc_to_warning(coro): +async def exc_to_warning(coro): try: - yield from coro + await coro except: logger.warning("asyncio coroutine terminated with exception", exc_info=True) -@asyncio.coroutine -def asyncio_wait_or_cancel(fs, **kwargs): +async def asyncio_wait_or_cancel(fs, **kwargs): fs = [asyncio.ensure_future(f) for f in fs] try: - d, p = yield from asyncio.wait(fs, **kwargs) + d, p = await asyncio.wait(fs, **kwargs) except: for f in fs: f.cancel() raise for f in p: f.cancel() - yield from asyncio.wait([f]) + await asyncio.wait([f]) return fs @@ -107,17 +105,15 @@ class TaskObject: def start(self): self.task = asyncio.ensure_future(self._do()) - @asyncio.coroutine - def stop(self): + async def stop(self): self.task.cancel() try: - yield from asyncio.wait_for(self.task, None) + await asyncio.wait_for(self.task, None) except asyncio.CancelledError: pass del self.task - @asyncio.coroutine - def _do(self): + async def _do(self): raise NotImplementedError @@ -129,13 +125,12 @@ class Condition: self._loop = asyncio.get_event_loop() self._waiters = collections.deque() - @asyncio.coroutine - def wait(self): + async def wait(self): """Wait until notified.""" fut = asyncio.Future(loop=self._loop) self._waiters.append(fut) try: - yield from fut + await fut finally: self._waiters.remove(fut) From 090a7e58715583a869fb61fc75336716e9438147 Mon Sep 17 00:00:00 2001 From: Sebastien Bourdeauducq Date: Sat, 3 Oct 2015 19:40:11 +0800 Subject: [PATCH 07/53] test/sync_struct: cleanup --- artiq/test/sync_struct.py | 47 +++++++++++++++------------------------ 1 file changed, 18 insertions(+), 29 deletions(-) diff --git a/artiq/test/sync_struct.py b/artiq/test/sync_struct.py index 00e4af878..ece90360b 100644 --- a/artiq/test/sync_struct.py +++ b/artiq/test/sync_struct.py @@ -29,18 +29,9 @@ def write_test_data(test_dict): test_dict["finished"] = True -async def start_server(publisher_future, test_dict_future): - test_dict = sync_struct.Notifier(dict()) - publisher = sync_struct.Publisher( - {"test": test_dict}) - await publisher.start(test_address, test_port) - publisher_future.set_result(publisher) - test_dict_future.set_result(test_dict) - - class SyncStructCase(unittest.TestCase): def init_test_dict(self, init): - self.test_dict = init + self.received_dict = init return init def notify(self, mod): @@ -52,29 +43,27 @@ class SyncStructCase(unittest.TestCase): self.loop = asyncio.new_event_loop() asyncio.set_event_loop(self.loop) - def test_recv(self): - loop = self.loop + async def _do_test_recv(self): self.receiving_done = asyncio.Event() - publisher = asyncio.Future() - test_dict = asyncio.Future() - asyncio.ensure_future(start_server(publisher, test_dict)) - loop.run_until_complete(publisher) - loop.run_until_complete(test_dict) - self.publisher = publisher.result() - test_dict = test_dict.result() - test_vector = dict() - write_test_data(test_vector) + test_dict = sync_struct.Notifier(dict()) + publisher = sync_struct.Publisher({"test": test_dict}) + await publisher.start(test_address, test_port) + + subscriber = sync_struct.Subscriber("test", self.init_test_dict, + self.notify) + await subscriber.connect(test_address, test_port) write_test_data(test_dict) - self.subscriber = sync_struct.Subscriber("test", self.init_test_dict, - self.notify) - loop.run_until_complete(self.subscriber.connect(test_address, - test_port)) - loop.run_until_complete(self.receiving_done.wait()) - self.assertEqual(self.test_dict, test_vector) - self.loop.run_until_complete(self.subscriber.close()) - self.loop.run_until_complete(self.publisher.stop()) + await self.receiving_done.wait() + + await subscriber.close() + await publisher.stop() + + self.assertEqual(self.received_dict, test_dict.read) + + def test_recv(self): + self.loop.run_until_complete(self._do_test_recv()) def tearDown(self): self.loop.close() From 50a5a3baf0b3e4524b2f5b7d47ce62b4bf4212f1 Mon Sep 17 00:00:00 2001 From: Sebastien Bourdeauducq Date: Sun, 4 Oct 2015 00:18:21 +0800 Subject: [PATCH 08/53] environment: attr_* -> setattr_* --- artiq/frontend/artiq_run.py | 4 +- artiq/language/environment.py | 6 +-- artiq/test/coredevice.py | 48 +++++++++---------- artiq/test/coredevice_vs_host.py | 26 +++++----- artiq/test/scheduler.py | 2 +- doc/manual/getting_started_core.rst | 18 +++---- doc/manual/getting_started_mgmt.rst | 2 +- examples/master/repository/arguments_demo.py | 38 +++++++-------- examples/master/repository/dds_test.py | 18 +++---- .../repository/flopping_f_simulation.py | 8 ++-- examples/master/repository/handover.py | 4 +- examples/master/repository/mandelbrot.py | 2 +- .../master/repository/photon_histogram.py | 24 +++++----- examples/master/repository/speed_benchmark.py | 34 ++++++------- examples/master/repository/tdr.py | 6 +-- examples/master/repository/transport.py | 18 +++---- examples/sim/al_spectroscopy.py | 20 ++++---- examples/sim/simple_simulation.py | 4 +- 18 files changed, 141 insertions(+), 141 deletions(-) diff --git a/artiq/frontend/artiq_run.py b/artiq/frontend/artiq_run.py index 91636f762..794abbf4e 100755 --- a/artiq/frontend/artiq_run.py +++ b/artiq/frontend/artiq_run.py @@ -22,8 +22,8 @@ logger = logging.getLogger(__name__) class ELFRunner(EnvExperiment): def build(self): - self.attr_device("core") - self.attr_argument("file") + self.setattr_device("core") + self.setattr_argument("file") def run(self): with open(self.file, "rb") as f: diff --git a/artiq/language/environment.py b/artiq/language/environment.py index e49246caf..99332e9a3 100644 --- a/artiq/language/environment.py +++ b/artiq/language/environment.py @@ -177,7 +177,7 @@ class HasEnvironment: raise return processor.process(argval) - def attr_argument(self, key, processor=None, group=None): + def setattr_argument(self, key, processor=None, group=None): """Sets an argument as attribute. The names of the argument and of the attribute are the same.""" setattr(self, key, self.get_argument(key, processor, group)) @@ -190,7 +190,7 @@ class HasEnvironment: raise ValueError("Device manager not present") return self.__dmgr.get(key) - def attr_device(self, key): + def setattr_device(self, key): """Sets a device driver as attribute. The names of the device driver and of the attribute are the same.""" setattr(self, key, self.get_device(key)) @@ -211,7 +211,7 @@ class HasEnvironment: else: raise - def attr_parameter(self, key, default=NoDefault): + def setattr_parameter(self, key, default=NoDefault): """Sets a parameter as attribute. The names of the argument and of the parameter are the same.""" setattr(self, key, self.get_parameter(key, default)) diff --git a/artiq/test/coredevice.py b/artiq/test/coredevice.py index 97347667e..33fad9098 100644 --- a/artiq/test/coredevice.py +++ b/artiq/test/coredevice.py @@ -11,8 +11,8 @@ from artiq.coredevice import runtime_exceptions class RTT(EnvExperiment): def build(self): - self.attr_device("core") - self.attr_device("ttl_inout") + self.setattr_device("core") + self.setattr_device("ttl_inout") def set_rtt(self, rtt): self.set_result("rtt", rtt) @@ -34,9 +34,9 @@ class RTT(EnvExperiment): class Loopback(EnvExperiment): def build(self): - self.attr_device("core") - self.attr_device("loop_in") - self.attr_device("loop_out") + self.setattr_device("core") + self.setattr_device("loop_in") + self.setattr_device("loop_out") def set_rtt(self, rtt): self.set_result("rtt", rtt) @@ -56,9 +56,9 @@ class Loopback(EnvExperiment): class ClockGeneratorLoopback(EnvExperiment): def build(self): - self.attr_device("core") - self.attr_device("loop_clock_in") - self.attr_device("loop_clock_out") + self.setattr_device("core") + self.setattr_device("loop_clock_in") + self.setattr_device("loop_clock_out") def set_count(self, count): self.set_result("count", count) @@ -78,8 +78,8 @@ class ClockGeneratorLoopback(EnvExperiment): class PulseRate(EnvExperiment): def build(self): - self.attr_device("core") - self.attr_device("ttl_out") + self.setattr_device("core") + self.setattr_device("ttl_out") def set_pulse_rate(self, pulse_rate): self.set_result("pulse_rate", pulse_rate) @@ -102,7 +102,7 @@ class PulseRate(EnvExperiment): class Watchdog(EnvExperiment): def build(self): - self.attr_device("core") + self.setattr_device("core") @kernel def run(self): @@ -113,9 +113,9 @@ class Watchdog(EnvExperiment): class LoopbackCount(EnvExperiment): def build(self): - self.attr_device("core") - self.attr_device("ttl_inout") - self.attr_argument("npulses") + self.setattr_device("core") + self.setattr_device("ttl_inout") + self.setattr_argument("npulses") def set_count(self, count): self.set_result("count", count) @@ -135,8 +135,8 @@ class LoopbackCount(EnvExperiment): class Underflow(EnvExperiment): def build(self): - self.attr_device("core") - self.attr_device("ttl_out") + self.setattr_device("core") + self.setattr_device("ttl_out") @kernel def run(self): @@ -147,8 +147,8 @@ class Underflow(EnvExperiment): class SequenceError(EnvExperiment): def build(self): - self.attr_device("core") - self.attr_device("ttl_out") + self.setattr_device("core") + self.setattr_device("ttl_out") @kernel def run(self): @@ -160,8 +160,8 @@ class SequenceError(EnvExperiment): class CollisionError(EnvExperiment): def build(self): - self.attr_device("core") - self.attr_device("ttl_out_serdes") + self.setattr_device("core") + self.setattr_device("ttl_out_serdes") @kernel def run(self): @@ -173,7 +173,7 @@ class CollisionError(EnvExperiment): class TimeKeepsRunning(EnvExperiment): def build(self): - self.attr_device("core") + self.setattr_device("core") def set_time_at_start(self, time_at_start): self.set_result("time_at_start", time_at_start) @@ -185,7 +185,7 @@ class TimeKeepsRunning(EnvExperiment): class Handover(EnvExperiment): def build(self): - self.attr_device("core") + self.setattr_device("core") @kernel def get_now(self): @@ -265,8 +265,8 @@ class CoredeviceTest(ExperimentCase): class RPCTiming(EnvExperiment): def build(self): - self.attr_device("core") - self.attr_argument("repeats", FreeValue(100)) + self.setattr_device("core") + self.setattr_argument("repeats", FreeValue(100)) def nop(self, x): pass diff --git a/artiq/test/coredevice_vs_host.py b/artiq/test/coredevice_vs_host.py index 31ef27da4..2a412233f 100644 --- a/artiq/test/coredevice_vs_host.py +++ b/artiq/test/coredevice_vs_host.py @@ -16,9 +16,9 @@ def _run_on_host(k_class, **arguments): class _Primes(EnvExperiment): def build(self): - self.attr_device("core") - self.attr_argument("output_list") - self.attr_argument("maximum") + self.setattr_device("core") + self.setattr_argument("output_list") + self.setattr_argument("maximum") @kernel def run(self): @@ -36,7 +36,7 @@ class _Primes(EnvExperiment): class _Misc(EnvExperiment): def build(self): - self.attr_device("core") + self.setattr_device("core") self.input = 84 self.al = [1, 2, 3, 4, 5] @@ -54,9 +54,9 @@ class _Misc(EnvExperiment): class _PulseLogger(EnvExperiment): def build(self): - self.attr_device("core") - self.attr_argument("output_list") - self.attr_argument("name") + self.setattr_device("core") + self.setattr_argument("output_list") + self.setattr_argument("name") def _append(self, t, l, f): if not hasattr(self, "first_timestamp"): @@ -81,8 +81,8 @@ class _PulseLogger(EnvExperiment): class _Pulses(EnvExperiment): def build(self): - self.attr_device("core") - self.attr_argument("output_list") + self.setattr_device("core") + self.setattr_argument("output_list") for name in "a", "b", "c", "d": pl = _PulseLogger(*self.dbs(), @@ -108,8 +108,8 @@ class _MyException(Exception): class _Exceptions(EnvExperiment): def build(self): - self.attr_device("core") - self.attr_argument("trace") + self.setattr_device("core") + self.setattr_argument("trace") @kernel def run(self): @@ -152,8 +152,8 @@ class _Exceptions(EnvExperiment): class _RPCExceptions(EnvExperiment): def build(self): - self.attr_device("core") - self.attr_argument("catch", FreeValue(False)) + self.setattr_device("core") + self.setattr_argument("catch", FreeValue(False)) self.success = False diff --git a/artiq/test/scheduler.py b/artiq/test/scheduler.py index 33e712fb5..3e5298056 100644 --- a/artiq/test/scheduler.py +++ b/artiq/test/scheduler.py @@ -18,7 +18,7 @@ class EmptyExperiment(EnvExperiment): class BackgroundExperiment(EnvExperiment): def build(self): - self.attr_device("scheduler") + self.setattr_device("scheduler") def run(self): while True: diff --git a/doc/manual/getting_started_core.rst b/doc/manual/getting_started_core.rst index 49f850cfe..b7a9c6f5e 100644 --- a/doc/manual/getting_started_core.rst +++ b/doc/manual/getting_started_core.rst @@ -13,14 +13,14 @@ As a very first step, we will turn on a LED on the core device. Create a file `` class LED(EnvExperiment): def build(self): - self.attr_device("core") - self.attr_device("led") + self.setattr_device("core") + self.setattr_device("led") @kernel def run(self): self.led.on() -The central part of our code is our ``LED`` class, that derives from :class:`artiq.language.environment.EnvExperiment`. Among other features, ``EnvExperiment`` calls our ``build`` method and provides the ``attr_device`` method that interfaces to the device database to create the appropriate device drivers and make those drivers accessible as ``self.core`` and ``self.led``. The ``@kernel`` decorator tells the system that the ``run`` method must be executed on the core device (instead of the host). The decorator uses ``self.core`` internally, which is why we request the core device using ``attr_device`` like any other. +The central part of our code is our ``LED`` class, that derives from :class:`artiq.language.environment.EnvExperiment`. Among other features, ``EnvExperiment`` calls our ``build`` method and provides the ``setattr_device`` method that interfaces to the device database to create the appropriate device drivers and make those drivers accessible as ``self.core`` and ``self.led``. The ``@kernel`` decorator tells the system that the ``run`` method must be executed on the core device (instead of the host). The decorator uses ``self.core`` internally, which is why we request the core device using ``setattr_device`` like any other. Copy the files ``ddb.pyon`` and ``pdb.pyon`` (containing the device and parameter databases) from the ``examples/master`` folder of ARTIQ into the same directory as ``led.py`` (alternatively, you can use the ``-d`` and ``-p`` options of ``artiq_run``). You can open the database files using a text editor - their contents are in a human-readable format. You will probably want to set the IP address of the core device in ``ddb.pyon`` so that the computer can connect to it (it is the ``host`` parameter of the ``comm`` entry). See :ref:`ddb` for more information. The example device database is designed for the NIST QC1 hardware on the KC705; see :ref:`board-ports` for RTIO channel assignments if you need to adapt the device database to a different hardware platform. @@ -48,8 +48,8 @@ Modify the code as follows: :: class LED(EnvExperiment): def build(self): - self.attr_device("core") - self.attr_device("led") + self.setattr_device("core") + self.setattr_device("led") @kernel def run(self): @@ -98,8 +98,8 @@ Create a new file ``rtio.py`` containing the following: :: class Tutorial(EnvExperiment): def build(self): - self.attr_device("core") - self.attr_device("ttl0") + self.setattr_device("core") + self.setattr_device("ttl0") @kernel def run(self): @@ -122,8 +122,8 @@ Try reducing the period of the generated waveform until the CPU cannot keep up w class Tutorial(EnvExperiment): def build(self): - self.attr_device("core") - self.attr_device("ttl0") + self.setattr_device("core") + self.setattr_device("ttl0") @kernel def run(self): diff --git a/doc/manual/getting_started_mgmt.rst b/doc/manual/getting_started_mgmt.rst index 5f0175999..40c8a2054 100644 --- a/doc/manual/getting_started_mgmt.rst +++ b/doc/manual/getting_started_mgmt.rst @@ -53,7 +53,7 @@ Experiments may have arguments whose values can be set in the GUI and used in th def build(self): - self.attr_argument("count", NumberValue(ndecimals=0)) + self.setattr_argument("count", NumberValue(ndecimals=0)) def run(self): for i in range(int(self.count)): diff --git a/examples/master/repository/arguments_demo.py b/examples/master/repository/arguments_demo.py index 08ef43ba2..dad66d8a2 100644 --- a/examples/master/repository/arguments_demo.py +++ b/examples/master/repository/arguments_demo.py @@ -3,10 +3,10 @@ from artiq import * class SubComponent1(HasEnvironment): def build(self): - self.attr_argument("sc1_scan", Scannable(default=NoScan(325)), - "Flux capacitor") - self.attr_argument("sc1_enum", EnumerationValue(["1", "2", "3"]), - "Flux capacitor") + self.setattr_argument("sc1_scan", Scannable(default=NoScan(325)), + "Flux capacitor") + self.setattr_argument("sc1_enum", EnumerationValue(["1", "2", "3"]), + "Flux capacitor") def do(self): print("SC1:") @@ -17,12 +17,12 @@ class SubComponent1(HasEnvironment): class SubComponent2(HasEnvironment): def build(self): - self.attr_argument("sc2_boolean", BooleanValue(False), - "Transporter") - self.attr_argument("sc2_scan", Scannable(default=NoScan(325)), - "Transporter") - self.attr_argument("sc2_enum", EnumerationValue(["3", "4", "5"]), - "Transporter") + self.setattr_argument("sc2_boolean", BooleanValue(False), + "Transporter") + self.setattr_argument("sc2_scan", Scannable(default=NoScan(325)), + "Transporter") + self.setattr_argument("sc2_enum", EnumerationValue(["3", "4", "5"]), + "Transporter") def do(self): print("SC2:") @@ -34,15 +34,15 @@ class SubComponent2(HasEnvironment): class ArgumentsDemo(EnvExperiment): def build(self): - self.attr_argument("free_value", FreeValue(None)) - self.attr_argument("number", NumberValue(42, unit="s", step=0.1, - ndecimals=4)) - self.attr_argument("string", StringValue("Hello World")) - self.attr_argument("scan", Scannable(global_max=400, - default=NoScan(325), - ndecimals=6)) - self.attr_argument("boolean", BooleanValue(True), "Group") - self.attr_argument("enum", EnumerationValue( + self.setattr_argument("free_value", FreeValue(None)) + self.setattr_argument("number", NumberValue(42, unit="s", step=0.1, + ndecimals=4)) + self.setattr_argument("string", StringValue("Hello World")) + self.setattr_argument("scan", Scannable(global_max=400, + default=NoScan(325), + ndecimals=6)) + self.setattr_argument("boolean", BooleanValue(True), "Group") + self.setattr_argument("enum", EnumerationValue( ["foo", "bar", "quux"], "foo"), "Group") self.sc1 = SubComponent1(parent=self) diff --git a/examples/master/repository/dds_test.py b/examples/master/repository/dds_test.py index ff4458abc..cc2d4e6db 100644 --- a/examples/master/repository/dds_test.py +++ b/examples/master/repository/dds_test.py @@ -5,15 +5,15 @@ class DDSTest(EnvExperiment): """DDS test""" def build(self): - self.attr_device("core") - self.attr_device("dds_bus") - self.attr_device("dds0") - self.attr_device("dds1") - self.attr_device("dds2") - self.attr_device("ttl0") - self.attr_device("ttl1") - self.attr_device("ttl2") - self.attr_device("led") + self.setattr_device("core") + self.setattr_device("dds_bus") + self.setattr_device("dds0") + self.setattr_device("dds1") + self.setattr_device("dds2") + self.setattr_device("ttl0") + self.setattr_device("ttl1") + self.setattr_device("ttl2") + self.setattr_device("led") @kernel def run(self): diff --git a/examples/master/repository/flopping_f_simulation.py b/examples/master/repository/flopping_f_simulation.py index f0b4f4461..90cb2468d 100644 --- a/examples/master/repository/flopping_f_simulation.py +++ b/examples/master/repository/flopping_f_simulation.py @@ -27,14 +27,14 @@ class FloppingF(EnvExperiment): """Flopping F simulation""" def build(self): - self.attr_argument("frequency_scan", Scannable( + self.setattr_argument("frequency_scan", Scannable( default=LinearScan(1000, 2000, 100))) - self.attr_argument("F0", NumberValue(1500, min=1000, max=2000)) - self.attr_argument("noise_amplitude", NumberValue(0.1, min=0, max=100, + self.setattr_argument("F0", NumberValue(1500, min=1000, max=2000)) + self.setattr_argument("noise_amplitude", NumberValue(0.1, min=0, max=100, step=0.01)) - self.attr_device("scheduler") + self.setattr_device("scheduler") def run(self): frequency = self.set_result("flopping_f_frequency", [], diff --git a/examples/master/repository/handover.py b/examples/master/repository/handover.py index 5870d27ba..ef3512bf8 100644 --- a/examples/master/repository/handover.py +++ b/examples/master/repository/handover.py @@ -3,8 +3,8 @@ from artiq import * class Handover(EnvExperiment): def build(self): - self.attr_device("core") - self.attr_device("led") + self.setattr_device("core") + self.setattr_device("led") @kernel def blink_once(self): diff --git a/examples/master/repository/mandelbrot.py b/examples/master/repository/mandelbrot.py index a14e58900..b03fa81d8 100644 --- a/examples/master/repository/mandelbrot.py +++ b/examples/master/repository/mandelbrot.py @@ -7,7 +7,7 @@ class Mandelbrot(EnvExperiment): """Mandelbrot set demo""" def build(self): - self.attr_device("core") + self.setattr_device("core") def col(self, i): sys.stdout.write(" .,-:;i+hHM$*#@ "[i]) diff --git a/examples/master/repository/photon_histogram.py b/examples/master/repository/photon_histogram.py index bad44e1b7..99f66c388 100644 --- a/examples/master/repository/photon_histogram.py +++ b/examples/master/repository/photon_histogram.py @@ -5,20 +5,20 @@ class PhotonHistogram(EnvExperiment): """Photon histogram""" def build(self): - self.attr_device("core") - self.attr_device("dds_bus") - self.attr_device("bd_dds") - self.attr_device("bd_sw") - self.attr_device("bdd_dds") - self.attr_device("bdd_sw") - self.attr_device("pmt") + self.setattr_device("core") + self.setattr_device("dds_bus") + self.setattr_device("bd_dds") + self.setattr_device("bd_sw") + self.setattr_device("bdd_dds") + self.setattr_device("bdd_sw") + self.setattr_device("pmt") - self.attr_argument("nbins", FreeValue(100)) - self.attr_argument("repeats", FreeValue(100)) + self.setattr_argument("nbins", FreeValue(100)) + self.setattr_argument("repeats", FreeValue(100)) - self.attr_parameter("cool_f", 230*MHz) - self.attr_parameter("detect_f", 220*MHz) - self.attr_parameter("detect_t", 100*us) + self.setattr_parameter("cool_f", 230*MHz) + self.setattr_parameter("detect_f", 220*MHz) + self.setattr_parameter("detect_t", 100*us) @kernel def program_cooling(self): diff --git a/examples/master/repository/speed_benchmark.py b/examples/master/repository/speed_benchmark.py index 1105d7f4c..8d8921bcb 100644 --- a/examples/master/repository/speed_benchmark.py +++ b/examples/master/repository/speed_benchmark.py @@ -13,7 +13,7 @@ class _PayloadNOP(EnvExperiment): class _PayloadCoreNOP(EnvExperiment): def build(self): - self.attr_device("core") + self.setattr_device("core") @kernel def run(self): @@ -22,7 +22,7 @@ class _PayloadCoreNOP(EnvExperiment): class _PayloadCoreSend100Ints(EnvExperiment): def build(self): - self.attr_device("core") + self.setattr_device("core") def devnull(self, d): pass @@ -35,7 +35,7 @@ class _PayloadCoreSend100Ints(EnvExperiment): class _PayloadCoreSend1MB(EnvExperiment): def build(self): - self.attr_device("core") + self.setattr_device("core") def devnull(self, d): pass @@ -48,7 +48,7 @@ class _PayloadCoreSend1MB(EnvExperiment): class _PayloadCorePrimes(EnvExperiment): def build(self): - self.attr_device("core") + self.setattr_device("core") def devnull(self, d): pass @@ -70,17 +70,17 @@ class _PayloadCorePrimes(EnvExperiment): class SpeedBenchmark(EnvExperiment): """Speed benchmark""" def build(self): - self.attr_argument("mode", EnumerationValue(["Single experiment", - "With pause", - "With scheduler"])) - self.attr_argument("payload", EnumerationValue(["NOP", - "CoreNOP", - "CoreSend100Ints", - "CoreSend1MB", - "CorePrimes"])) - self.attr_argument("nruns", NumberValue(10, min=1, max=1000, ndecimals=0)) - self.attr_device("core") - self.attr_device("scheduler") + self.setattr_argument("mode", EnumerationValue(["Single experiment", + "With pause", + "With scheduler"])) + self.setattr_argument("payload", EnumerationValue(["NOP", + "CoreNOP", + "CoreSend100Ints", + "CoreSend1MB", + "CorePrimes"])) + self.setattr_argument("nruns", NumberValue(10, min=1, max=1000, ndecimals=0)) + self.setattr_device("core") + self.setattr_device("scheduler") def run_with_scheduler(self): nruns = int(self.nruns) @@ -128,8 +128,8 @@ class SpeedBenchmark(EnvExperiment): class _Report(EnvExperiment): def build(self): - self.attr_argument("start_time") - self.attr_argument("nruns") + self.setattr_argument("start_time") + self.setattr_argument("nruns") def run(self): end_time = time.monotonic() diff --git a/examples/master/repository/tdr.py b/examples/master/repository/tdr.py index 9058a06fd..a6ac2b127 100644 --- a/examples/master/repository/tdr.py +++ b/examples/master/repository/tdr.py @@ -31,9 +31,9 @@ class TDR(EnvExperiment): This is also equivalent to a loopback tester or a delay measurement. """ def build(self): - self.attr_device("core") - self.attr_device("pmt0") - self.attr_device("ttl2") + self.setattr_device("core") + self.setattr_device("pmt0") + self.setattr_device("ttl2") def run(self): n = 1000 # repetitions diff --git a/examples/master/repository/transport.py b/examples/master/repository/transport.py index 9d5af3d6c..57100d244 100644 --- a/examples/master/repository/transport.py +++ b/examples/master/repository/transport.py @@ -16,16 +16,16 @@ class Transport(EnvExperiment): """Transport""" def build(self): - self.attr_device("core") - self.attr_device("bd") - self.attr_device("bdd") - self.attr_device("pmt") - self.attr_device("electrodes") + self.setattr_device("core") + self.setattr_device("bd") + self.setattr_device("bdd") + self.setattr_device("pmt") + self.setattr_device("electrodes") - self.attr_argument("wait_at_stop", FreeValue(100*us)) - self.attr_argument("speed", FreeValue(1.5)) - self.attr_argument("repeats", FreeValue(100)) - self.attr_argument("nbins", FreeValue(100)) + self.setattr_argument("wait_at_stop", FreeValue(100*us)) + self.setattr_argument("speed", FreeValue(1.5)) + self.setattr_argument("repeats", FreeValue(100)) + self.setattr_argument("nbins", FreeValue(100)) def calc_waveforms(self, stop): t = transport_data["t"][:stop]*self.speed diff --git a/examples/sim/al_spectroscopy.py b/examples/sim/al_spectroscopy.py index 7f52fc32c..df038aa63 100644 --- a/examples/sim/al_spectroscopy.py +++ b/examples/sim/al_spectroscopy.py @@ -5,16 +5,16 @@ class AluminumSpectroscopy(EnvExperiment): """Aluminum spectroscopy (simulation)""" def build(self): - self.attr_device("core") - self.attr_device("mains_sync") - self.attr_device("laser_cooling") - self.attr_device("spectroscopy") - self.attr_device("spectroscopy_b") - self.attr_device("state_detection") - self.attr_device("pmt") - self.attr_parameter("spectroscopy_freq", 432*MHz) - self.attr_argument("photon_limit_low", FreeValue(10)) - self.attr_argument("photon_limit_high", FreeValue(15)) + self.setattr_device("core") + self.setattr_device("mains_sync") + self.setattr_device("laser_cooling") + self.setattr_device("spectroscopy") + self.setattr_device("spectroscopy_b") + self.setattr_device("state_detection") + self.setattr_device("pmt") + self.setattr_parameter("spectroscopy_freq", 432*MHz) + self.setattr_argument("photon_limit_low", FreeValue(10)) + self.setattr_argument("photon_limit_high", FreeValue(15)) @kernel def run(self): diff --git a/examples/sim/simple_simulation.py b/examples/sim/simple_simulation.py index 602fcbd54..59c4875e4 100644 --- a/examples/sim/simple_simulation.py +++ b/examples/sim/simple_simulation.py @@ -5,9 +5,9 @@ class SimpleSimulation(EnvExperiment): """Simple simulation""" def build(self): - self.attr_device("core") + self.setattr_device("core") for wo in "abcd": - self.attr_device(wo) + self.setattr_device(wo) @kernel def run(self): From 0e3927b01adb007024ae14000b57497137b0d77b Mon Sep 17 00:00:00 2001 From: Sebastien Bourdeauducq Date: Sun, 4 Oct 2015 17:38:07 +0800 Subject: [PATCH 09/53] master: support DDB rescan --- artiq/frontend/artiq_client.py | 38 ++++++++++++---------------------- artiq/frontend/artiq_master.py | 3 ++- artiq/master/ddb.py | 21 +++++++++++++++++++ 3 files changed, 36 insertions(+), 26 deletions(-) create mode 100644 artiq/master/ddb.py diff --git a/artiq/frontend/artiq_client.py b/artiq/frontend/artiq_client.py index 216c09809..f3a15fa24 100755 --- a/artiq/frontend/artiq_client.py +++ b/artiq/frontend/artiq_client.py @@ -61,16 +61,6 @@ def get_argparser(): parser_delete.add_argument("rid", type=int, help="run identifier (RID)") - parser_set_device = subparsers.add_parser( - "set-device", help="add or modify a device") - parser_set_device.add_argument("name", help="name of the device") - parser_set_device.add_argument("description", - help="description in PYON format") - - parser_del_device = subparsers.add_parser( - "del-device", help="delete a device") - parser_del_device.add_argument("name", help="name of the device") - parser_set_parameter = subparsers.add_parser( "set-parameter", help="add or modify a parameter") parser_set_parameter.add_argument("name", help="name of the parameter") @@ -87,11 +77,14 @@ def get_argparser(): "what", help="select object to show: schedule/log/devices/parameters") - parser_scan = subparsers.add_parser("scan-repository", - help="trigger a repository (re)scan") - parser_scan.add_argument("revision", default=None, nargs="?", - help="use a specific repository revision " - "(defaults to head)") + subparsers.add_parser( + "scan-ddb", help="trigger a device database (re)scan") + + parser_scan_repos = subparsers.add_parser( + "scan-repository", help="trigger a repository (re)scan") + parser_scan_repos.add_argument("revision", default=None, nargs="?", + help="use a specific repository revision " + "(defaults to head)") return parser @@ -131,14 +124,6 @@ def _action_delete(remote, args): remote.delete(args.rid) -def _action_set_device(remote, args): - remote.set(args.name, pyon.decode(args.description)) - - -def _action_del_device(remote, args): - remote.delete(args.name) - - def _action_set_parameter(remote, args): remote.set(args.name, pyon.decode(args.value)) @@ -147,6 +132,10 @@ def _action_del_parameter(remote, args): remote.delete(args.name) +def _action_scan_ddb(remote, args): + remote.scan() + + def _action_scan_repository(remote, args): remote.scan_async(args.revision) @@ -275,10 +264,9 @@ def main(): target_name = { "submit": "master_schedule", "delete": "master_schedule", - "set_device": "master_ddb", - "del_device": "master_ddb", "set_parameter": "master_pdb", "del_parameter": "master_pdb", + "scan_ddb": "master_ddb", "scan_repository": "master_repository" }[action] remote = Client(args.server, port, target_name) diff --git a/artiq/frontend/artiq_master.py b/artiq/frontend/artiq_master.py index 2d6fffd18..0373873f9 100755 --- a/artiq/frontend/artiq_master.py +++ b/artiq/frontend/artiq_master.py @@ -7,6 +7,7 @@ import os from artiq.protocols.pc_rpc import Server from artiq.protocols.sync_struct import Notifier, Publisher, process_mod +from artiq.master.ddb import DDB from artiq.protocols.file_db import FlatFileDB from artiq.master.scheduler import Scheduler from artiq.master.worker_db import get_last_rid @@ -64,7 +65,7 @@ def main(): loop = asyncio.get_event_loop() atexit.register(lambda: loop.close()) - ddb = FlatFileDB(args.ddb) + ddb = DDB(args.ddb) pdb = FlatFileDB(args.pdb) rtr = Notifier(dict()) log = Log(1000) diff --git a/artiq/master/ddb.py b/artiq/master/ddb.py new file mode 100644 index 000000000..9a2a1b34d --- /dev/null +++ b/artiq/master/ddb.py @@ -0,0 +1,21 @@ +from artiq.protocols.sync_struct import Notifier +from artiq.protocols import pyon + + +class DDB: + def __init__(self, backing_file): + self.backing_file = backing_file + self.data = Notifier(pyon.load_file(self.backing_file)) + + def scan(self): + new_data = pyon.load_file(self.backing_file) + + for k in list(self.data.read.keys()): + if k not in new_data: + del self.data[k] + for k in new_data.keys(): + if k not in self.data.read or self.data.read[k] != new_data[k]: + self.data[k] = new_data[k] + + def get(self, key): + return self.data.read[key] From 168af9589e53c5a20fbea0439319b3a81b60a99c Mon Sep 17 00:00:00 2001 From: Sebastien Bourdeauducq Date: Sun, 4 Oct 2015 17:55:57 +0800 Subject: [PATCH 10/53] gui/moninj: fix problems with dynamic modification of DDB --- artiq/gui/moninj.py | 30 ++++++++++++++++++++---------- 1 file changed, 20 insertions(+), 10 deletions(-) diff --git a/artiq/gui/moninj.py b/artiq/gui/moninj.py index 835fb58ab..c14c099c2 100644 --- a/artiq/gui/moninj.py +++ b/artiq/gui/moninj.py @@ -23,9 +23,9 @@ _mode_enc = { class _TTLWidget(QtGui.QFrame): - def __init__(self, send_to_device, channel, force_out, title): - self.send_to_device = send_to_device + def __init__(self, channel, send_to_device, force_out, title): self.channel = channel + self.send_to_device = send_to_device self.force_out = force_out QtGui.QFrame.__init__(self) @@ -119,7 +119,8 @@ class _TTLWidget(QtGui.QFrame): class _DDSWidget(QtGui.QFrame): - def __init__(self, sysclk, title): + def __init__(self, channel, sysclk, title): + self.channel = channel self.sysclk = sysclk QtGui.QFrame.__init__(self) @@ -163,9 +164,11 @@ class _DeviceManager: self[k] = v def __setitem__(self, k, v): - self.ddb[k] = v if k in self.ttl_widgets: del self[k] + if k in self.dds_widgets: + del self[k] + self.ddb[k] = v if not isinstance(v, dict): return try: @@ -176,14 +179,15 @@ class _DeviceManager: if v["module"] == "artiq.coredevice.ttl": channel = v["arguments"]["channel"] force_out = v["class"] == "TTLOut" - self.ttl_widgets[channel] = _TTLWidget( - self.send_to_device, channel, force_out, title) + self.ttl_widgets[k] = _TTLWidget( + channel, self.send_to_device, force_out, title) self.ttl_cb() if (v["module"] == "artiq.coredevice.dds" and v["class"] in {"AD9858", "AD9914"}): channel = v["arguments"]["channel"] sysclk = v["arguments"]["sysclk"] - self.dds_widgets[channel] = _DDSWidget(sysclk, title) + self.dds_widgets[channel] = _DDSWidget( + channel, sysclk, title) self.dds_cb() except KeyError: pass @@ -191,8 +195,13 @@ class _DeviceManager: def __delitem__(self, k): del self.ddb[k] if k in self.ttl_widgets: + self.ttl_widgets[k].deleteLater() del self.ttl_widgets[k] self.ttl_cb() + if k in self.dds_widgets: + self.dds_widgets[k].deleteLater() + del self.dds_widgets[k] + self.dds_cb() def get_core_addr(self): try: @@ -261,16 +270,17 @@ class MonInj(TaskObject): try: ttl_levels, ttl_oes, ttl_overrides = \ struct.unpack(">QQQ", data[:8*3]) - for channel, w in self.dm.ttl_widgets.items(): + for w in self.dm.ttl_widgets.values(): + channel = w.channel w.set_value(ttl_levels & (1 << channel), ttl_oes & (1 << channel), ttl_overrides & (1 << channel)) dds_data = data[8*3:] ndds = len(dds_data)//4 ftws = struct.unpack(">" + "I"*ndds, dds_data) - for channel, w in self.dm.dds_widgets.items(): + for w in self.dm.dds_widgets.values(): try: - ftw = ftws[channel] + ftw = ftws[w.channel] except KeyError: pass else: From 512bc794843625a6aa5b85ff894918b57330bd85 Mon Sep 17 00:00:00 2001 From: Sebastien Bourdeauducq Date: Sun, 4 Oct 2015 18:27:44 +0800 Subject: [PATCH 11/53] master: consistent db function argument names --- artiq/master/worker_impl.py | 7 ++++--- artiq/protocols/file_db.py | 16 ++++++++-------- 2 files changed, 12 insertions(+), 11 deletions(-) diff --git a/artiq/master/worker_impl.py b/artiq/master/worker_impl.py index f8ff39746..b029e37d8 100644 --- a/artiq/master/worker_impl.py +++ b/artiq/master/worker_impl.py @@ -63,12 +63,13 @@ class LogForwarder: class ParentDDB: - get = make_parent_action("get_device", "name", KeyError) + get_ddb = make_parent_action("get_ddb", "") + get = make_parent_action("get_device", "key", KeyError) class ParentPDB: - get = make_parent_action("get_parameter", "name", KeyError) - set = make_parent_action("set_parameter", "name value") + get = make_parent_action("get_parameter", "key", KeyError) + set = make_parent_action("set_parameter", "key value") update_rt_results = make_parent_action("update_rt_results", "mod") diff --git a/artiq/protocols/file_db.py b/artiq/protocols/file_db.py index 744eff687..089763cc1 100644 --- a/artiq/protocols/file_db.py +++ b/artiq/protocols/file_db.py @@ -13,19 +13,19 @@ class FlatFileDB: def save(self): pyon.store_file(self.filename, self.data.read) - def get(self, name): - return self.data.read[name] + def get(self, key): + return self.data.read[key] - def set(self, name, value): - self.data[name] = value + def set(self, key, value): + self.data[key] = value self.save() timestamp = time() for hook in self.hooks: - hook.set(timestamp, name, value) + hook.set(timestamp, key, value) - def delete(self, name): - del self.data[name] + def delete(self, key): + del self.data[key] self.save() timestamp = time() for hook in self.hooks: - hook.delete(timestamp, name) + hook.delete(timestamp, key) From b3584bc19077c01ae320d1e8dc9e065ab0bf6395 Mon Sep 17 00:00:00 2001 From: Sebastien Bourdeauducq Date: Sun, 4 Oct 2015 18:29:39 +0800 Subject: [PATCH 12/53] language,master,run: support raw access to DDB from experiments. Closes #123 --- artiq/frontend/artiq_master.py | 3 ++- artiq/frontend/artiq_run.py | 3 ++- artiq/language/environment.py | 6 ++++++ artiq/master/ddb.py | 3 +++ artiq/master/repository.py | 12 ++++++++---- artiq/master/worker.py | 4 ++-- artiq/master/worker_db.py | 4 ++++ artiq/master/worker_impl.py | 6 ++++-- 8 files changed, 31 insertions(+), 10 deletions(-) diff --git a/artiq/frontend/artiq_master.py b/artiq/frontend/artiq_master.py index 0373873f9..e0f3b740d 100755 --- a/artiq/frontend/artiq_master.py +++ b/artiq/frontend/artiq_master.py @@ -74,11 +74,12 @@ def main(): repo_backend = GitBackend(args.repository) else: repo_backend = FilesystemBackend(args.repository) - repository = Repository(repo_backend, log.log) + repository = Repository(repo_backend, ddb.get_ddb, log.log) atexit.register(repository.close) repository.scan_async() worker_handlers = { + "get_ddb": ddb.get_ddb, "get_device": ddb.get, "get_parameter": pdb.get, "set_parameter": pdb.set, diff --git a/artiq/frontend/artiq_run.py b/artiq/frontend/artiq_run.py index 794abbf4e..3474f4740 100755 --- a/artiq/frontend/artiq_run.py +++ b/artiq/frontend/artiq_run.py @@ -13,6 +13,7 @@ import h5py from artiq.language.environment import EnvExperiment from artiq.protocols.file_db import FlatFileDB +from artiq.master.ddb import DDB from artiq.master.worker_db import DeviceManager, ResultDB from artiq.tools import * @@ -111,7 +112,7 @@ def run(with_file=False): args = get_argparser(with_file).parse_args() init_logger(args) - dmgr = DeviceManager(FlatFileDB(args.ddb), + dmgr = DeviceManager(DDB(args.ddb), virtual_devices={"scheduler": DummyScheduler()}) pdb = FlatFileDB(args.pdb) pdb.hooks.append(SimpleParamLogger()) diff --git a/artiq/language/environment.py b/artiq/language/environment.py index 99332e9a3..c2fe8ab51 100644 --- a/artiq/language/environment.py +++ b/artiq/language/environment.py @@ -182,6 +182,12 @@ class HasEnvironment: attribute are the same.""" setattr(self, key, self.get_argument(key, processor, group)) + def get_ddb(self): + """Returns the full contents of the device database.""" + if self.__parent is not None: + return self.__parent.get_ddb() + return self.__dmgr.get_ddb() + def get_device(self, key): """Creates and returns a device driver.""" if self.__parent is not None: diff --git a/artiq/master/ddb.py b/artiq/master/ddb.py index 9a2a1b34d..b798422c8 100644 --- a/artiq/master/ddb.py +++ b/artiq/master/ddb.py @@ -17,5 +17,8 @@ class DDB: if k not in self.data.read or self.data.read[k] != new_data[k]: self.data[k] = new_data[k] + def get_ddb(self): + return self.data.read + def get(self, key): return self.data.read[key] diff --git a/artiq/master/repository.py b/artiq/master/repository.py index 7560cd240..c8024104d 100644 --- a/artiq/master/repository.py +++ b/artiq/master/repository.py @@ -12,12 +12,15 @@ from artiq.tools import exc_to_warning logger = logging.getLogger(__name__) -async def _scan_experiments(wd, log): +async def _scan_experiments(wd, get_ddb, log): r = dict() for f in os.listdir(wd): if f.endswith(".py"): try: - worker = Worker({"log": lambda message: log("scan", message)}) + worker = Worker({ + "get_ddb": get_ddb, + "log": lambda message: log("scan", message) + }) try: description = await worker.examine(os.path.join(wd, f)) finally: @@ -53,8 +56,9 @@ def _sync_explist(target, source): class Repository: - def __init__(self, backend, log_fn): + def __init__(self, backend, get_ddb_fn, log_fn): self.backend = backend + self.get_ddb_fn = get_ddb_fn self.log_fn = log_fn self.cur_rev = self.backend.get_head_rev() @@ -77,7 +81,7 @@ class Repository: wd, _ = self.backend.request_rev(new_cur_rev) self.backend.release_rev(self.cur_rev) self.cur_rev = new_cur_rev - new_explist = await _scan_experiments(wd, self.log_fn) + new_explist = await _scan_experiments(wd, self.get_ddb_fn, self.log_fn) _sync_explist(self.explist, new_explist) finally: diff --git a/artiq/master/worker.py b/artiq/master/worker.py index 104cd8a15..e94018785 100644 --- a/artiq/master/worker.py +++ b/artiq/master/worker.py @@ -244,7 +244,7 @@ class Worker: def register(class_name, name, arguments): r[class_name] = {"name": name, "arguments": arguments} self.register_experiment = register - await self._worker_action({"action": "examine", - "file": file}, timeout) + await self._worker_action({"action": "examine", "file": file}, + timeout) del self.register_experiment return r diff --git a/artiq/master/worker_db.py b/artiq/master/worker_db.py index a4664415a..fce88f3a6 100644 --- a/artiq/master/worker_db.py +++ b/artiq/master/worker_db.py @@ -136,6 +136,10 @@ class DeviceManager: self.virtual_devices = virtual_devices self.active_devices = OrderedDict() + def get_ddb(self): + """Returns the full contents of the device database.""" + return self.ddb.get_ddb() + def get(self, name): """Get the device driver or controller client corresponding to a device database entry.""" diff --git a/artiq/master/worker_impl.py b/artiq/master/worker_impl.py index b029e37d8..8f1572cbb 100644 --- a/artiq/master/worker_impl.py +++ b/artiq/master/worker_impl.py @@ -122,7 +122,9 @@ register_experiment = make_parent_action("register_experiment", "class_name name arguments") -class DummyDMGR: +class ExamineDMGR: + get_ddb = make_parent_action("get_ddb", "") + def get(self, name): return None @@ -208,7 +210,7 @@ def main(): f.close() put_object({"action": "completed"}) elif action == "examine": - examine(DummyDMGR(), DummyPDB(), ResultDB(), obj["file"]) + examine(ExamineDMGR(), DummyPDB(), ResultDB(), obj["file"]) put_object({"action": "completed"}) elif action == "terminate": break From 078a37bf1f01bbe297f984d1243056e9f5ddc6c3 Mon Sep 17 00:00:00 2001 From: Sebastien Bourdeauducq Date: Sun, 4 Oct 2015 18:30:29 +0800 Subject: [PATCH 13/53] examples: add basic DDS frequency setter with dynamic DDB scan --- examples/master/repository/dds_setter.py | 25 ++++++++++++++++++++++++ 1 file changed, 25 insertions(+) create mode 100644 examples/master/repository/dds_setter.py diff --git a/examples/master/repository/dds_setter.py b/examples/master/repository/dds_setter.py new file mode 100644 index 000000000..ebcd72bc1 --- /dev/null +++ b/examples/master/repository/dds_setter.py @@ -0,0 +1,25 @@ +from operator import itemgetter + +from artiq import * + + +class DDSSetter(EnvExperiment): + """DDS Setter""" + def build(self): + self.dds = dict() + + ddb = self.get_ddb() + for k, v in sorted(ddb.items(), key=itemgetter(0)): + if (isinstance(v, dict) + and v["type"] == "local" + and v["module"] == "artiq.coredevice.dds" + and v["class"] in {"AD9858", "AD9914"}): + self.dds[k] = { + "driver": self.get_device(k), + "frequency": self.get_argument("{}_frequency".format(k), + NumberValue()) + } + + def run(self): + for k, v in self.dds.items(): + v["driver"].set(v["frequency"]) From e46ba83513f5ab8628c463a252ba43789e0375cf Mon Sep 17 00:00:00 2001 From: Sebastien Bourdeauducq Date: Sun, 4 Oct 2015 22:53:51 +0800 Subject: [PATCH 14/53] rtio/dds: use rio_phy domain to reset FTW tracker. Closes #120 --- artiq/gateware/rtio/phy/dds.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/artiq/gateware/rtio/phy/dds.py b/artiq/gateware/rtio/phy/dds.py index 37dab89f4..f0a8e9c8c 100644 --- a/artiq/gateware/rtio/phy/dds.py +++ b/artiq/gateware/rtio/phy/dds.py @@ -36,13 +36,13 @@ class _AD9xxx(Module): ftws = [Signal(32) for i in range(nchannels)] for c, ftw in enumerate(ftws): if flen(pads.d) == 8: - self.sync.rio += \ + self.sync.rio_phy += \ If(selected(c), [ If(current_address == ftw_base+i, ftw[i*8:(i+1)*8].eq(current_data)) for i in range(4)]) elif flen(pads.d) == 16: - self.sync.rio += \ + self.sync.rio_phy += \ If(selected(c), [ If(current_address == ftw_base+2*i, ftw[i*16:(i+1)*16].eq(current_data)) @@ -51,7 +51,7 @@ class _AD9xxx(Module): raise NotImplementedError # FTW to probe on FUD - self.sync.rio += If(current_address == 2**flen(pads.a), [ + self.sync.rio_phy += If(current_address == 2**flen(pads.a), [ If(selected(c), probe.eq(ftw)) for c, (probe, ftw) in enumerate(zip(self.probes, ftws))]) From fcb668f4655e0e68378f600d7afd6f0b225644aa Mon Sep 17 00:00:00 2001 From: Sebastien Bourdeauducq Date: Sun, 4 Oct 2015 23:03:31 +0800 Subject: [PATCH 15/53] travis,conda: python 3.5 --- .travis.yml | 4 ++-- .travis/get-anaconda.sh | 2 +- conda/artiq/build.sh | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.travis.yml b/.travis.yml index 216fe090a..525180b64 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,6 +1,6 @@ language: python python: - - '3.4' + - '3.5' sudo: false env: global: @@ -12,7 +12,7 @@ before_install: - if [ $BUILD_SOC -ne 0 ]; then ./.travis/get-xilinx.sh; fi - . ./.travis/get-toolchain.sh - . ./.travis/get-anaconda.sh - - source $HOME/miniconda/bin/activate py34 + - source $HOME/miniconda/bin/activate py35 - conda install -q pip coverage anaconda-client migen cython - pip install coveralls install: diff --git a/.travis/get-anaconda.sh b/.travis/get-anaconda.sh index af13fe6e4..790a723ad 100755 --- a/.travis/get-anaconda.sh +++ b/.travis/get-anaconda.sh @@ -9,5 +9,5 @@ conda config --set always_yes yes --set changeps1 no conda update -q conda conda info -a conda install conda-build jinja2 -conda create -q -n py34 python=$TRAVIS_PYTHON_VERSION +conda create -q -n py35 python=$TRAVIS_PYTHON_VERSION conda config --add channels https://conda.anaconda.org/m-labs/channel/dev diff --git a/conda/artiq/build.sh b/conda/artiq/build.sh index 098b19d4f..959e8a692 100755 --- a/conda/artiq/build.sh +++ b/conda/artiq/build.sh @@ -11,7 +11,7 @@ $PYTHON setup.py install --single-version-externally-managed --record=record.txt git clone --recursive https://github.com/m-labs/misoc export MSCDIR=$SRC_DIR/misoc -ARTIQ_PREFIX=$PREFIX/lib/python3.4/site-packages/artiq +ARTIQ_PREFIX=$PREFIX/lib/python3.5/site-packages/artiq BIN_PREFIX=$ARTIQ_PREFIX/binaries/ mkdir -p $ARTIQ_PREFIX/misc mkdir -p $BIN_PREFIX/kc705 $BIN_PREFIX/pipistrello From eaa317e08f635206f592f0c4984a98d689394825 Mon Sep 17 00:00:00 2001 From: Sebastien Bourdeauducq Date: Sun, 4 Oct 2015 23:16:40 +0800 Subject: [PATCH 16/53] travis: switch to py3.5 sooner --- .travis/get-anaconda.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis/get-anaconda.sh b/.travis/get-anaconda.sh index 790a723ad..2bdfb45e8 100755 --- a/.travis/get-anaconda.sh +++ b/.travis/get-anaconda.sh @@ -4,10 +4,10 @@ export PATH=$HOME/miniconda/bin:$PATH wget http://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh -O miniconda.sh bash miniconda.sh -b -p $HOME/miniconda +conda create -q -n py35 python=$TRAVIS_PYTHON_VERSION hash -r conda config --set always_yes yes --set changeps1 no conda update -q conda conda info -a conda install conda-build jinja2 -conda create -q -n py35 python=$TRAVIS_PYTHON_VERSION conda config --add channels https://conda.anaconda.org/m-labs/channel/dev From 3dfbceb5c62afa908adf3a1117bde77c58aa9384 Mon Sep 17 00:00:00 2001 From: Sebastien Bourdeauducq Date: Sun, 4 Oct 2015 23:19:56 +0800 Subject: [PATCH 17/53] travis: yet another attempt at fixing py35 installation... --- .travis/get-anaconda.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis/get-anaconda.sh b/.travis/get-anaconda.sh index 2bdfb45e8..74b7e9221 100755 --- a/.travis/get-anaconda.sh +++ b/.travis/get-anaconda.sh @@ -4,9 +4,9 @@ export PATH=$HOME/miniconda/bin:$PATH wget http://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh -O miniconda.sh bash miniconda.sh -b -p $HOME/miniconda -conda create -q -n py35 python=$TRAVIS_PYTHON_VERSION hash -r conda config --set always_yes yes --set changeps1 no +conda create -q -n py35 python=$TRAVIS_PYTHON_VERSION conda update -q conda conda info -a conda install conda-build jinja2 From 8db4d7a5f9a0dccba9a331a2fbe65df322a8c3c5 Mon Sep 17 00:00:00 2001 From: Sebastien Bourdeauducq Date: Sun, 4 Oct 2015 23:25:02 +0800 Subject: [PATCH 18/53] travis: yet another attempt at fixing py35 installation (2)... --- .travis/get-anaconda.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/.travis/get-anaconda.sh b/.travis/get-anaconda.sh index 74b7e9221..e214bbad3 100755 --- a/.travis/get-anaconda.sh +++ b/.travis/get-anaconda.sh @@ -7,6 +7,7 @@ bash miniconda.sh -b -p $HOME/miniconda hash -r conda config --set always_yes yes --set changeps1 no conda create -q -n py35 python=$TRAVIS_PYTHON_VERSION +source $HOME/miniconda/bin/activate py35 conda update -q conda conda info -a conda install conda-build jinja2 From 0279e2110aef3d9cbbd0e1e0c1b04d8a5d144634 Mon Sep 17 00:00:00 2001 From: Sebastien Bourdeauducq Date: Sun, 4 Oct 2015 23:29:01 +0800 Subject: [PATCH 19/53] travis: yet another attempt at fixing py35 installation (3)... --- .travis/get-anaconda.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis/get-anaconda.sh b/.travis/get-anaconda.sh index e214bbad3..7c77a4607 100755 --- a/.travis/get-anaconda.sh +++ b/.travis/get-anaconda.sh @@ -7,7 +7,7 @@ bash miniconda.sh -b -p $HOME/miniconda hash -r conda config --set always_yes yes --set changeps1 no conda create -q -n py35 python=$TRAVIS_PYTHON_VERSION -source $HOME/miniconda/bin/activate py35 +. $HOME/miniconda/bin/activate py35 conda update -q conda conda info -a conda install conda-build jinja2 From 322cf54aeb77c7998fa620194b3dad4e63e18384 Mon Sep 17 00:00:00 2001 From: Sebastien Bourdeauducq Date: Sun, 4 Oct 2015 23:33:50 +0800 Subject: [PATCH 20/53] travis: yet another attempt at fixing py35 installation (4)... --- .travis/get-anaconda.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.travis/get-anaconda.sh b/.travis/get-anaconda.sh index 7c77a4607..7290395a6 100755 --- a/.travis/get-anaconda.sh +++ b/.travis/get-anaconda.sh @@ -1,4 +1,4 @@ -#!/bin/sh +#!/bin/bash # Copyright (C) 2014, 2015 Robert Jordens export PATH=$HOME/miniconda/bin:$PATH @@ -7,7 +7,7 @@ bash miniconda.sh -b -p $HOME/miniconda hash -r conda config --set always_yes yes --set changeps1 no conda create -q -n py35 python=$TRAVIS_PYTHON_VERSION -. $HOME/miniconda/bin/activate py35 +source $HOME/miniconda/bin/activate py35 conda update -q conda conda info -a conda install conda-build jinja2 From 342e72bed66ee3ea4183ee24b7f8923496b0b6aa Mon Sep 17 00:00:00 2001 From: Sebastien Bourdeauducq Date: Mon, 5 Oct 2015 00:09:36 +0800 Subject: [PATCH 21/53] more travis fixing attempts --- .travis.yml | 2 +- .travis/get-anaconda.sh | 5 ++--- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/.travis.yml b/.travis.yml index 525180b64..2271bbd2e 100644 --- a/.travis.yml +++ b/.travis.yml @@ -16,7 +16,7 @@ before_install: - conda install -q pip coverage anaconda-client migen cython - pip install coveralls install: - - conda build conda/artiq + - conda build --python 3.5 conda/artiq - conda install -q artiq --use-local script: - coverage run --source=artiq setup.py test diff --git a/.travis/get-anaconda.sh b/.travis/get-anaconda.sh index 7290395a6..790a723ad 100755 --- a/.travis/get-anaconda.sh +++ b/.travis/get-anaconda.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/bin/sh # Copyright (C) 2014, 2015 Robert Jordens export PATH=$HOME/miniconda/bin:$PATH @@ -6,9 +6,8 @@ wget http://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh -O min bash miniconda.sh -b -p $HOME/miniconda hash -r conda config --set always_yes yes --set changeps1 no -conda create -q -n py35 python=$TRAVIS_PYTHON_VERSION -source $HOME/miniconda/bin/activate py35 conda update -q conda conda info -a conda install conda-build jinja2 +conda create -q -n py35 python=$TRAVIS_PYTHON_VERSION conda config --add channels https://conda.anaconda.org/m-labs/channel/dev From 66f82a13d3cac15752772409c437ed211d36c920 Mon Sep 17 00:00:00 2001 From: Sebastien Bourdeauducq Date: Tue, 6 Oct 2015 00:30:41 +0800 Subject: [PATCH 22/53] language,gui: support scaling of number entries --- artiq/gui/explorer.py | 18 ++++---- artiq/gui/scan.py | 45 +++++++++++--------- artiq/gui/tools.py | 17 ++++++++ artiq/language/environment.py | 13 ++++-- artiq/language/scan.py | 14 ++++-- examples/master/repository/arguments_demo.py | 6 ++- 6 files changed, 77 insertions(+), 36 deletions(-) diff --git a/artiq/gui/explorer.py b/artiq/gui/explorer.py index fe6cb58cb..75ea4f338 100644 --- a/artiq/gui/explorer.py +++ b/artiq/gui/explorer.py @@ -6,7 +6,7 @@ from pyqtgraph import LayoutWidget from artiq.protocols.sync_struct import Subscriber from artiq.protocols import pyon -from artiq.gui.tools import DictSyncModel +from artiq.gui.tools import si_prefix, DictSyncModel from artiq.gui.scan import ScanController @@ -74,26 +74,28 @@ class _EnumerationEntry(QtGui.QComboBox): class _NumberEntry(QtGui.QDoubleSpinBox): def __init__(self, procdesc): QtGui.QDoubleSpinBox.__init__(self) + self.scale = procdesc["scale"] self.setDecimals(procdesc["ndecimals"]) - self.setSingleStep(procdesc["step"]) + self.setSingleStep(procdesc["step"]/self.scale) if procdesc["min"] is not None: - self.setMinimum(procdesc["min"]) + self.setMinimum(procdesc["min"]/self.scale) else: self.setMinimum(float("-inf")) if procdesc["max"] is not None: - self.setMaximum(procdesc["max"]) + self.setMaximum(procdesc["max"]/self.scale) else: self.setMaximum(float("inf")) - if procdesc["unit"]: - self.setSuffix(" " + procdesc["unit"]) + suffix = si_prefix(self.scale) + procdesc["unit"] + if suffix: + self.setSuffix(" " + suffix) if "default" in procdesc: self.set_argument_value(procdesc["default"]) def get_argument_value(self): - return self.value() + return self.value()*self.scale def set_argument_value(self, value): - self.setValue(value) + self.setValue(value/self.scale) class _StringEntry(QtGui.QLineEdit): diff --git a/artiq/gui/scan.py b/artiq/gui/scan.py index 0065b421d..7ed69ed46 100644 --- a/artiq/gui/scan.py +++ b/artiq/gui/scan.py @@ -1,25 +1,28 @@ from quamash import QtGui from pyqtgraph import LayoutWidget +from artiq.gui.tools import si_prefix + class _Range(LayoutWidget): - def __init__(self, global_min, global_max, global_step, unit, ndecimals): + def __init__(self, global_min, global_max, global_step, suffix, scale, ndecimals): LayoutWidget.__init__(self) + self.scale = scale def apply_properties(spinbox): spinbox.setDecimals(ndecimals) if global_min is not None: - spinbox.setMinimum(global_min) + spinbox.setMinimum(global_min/self.scale) else: spinbox.setMinimum(float("-inf")) if global_max is not None: - spinbox.setMaximum(global_max) + spinbox.setMaximum(global_max/self.scale) else: spinbox.setMaximum(float("inf")) if global_step is not None: - spinbox.setSingleStep(global_step) - if unit: - spinbox.setSuffix(" " + unit) + spinbox.setSingleStep(global_step/self.scale) + if suffix: + spinbox.setSuffix(" " + suffix) self.addWidget(QtGui.QLabel("Min:"), 0, 0) self.min = QtGui.QDoubleSpinBox() @@ -38,8 +41,8 @@ class _Range(LayoutWidget): self.addWidget(self.npoints, 0, 5) def set_values(self, min, max, npoints): - self.min.setValue(min) - self.max.setValue(max) + self.min.setValue(min/self.scale) + self.max.setValue(max/self.scale) self.npoints.setValue(npoints) def get_values(self): @@ -48,8 +51,8 @@ class _Range(LayoutWidget): if min > max: raise ValueError("Minimum scan boundary must be less than maximum") return { - "min": min, - "max": max, + "min": min*self.scale, + "max": max*self.scale, "npoints": self.npoints.value() } @@ -61,33 +64,35 @@ class ScanController(LayoutWidget): self.stack = QtGui.QStackedWidget() self.addWidget(self.stack, 1, 0, colspan=4) + self.scale = procdesc["scale"] + gmin, gmax = procdesc["global_min"], procdesc["global_max"] gstep = procdesc["global_step"] - unit = procdesc["unit"] + suffix = si_prefix(self.scale) + procdesc["unit"] ndecimals = procdesc["ndecimals"] self.v_noscan = QtGui.QDoubleSpinBox() self.v_noscan.setDecimals(ndecimals) if gmin is not None: - self.v_noscan.setMinimum(gmin) + self.v_noscan.setMinimum(gmin/self.scale) else: self.v_noscan.setMinimum(float("-inf")) if gmax is not None: - self.v_noscan.setMaximum(gmax) + self.v_noscan.setMaximum(gmax/self.scale) else: self.v_noscan.setMaximum(float("inf")) - self.v_noscan.setSingleStep(gstep) - if unit: - self.v_noscan.setSuffix(" " + unit) + self.v_noscan.setSingleStep(gstep/self.scale) + if suffix: + self.v_noscan.setSuffix(" " + suffix) self.v_noscan_gr = LayoutWidget() self.v_noscan_gr.addWidget(QtGui.QLabel("Value:"), 0, 0) self.v_noscan_gr.addWidget(self.v_noscan, 0, 1) self.stack.addWidget(self.v_noscan_gr) - self.v_linear = _Range(gmin, gmax, gstep, unit, ndecimals) + self.v_linear = _Range(gmin, gmax, gstep, suffix, self.scale, ndecimals) self.stack.addWidget(self.v_linear) - self.v_random = _Range(gmin, gmax, gstep, unit, ndecimals) + self.v_random = _Range(gmin, gmax, gstep, suffix, self.scale, ndecimals) self.stack.addWidget(self.v_random) self.v_explicit = QtGui.QLineEdit() @@ -124,7 +129,7 @@ class ScanController(LayoutWidget): def get_argument_value(self): if self.noscan.isChecked(): - return {"ty": "NoScan", "value": self.v_noscan.value()} + return {"ty": "NoScan", "value": self.v_noscan.value()*self.scale} elif self.linear.isChecked(): d = {"ty": "LinearScan"} d.update(self.v_linear.get_values()) @@ -140,7 +145,7 @@ class ScanController(LayoutWidget): def set_argument_value(self, d): if d["ty"] == "NoScan": self.noscan.setChecked(True) - self.v_noscan.setValue(d["value"]) + self.v_noscan.setValue(d["value"]/self.scale) elif d["ty"] == "LinearScan": self.linear.setChecked(True) self.v_linear.set_values(d["min"], d["max"], d["npoints"]) diff --git a/artiq/gui/tools.py b/artiq/gui/tools.py index ecce285ed..265a46491 100644 --- a/artiq/gui/tools.py +++ b/artiq/gui/tools.py @@ -37,6 +37,23 @@ def short_format(v): return r +def si_prefix(scale): + try: + return { + 1e-12: "p", + 1e-9: "n", + 1e-6: "u", + 1e-3: "m", + 1.0: "", + 1e3: "k", + 1e6: "M", + 1e9: "G", + 1e12: "T" + }[scale] + except KeyError: + return "[x{}]".format(scale) + + class _SyncSubstruct: def __init__(self, update_cb, ref): self.update_cb = update_cb diff --git a/artiq/language/environment.py b/artiq/language/environment.py index c2fe8ab51..b2b1c2a1d 100644 --- a/artiq/language/environment.py +++ b/artiq/language/environment.py @@ -73,16 +73,22 @@ class NumberValue(_SimpleArgProcessor): :param unit: A string representing the unit of the value, for user interface (UI) purposes. + :param scale: The scale of value for UI purposes. The corresponding SI + prefix is shown in front of the unit, and the displayed value is + divided by the scale. :param step: The step with which the value should be modified by up/down - buttons in a UI. + buttons in a UI. The default is the scale divided by 10. :param min: The minimum value of the argument. :param max: The maximum value of the argument. :param ndecimals: The number of decimals a UI should use. """ - def __init__(self, default=NoDefault, unit="", step=1.0, - min=None, max=None, ndecimals=2): + def __init__(self, default=NoDefault, unit="", scale=1.0, + step=None, min=None, max=None, ndecimals=2): + if step is None: + step = scale/10.0 _SimpleArgProcessor.__init__(self, default) self.unit = unit + self.scale = scale self.step = step self.min = min self.max = max @@ -91,6 +97,7 @@ class NumberValue(_SimpleArgProcessor): def describe(self): d = _SimpleArgProcessor.describe(self) d["unit"] = self.unit + d["scale"] = self.scale d["step"] = self.step d["min"] = self.min d["max"] = self.max diff --git a/artiq/language/scan.py b/artiq/language/scan.py index 9f4b9e468..dabd49af8 100644 --- a/artiq/language/scan.py +++ b/artiq/language/scan.py @@ -121,17 +121,24 @@ class Scannable: range of its input widgets. :param global_max: Same as global_min, but for the maximum value. :param global_step: The step with which the value should be modified by - up/down buttons in a user interface. + up/down buttons in a user interface. The default is the scale divided + by 10. :param unit: A string representing the unit of the scanned variable, for user interface (UI) purposes. + :param scale: The scale of value for UI purposes. The corresponding SI + prefix is shown in front of the unit, and the displayed value is + divided by the scale. :param ndecimals: The number of decimals a UI should use. """ - def __init__(self, default=NoDefault, unit="", - global_step=1.0, global_min=None, global_max=None, + def __init__(self, default=NoDefault, unit="", scale=1.0, + global_step=None, global_min=None, global_max=None, ndecimals=2): + if global_step is None: + global_step = scale/10.0 if default is not NoDefault: self.default_value = default self.unit = unit + self.scale = scale self.global_step = global_step self.global_min = global_min self.global_max = global_max @@ -155,6 +162,7 @@ class Scannable: if hasattr(self, "default_value"): d["default"] = self.default_value.describe() d["unit"] = self.unit + d["scale"] = self.scale d["global_step"] = self.global_step d["global_min"] = self.global_min d["global_max"] = self.global_max diff --git a/examples/master/repository/arguments_demo.py b/examples/master/repository/arguments_demo.py index dad66d8a2..d52083faa 100644 --- a/examples/master/repository/arguments_demo.py +++ b/examples/master/repository/arguments_demo.py @@ -3,7 +3,8 @@ from artiq import * class SubComponent1(HasEnvironment): def build(self): - self.setattr_argument("sc1_scan", Scannable(default=NoScan(325)), + self.setattr_argument("sc1_scan", Scannable(default=NoScan(3250), + scale=1e3, unit="Hz"), "Flux capacitor") self.setattr_argument("sc1_enum", EnumerationValue(["1", "2", "3"]), "Flux capacitor") @@ -35,7 +36,8 @@ class SubComponent2(HasEnvironment): class ArgumentsDemo(EnvExperiment): def build(self): self.setattr_argument("free_value", FreeValue(None)) - self.setattr_argument("number", NumberValue(42, unit="s", step=0.1, + self.setattr_argument("number", NumberValue(42e-6, + unit="s", scale=1e-6, ndecimals=4)) self.setattr_argument("string", StringValue("Hello World")) self.setattr_argument("scan", Scannable(global_max=400, From d94f0211a628fe6f7fadd897e7b61afd92bdbeee Mon Sep 17 00:00:00 2001 From: Sebastien Bourdeauducq Date: Tue, 6 Oct 2015 13:35:30 +0800 Subject: [PATCH 23/53] test/scheduler: cleanup --- artiq/test/scheduler.py | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/artiq/test/scheduler.py b/artiq/test/scheduler.py index 3e5298056..89e89e146 100644 --- a/artiq/test/scheduler.py +++ b/artiq/test/scheduler.py @@ -57,11 +57,6 @@ def _get_basic_steps(rid, expid, priority=0, flush=False): ] -_handlers = { - "init_rt_results": lambda description: None -} - - class SchedulerCase(unittest.TestCase): def setUp(self): if os.name == "nt": @@ -72,7 +67,7 @@ class SchedulerCase(unittest.TestCase): def test_steps(self): loop = self.loop - scheduler = Scheduler(0, _handlers, None) + scheduler = Scheduler(0, dict(), None) expid = _get_expid("EmptyExperiment") expect = _get_basic_steps(1, expid) @@ -108,7 +103,7 @@ class SchedulerCase(unittest.TestCase): def test_pause(self): loop = self.loop - scheduler = Scheduler(0, _handlers, None) + scheduler = Scheduler(0, dict(), None) expid_bg = _get_expid("BackgroundExperiment") expid = _get_expid("EmptyExperiment") @@ -139,7 +134,7 @@ class SchedulerCase(unittest.TestCase): def test_flush(self): loop = self.loop - scheduler = Scheduler(0, _handlers, None) + scheduler = Scheduler(0, dict(), None) expid = _get_expid("EmptyExperiment") expect = _get_basic_steps(1, expid, 1, True) From 139072d402d00a9fc78d92fbc0d21ab44e217ec8 Mon Sep 17 00:00:00 2001 From: Sebastien Bourdeauducq Date: Tue, 6 Oct 2015 13:50:00 +0800 Subject: [PATCH 24/53] Graceful experiment termination. Closes #76 --- artiq/frontend/artiq_client.py | 7 +++++- artiq/gui/schedule.py | 18 ++++++++++----- artiq/language/core.py | 8 ++++++- artiq/master/scheduler.py | 18 ++++++++++++++- artiq/master/worker.py | 4 ++-- artiq/master/worker_impl.py | 8 +++++-- artiq/test/scheduler.py | 40 ++++++++++++++++++++++++++++------ 7 files changed, 84 insertions(+), 19 deletions(-) diff --git a/artiq/frontend/artiq_client.py b/artiq/frontend/artiq_client.py index f3a15fa24..c5e153302 100755 --- a/artiq/frontend/artiq_client.py +++ b/artiq/frontend/artiq_client.py @@ -58,6 +58,8 @@ def get_argparser(): parser_delete = subparsers.add_parser("delete", help="delete an experiment " "from the schedule") + parser_delete.add_argument("-g", action="store_true", + help="request graceful termination") parser_delete.add_argument("rid", type=int, help="run identifier (RID)") @@ -121,7 +123,10 @@ def _action_submit(remote, args): def _action_delete(remote, args): - remote.delete(args.rid) + if args.g: + remote.request_termination(args.rid) + else: + remote.delete(args.rid) def _action_set_parameter(remote, args): diff --git a/artiq/gui/schedule.py b/artiq/gui/schedule.py index f1606caf9..53d5197e9 100644 --- a/artiq/gui/schedule.py +++ b/artiq/gui/schedule.py @@ -1,5 +1,6 @@ import asyncio import time +from functools import partial from quamash import QtGui, QtCore from pyqtgraph import dockarea @@ -71,10 +72,14 @@ class ScheduleDock(dockarea.Dock): self.addWidget(self.table) self.table.setContextMenuPolicy(QtCore.Qt.ActionsContextMenu) + request_termination_action = QtGui.QAction("Request termination", self.table) + request_termination_action.triggered.connect(partial(self.delete_clicked, True)) + self.table.addAction(request_termination_action) delete_action = QtGui.QAction("Delete", self.table) - delete_action.triggered.connect(self.delete_clicked) + delete_action.triggered.connect(partial(self.delete_clicked, False)) self.table.addAction(delete_action) + async def sub_connect(self, host, port): self.subscriber = Subscriber("schedule", self.init_schedule_model) await self.subscriber.connect(host, port) @@ -87,13 +92,16 @@ class ScheduleDock(dockarea.Dock): self.table.setModel(self.table_model) return self.table_model - async def delete(self, rid): - await self.schedule_ctl.delete(rid) + async def delete(self, rid, graceful): + if graceful: + await self.schedule_ctl.request_termination(rid) + else: + await self.schedule_ctl.delete(rid) - def delete_clicked(self): + def delete_clicked(self, graceful): idx = self.table.selectedIndexes() if idx: row = idx[0].row() rid = self.table_model.row_to_key[row] self.status_bar.showMessage("Deleted RID {}".format(rid)) - asyncio.ensure_future(self.delete(rid)) + asyncio.ensure_future(self.delete(rid, graceful)) diff --git a/artiq/language/core.py b/artiq/language/core.py index a4ca0b883..da362a0e5 100644 --- a/artiq/language/core.py +++ b/artiq/language/core.py @@ -6,7 +6,8 @@ from collections import namedtuple from functools import wraps -__all__ = ["int64", "round64", "kernel", "portable", +__all__ = ["int64", "round64", "TerminationRequested", + "kernel", "portable", "set_time_manager", "set_syscall_manager", "set_watchdog_factory", "RuntimeException", "EncodedException"] @@ -77,6 +78,11 @@ def round64(x): return int64(round(x)) +class TerminationRequested(Exception): + """Raised by ``pause`` when the user has requested termination.""" + pass + + _KernelFunctionInfo = namedtuple("_KernelFunctionInfo", "core_name k_function") diff --git a/artiq/master/scheduler.py b/artiq/master/scheduler.py index fdd925287..41abce3db 100644 --- a/artiq/master/scheduler.py +++ b/artiq/master/scheduler.py @@ -57,6 +57,7 @@ class Run: self.flush = flush self.worker = Worker(pool.worker_handlers) + self.termination_requested = False self._status = RunStatus.pending @@ -267,7 +268,12 @@ class RunStage(TaskObject): try: if run.status == RunStatus.paused: run.status = RunStatus.running - completed = await run.resume() + # clear "termination requested" flag now + # so that if it is set again during the resume, this + # results in another exception. + request_termination = run.termination_requested + run.termination_requested = False + completed = await run.resume(request_termination) else: run.status = RunStatus.running completed = await run.run() @@ -422,3 +428,13 @@ class Scheduler: def delete(self, rid): self._deleter.delete(rid) + + def request_termination(self, rid): + for pipeline in self._pipelines.values(): + if rid in pipeline.pool.runs: + run = pipeline.pool.runs[rid] + if run.status == RunStatus.running or run.status == RunStatus.paused: + run.termination_requested = True + else: + self.delete(rid) + break diff --git a/artiq/master/worker.py b/artiq/master/worker.py index e94018785..faa2dcf18 100644 --- a/artiq/master/worker.py +++ b/artiq/master/worker.py @@ -221,12 +221,12 @@ class Worker: self.yield_time = time.monotonic() return completed - async def resume(self): + async def resume(self, request_termination): stop_duration = time.monotonic() - self.yield_time for wid, expiry in self.watchdogs: self.watchdogs[wid] += stop_duration completed = await self._worker_action({"status": "ok", - "data": None}) + "data": request_termination}) if not completed: self.yield_time = time.monotonic() return completed diff --git a/artiq/master/worker_impl.py b/artiq/master/worker_impl.py index 8f1572cbb..db1229d2e 100644 --- a/artiq/master/worker_impl.py +++ b/artiq/master/worker_impl.py @@ -6,7 +6,7 @@ from artiq.protocols import pyon from artiq.tools import file_import from artiq.master.worker_db import DeviceManager, ResultDB, get_hdf5_output from artiq.language.environment import is_experiment -from artiq.language.core import set_watchdog_factory +from artiq.language.core import set_watchdog_factory, TerminationRequested def get_object(): @@ -93,7 +93,11 @@ set_watchdog_factory(Watchdog) class Scheduler: - pause = staticmethod(make_parent_action("pause", "")) + pause_noexc = staticmethod(make_parent_action("pause", "")) + + def pause(self): + if self.pause_noexc(): + raise TerminationRequested submit = staticmethod(make_parent_action("scheduler_submit", "pipeline_name expid priority due_date flush")) diff --git a/artiq/test/scheduler.py b/artiq/test/scheduler.py index 89e89e146..e92ac638c 100644 --- a/artiq/test/scheduler.py +++ b/artiq/test/scheduler.py @@ -21,9 +21,12 @@ class BackgroundExperiment(EnvExperiment): self.setattr_device("scheduler") def run(self): - while True: - self.scheduler.pause() - sleep(0.2) + try: + while True: + self.scheduler.pause() + sleep(0.2) + except TerminationRequested: + self.set_parameter("termination_ok", True) def _get_expid(name): @@ -103,13 +106,25 @@ class SchedulerCase(unittest.TestCase): def test_pause(self): loop = self.loop - scheduler = Scheduler(0, dict(), None) + + termination_ok = False + def check_termination(key, value): + nonlocal termination_ok + self.assertEqual(key, "termination_ok") + self.assertEqual(value, True) + termination_ok = True + handlers = { + "set_parameter": check_termination + } + scheduler = Scheduler(0, handlers, None) + expid_bg = _get_expid("BackgroundExperiment") expid = _get_expid("EmptyExperiment") expect = _get_basic_steps(1, expid) background_running = asyncio.Event() - done = asyncio.Event() + empty_completed = asyncio.Event() + background_completed = asyncio.Event() expect_idx = 0 def notify(mod): nonlocal expect_idx @@ -118,18 +133,29 @@ class SchedulerCase(unittest.TestCase): "key": "status", "action": "setitem"}: background_running.set() + if mod == {"path": [0], + "value": "deleting", + "key": "status", + "action": "setitem"}: + background_completed.set() if mod["path"] == [1] or (mod["path"] == [] and mod["key"] == 1): self.assertEqual(mod, expect[expect_idx]) expect_idx += 1 if expect_idx >= len(expect): - done.set() + empty_completed.set() scheduler.notifier.publish = notify scheduler.start() scheduler.submit("main", expid_bg, -99, None, False) loop.run_until_complete(background_running.wait()) scheduler.submit("main", expid, 0, None, False) - loop.run_until_complete(done.wait()) + loop.run_until_complete(empty_completed.wait()) + + self.assertFalse(termination_ok) + scheduler.request_termination(0) + loop.run_until_complete(background_completed.wait()) + self.assertTrue(termination_ok) + loop.run_until_complete(scheduler.stop()) def test_flush(self): From 3cf53667c8e2c5b4fe5a87ae7a3624aba0139590 Mon Sep 17 00:00:00 2001 From: Sebastien Bourdeauducq Date: Tue, 6 Oct 2015 13:50:29 +0800 Subject: [PATCH 25/53] examples: add RunForever experiment to demonstrate graceful termination --- examples/master/repository/run_forever.py | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) create mode 100644 examples/master/repository/run_forever.py diff --git a/examples/master/repository/run_forever.py b/examples/master/repository/run_forever.py new file mode 100644 index 000000000..73f6e2e20 --- /dev/null +++ b/examples/master/repository/run_forever.py @@ -0,0 +1,18 @@ +from itertools import count +from time import sleep + +from artiq import * + + +class RunForever(EnvExperiment): + def build(self): + self.setattr_device("scheduler") + + def run(self): + try: + for i in count(): + self.scheduler.pause() + sleep(1) + print("ping", i) + except TerminationRequested: + print("Terminated gracefully") From 5f89d1a78f71f533ad41093eeb1abeebc27634df Mon Sep 17 00:00:00 2001 From: Sebastien Bourdeauducq Date: Tue, 6 Oct 2015 18:12:57 +0800 Subject: [PATCH 26/53] doc: document core device driver. Closes #119 --- artiq/coredevice/core.py | 19 +++++++++++++++++-- doc/manual/core_drivers_reference.rst | 8 +++++++- 2 files changed, 24 insertions(+), 3 deletions(-) diff --git a/artiq/coredevice/core.py b/artiq/coredevice/core.py index 79ba4d969..e9f6b57e3 100644 --- a/artiq/coredevice/core.py +++ b/artiq/coredevice/core.py @@ -46,10 +46,22 @@ def _no_debug_unparse(label, node): class Core: - def __init__(self, dmgr, ref_period=8*ns, external_clock=False): - self.comm = dmgr.get("comm") + """Core device driver. + + :param ref_period: period of the reference clock for the RTIO subsystem. + On platforms that use clock multiplication and SERDES-based PHYs, + this is the period after multiplication. For example, with a RTIO core + clocked at 125MHz and a SERDES multiplication factor of 8, the + reference period is 1ns. + The time machine unit is equal to this period. + :param external_clock: whether the core device should switch to its + external RTIO clock input instead of using its internal oscillator. + :param comm_device: name of the device used for communications. + """ + def __init__(self, dmgr, ref_period=8*ns, external_clock=False, comm_device="comm"): self.ref_period = ref_period self.external_clock = external_clock + self.comm = dmgr.get(comm_device) self.first_run = True self.core = self @@ -120,10 +132,13 @@ class Core: @kernel def get_rtio_counter_mu(self): + """Return the current value of the hardware RTIO counter.""" return syscall("rtio_get_counter") @kernel def break_realtime(self): + """Set the timeline to the current value of the hardware RTIO counter + plus a margin of 125000 machine units.""" min_now = syscall("rtio_get_counter") + 125000 if now_mu() < min_now: at_mu(min_now) diff --git a/doc/manual/core_drivers_reference.rst b/doc/manual/core_drivers_reference.rst index 56fca095b..667305859 100644 --- a/doc/manual/core_drivers_reference.rst +++ b/doc/manual/core_drivers_reference.rst @@ -1,7 +1,13 @@ Core drivers reference ====================== -These drivers are for peripherals closely integrated into the core device, which do not use the controller mechanism. +These drivers are for the core device and the peripherals closely integrated into it, which do not use the controller mechanism. + +:mod:`artiq.coredevice.core` module +----------------------------------- + +.. automodule:: artiq.coredevice.core + :members: :mod:`artiq.coredevice.ttl` module ----------------------------------- From 566af5499d5d42c9c9a4f8265beefe61db31d746 Mon Sep 17 00:00:00 2001 From: Sebastien Bourdeauducq Date: Fri, 9 Oct 2015 11:46:07 +0800 Subject: [PATCH 27/53] conda: update pyqtgraph, fix version string. closes #132 --- conda/pyqtgraph/meta.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/conda/pyqtgraph/meta.yaml b/conda/pyqtgraph/meta.yaml index 9be6a6dda..f060acf0b 100644 --- a/conda/pyqtgraph/meta.yaml +++ b/conda/pyqtgraph/meta.yaml @@ -1,10 +1,10 @@ package: name: pyqtgraph - version: 0.9.10~a6d5e28 + version: 0.9.10.1036edf source: git_url: https://github.com/pyqtgraph/pyqtgraph.git - git_rev: a6d5e28 + git_rev: 1036edf requirements: build: From 37e873d5aeabd08fb4ee87ecce92db5a39536bb6 Mon Sep 17 00:00:00 2001 From: Sebastien Bourdeauducq Date: Fri, 9 Oct 2015 12:35:10 +0800 Subject: [PATCH 28/53] gitignore: add results created during tests --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 548f72714..ce877f17f 100644 --- a/.gitignore +++ b/.gitignore @@ -11,4 +11,5 @@ doc/manual/_build /dist /*.egg-info /.coverage +artiq/test/results examples/master/results From 6e3c411d1e845fa7f4e09d39ce0824ff419f968d Mon Sep 17 00:00:00 2001 From: whitequark Date: Fri, 9 Oct 2015 14:26:31 +0300 Subject: [PATCH 29/53] conda: add pyelftools recipe. --- conda/pyelftools/bld.bat | 1 + conda/pyelftools/build.sh | 1 + conda/pyelftools/meta.yaml | 24 ++++++++++++++++++++++++ 3 files changed, 26 insertions(+) create mode 100644 conda/pyelftools/bld.bat create mode 100644 conda/pyelftools/build.sh create mode 100644 conda/pyelftools/meta.yaml diff --git a/conda/pyelftools/bld.bat b/conda/pyelftools/bld.bat new file mode 100644 index 000000000..39b5e1fee --- /dev/null +++ b/conda/pyelftools/bld.bat @@ -0,0 +1 @@ +%PYTHON% setup.py install diff --git a/conda/pyelftools/build.sh b/conda/pyelftools/build.sh new file mode 100644 index 000000000..5a5aeeb48 --- /dev/null +++ b/conda/pyelftools/build.sh @@ -0,0 +1 @@ +$PYTHON setup.py install diff --git a/conda/pyelftools/meta.yaml b/conda/pyelftools/meta.yaml new file mode 100644 index 000000000..5b64122a9 --- /dev/null +++ b/conda/pyelftools/meta.yaml @@ -0,0 +1,24 @@ +package: + name: pyelftools + version: 0.23 + +source: + git_url: https://github.com/eliben/pyelftools.git + git_tag: v0.23 + +build: + number: 1 + +requirements: + build: + - python + - setuptools + +test: + imports: + - elftools + +about: + home: https://github.com/eliben/pyelftools.git + license: Public domain + summary: 'Library for analyzing ELF files and DWARF debugging information' From 354c3c8244ef44e1c5a2088facdc3bdb99ee89b4 Mon Sep 17 00:00:00 2001 From: whitequark Date: Fri, 9 Oct 2015 15:01:03 +0300 Subject: [PATCH 30/53] conda: fix pyelftools dependencies. --- conda/pyelftools/meta.yaml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/conda/pyelftools/meta.yaml b/conda/pyelftools/meta.yaml index 5b64122a9..f65b271dd 100644 --- a/conda/pyelftools/meta.yaml +++ b/conda/pyelftools/meta.yaml @@ -7,12 +7,14 @@ source: git_tag: v0.23 build: - number: 1 + number: 0 requirements: build: - python - setuptools + run: + - python test: imports: From 06d5a0f58ddebf2228aa1a827f5b48282f48f50d Mon Sep 17 00:00:00 2001 From: whitequark Date: Fri, 9 Oct 2015 15:05:12 +0300 Subject: [PATCH 31/53] conda: add README. --- conda/README.md | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) create mode 100644 conda/README.md diff --git a/conda/README.md b/conda/README.md new file mode 100644 index 000000000..e37f241c8 --- /dev/null +++ b/conda/README.md @@ -0,0 +1,18 @@ +Uploading conda packages (Python 3.5) +===================================== + +Preparing: + + 1. [Install miniconda][miniconda] + 2. `conda update -q conda` + 3. `conda install conda-build` + 4. `conda create -q -n py35 python=3.5` + 5. `conda config --add channels https://conda.anaconda.org/m-labs/channel/dev` + +Building: + + 1. `source activate py35` + 2. `conda build pkgname --python 3.5`; this command displays a path to the freshly built package + 3. `anaconda upload -c main -c dev` + +[miniconda]: http://conda.pydata.org/docs/install/quick.html#linux-miniconda-install From 87b573c8131d1961d466428a9ad9a5f69d20bf57 Mon Sep 17 00:00:00 2001 From: whitequark Date: Fri, 9 Oct 2015 15:21:50 +0300 Subject: [PATCH 32/53] travis: use main conda m-labs channel. --- .travis/get-anaconda.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis/get-anaconda.sh b/.travis/get-anaconda.sh index 790a723ad..bf76e2fa3 100755 --- a/.travis/get-anaconda.sh +++ b/.travis/get-anaconda.sh @@ -10,4 +10,4 @@ conda update -q conda conda info -a conda install conda-build jinja2 conda create -q -n py35 python=$TRAVIS_PYTHON_VERSION -conda config --add channels https://conda.anaconda.org/m-labs/channel/dev +conda config --add channels https://conda.anaconda.org/m-labs/channel/main From 0cc8f19b6b441b323b8ab7647c53595029c33e1e Mon Sep 17 00:00:00 2001 From: whitequark Date: Fri, 9 Oct 2015 16:14:13 +0300 Subject: [PATCH 33/53] travis: expand matrix to run a build with BUILD_SOC=0. BUILD_SOC=1 build takes over 40 minutes, most of them spent in Xilinx tools, which means all the other changes cannot be tested quickly. --- .travis.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 2271bbd2e..e2e7a4033 100644 --- a/.travis.yml +++ b/.travis.yml @@ -4,8 +4,10 @@ python: sudo: false env: global: - - BUILD_SOC=1 - secure: "DUk/Ihg8KbbzEgPF0qrHqlxU8e8eET9i/BtzNvFddIGX4HP/P2qz0nk3cVkmjuWhqJXSbC22RdKME9qqPzw6fJwJ6dpJ3OR6dDmSd7rewavq+niwxu52PVa+yK8mL4yf1terM7QQ5tIRf+yUL9qGKrZ2xyvEuRit6d4cFep43Ws=" + matrix: + - BUILD_SOC=0 + - BUILD_SOC=1 before_install: - mkdir -p $HOME/.mlabs - if [ $TRAVIS_PULL_REQUEST != false ]; then BUILD_SOC=0; fi From a9230d33f2e9ec627974546fd02754e7b52e977d Mon Sep 17 00:00:00 2001 From: whitequark Date: Fri, 9 Oct 2015 16:27:04 +0300 Subject: [PATCH 34/53] conda: skip bitstream builds if BUILD_SOC=0. --- .travis.yml | 10 +++++++--- .travis/get-xilinx.sh | 2 +- conda/artiq/build.sh | 12 ++++++------ 3 files changed, 14 insertions(+), 10 deletions(-) diff --git a/.travis.yml b/.travis.yml index e2e7a4033..7ed278574 100644 --- a/.travis.yml +++ b/.travis.yml @@ -14,6 +14,7 @@ before_install: - if [ $BUILD_SOC -ne 0 ]; then ./.travis/get-xilinx.sh; fi - . ./.travis/get-toolchain.sh - . ./.travis/get-anaconda.sh + - echo "BUILD_SOC=$BUILD_SOC" >> $HOME/.mlabs/build_settings.sh - source $HOME/miniconda/bin/activate py35 - conda install -q pip coverage anaconda-client migen cython - pip install coveralls @@ -24,9 +25,12 @@ script: - coverage run --source=artiq setup.py test - make -C doc/manual html after_success: - - anaconda -q login --hostname $(hostname) --username $binstar_login --password $binstar_password - - if [ "$TRAVIS_BRANCH" == "master" ]; then anaconda -q upload --user $binstar_login --channel dev --force $HOME/miniconda/conda-bld/linux-64/artiq-*.tar.bz2; fi - - anaconda -q logout + - | + if [ "$TRAVIS_BRANCH" == "master" -a $BUILD_SOC -eq 1 ]; then + anaconda -q login --hostname $(hostname) --username $binstar_login --password $binstar_password + anaconda -q upload --user $binstar_login --channel dev --force $HOME/miniconda/conda-bld/linux-64/artiq-*.tar.bz2 + anaconda -q logout + fi - coveralls notifications: email: diff --git a/.travis/get-xilinx.sh b/.travis/get-xilinx.sh index ccb6a5059..0f37893b3 100755 --- a/.travis/get-xilinx.sh +++ b/.travis/get-xilinx.sh @@ -30,7 +30,7 @@ git clone https://github.com/fallen/impersonate_macaddress make -C impersonate_macaddress # Tell mibuild where Xilinx toolchains are installed # and feed it the mac address corresponding to the license -cat > $HOME/.mlabs/build_settings.sh << EOF +cat >> $HOME/.mlabs/build_settings.sh << EOF MISOC_EXTRA_VIVADO_CMDLINE="-Ob vivado_path $HOME/Xilinx/Vivado" MISOC_EXTRA_ISE_CMDLINE="-Ob ise_path $HOME/opt/Xilinx/" export MACADDR=$macaddress diff --git a/conda/artiq/build.sh b/conda/artiq/build.sh index 959e8a692..4d574de48 100755 --- a/conda/artiq/build.sh +++ b/conda/artiq/build.sh @@ -20,14 +20,14 @@ mkdir -p $BIN_PREFIX/kc705 $BIN_PREFIX/pipistrello cd $SRC_DIR/misoc; $PYTHON make.py -X ../soc -t artiq_kc705 build-headers build-bios; cd - make -C soc/runtime clean runtime.fbi -cd $SRC_DIR/misoc; $PYTHON make.py -X ../soc -t artiq_kc705 $MISOC_EXTRA_VIVADO_CMDLINE build-bitstream; cd - +[ "$BUILD_SOC" != "0" ] && (cd $SRC_DIR/misoc; $PYTHON make.py -X ../soc -t artiq_kc705 $MISOC_EXTRA_VIVADO_CMDLINE build-bitstream) # install KC705 NIST_QC1 binaries mkdir -p $BIN_PREFIX/kc705/nist_qc1 cp soc/runtime/runtime.fbi $BIN_PREFIX/kc705/nist_qc1/ cp $SRC_DIR/misoc/software/bios/bios.bin $BIN_PREFIX/kc705/ -cp $SRC_DIR/misoc/build/artiq_kc705-nist_qc1-kc705.bit $BIN_PREFIX/kc705/ +[ "$BUILD_SOC" != "0" ] && cp $SRC_DIR/misoc/build/artiq_kc705-nist_qc1-kc705.bit $BIN_PREFIX/kc705/ wget http://sionneau.net/artiq/binaries/kc705/flash_proxy/bscan_spi_kc705.bit mv bscan_spi_kc705.bit $BIN_PREFIX/kc705/ @@ -35,13 +35,13 @@ mv bscan_spi_kc705.bit $BIN_PREFIX/kc705/ cd $SRC_DIR/misoc; $PYTHON make.py -X ../soc -t artiq_pipistrello build-headers build-bios; cd - make -C soc/runtime clean runtime.fbi -cd $SRC_DIR/misoc; $PYTHON make.py -X ../soc -t artiq_pipistrello $MISOC_EXTRA_ISE_CMDLINE build-bitstream; cd - +[ "$BUILD_SOC" != "0" ] && (cd $SRC_DIR/misoc; $PYTHON make.py -X ../soc -t artiq_pipistrello $MISOC_EXTRA_ISE_CMDLINE build-bitstream) # install Pipistrello binaries cp soc/runtime/runtime.fbi $BIN_PREFIX/pipistrello/ cp $SRC_DIR/misoc/software/bios/bios.bin $BIN_PREFIX/pipistrello/ -cp $SRC_DIR/misoc/build/artiq_pipistrello-nist_qc1-pipistrello.bit $BIN_PREFIX/pipistrello/ +[ "$BUILD_SOC" != "0" ] && cp $SRC_DIR/misoc/build/artiq_pipistrello-nist_qc1-pipistrello.bit $BIN_PREFIX/pipistrello/ wget https://people.phys.ethz.ch/~robertjo/bscan_spi_lx45_csg324.bit mv bscan_spi_lx45_csg324.bit $BIN_PREFIX/pipistrello/ @@ -49,13 +49,13 @@ mv bscan_spi_lx45_csg324.bit $BIN_PREFIX/pipistrello/ cd $SRC_DIR/misoc; $PYTHON make.py -X ../soc -t artiq_kc705 -s NIST_QC2 build-headers; cd - make -C soc/runtime clean runtime.fbi -cd $SRC_DIR/misoc; $PYTHON make.py -X ../soc -t artiq_kc705 -s NIST_QC2 $MISOC_EXTRA_VIVADO_CMDLINE build-bitstream; cd - +[ "$BUILD_SOC" != "0" ] && (cd $SRC_DIR/misoc; $PYTHON make.py -X ../soc -t artiq_kc705 -s NIST_QC2 $MISOC_EXTRA_VIVADO_CMDLINE build-bitstream) # install KC705 NIST_QC2 binaries mkdir -p $BIN_PREFIX/kc705/nist_qc2 cp soc/runtime/runtime.fbi $BIN_PREFIX/kc705/nist_qc2/ -cp $SRC_DIR/misoc/build/artiq_kc705-nist_qc2-kc705.bit $BIN_PREFIX/kc705/ +[ "$BUILD_SOC" != "0" ] && cp $SRC_DIR/misoc/build/artiq_kc705-nist_qc2-kc705.bit $BIN_PREFIX/kc705/ cp artiq/frontend/artiq_flash.sh $PREFIX/bin From db061bc43dc8047529e69ec4852122f5b039a3cb Mon Sep 17 00:00:00 2001 From: whitequark Date: Fri, 9 Oct 2015 20:17:57 +0300 Subject: [PATCH 35/53] Revert "travis: use main conda m-labs channel." This reverts commit 87b573c8131d1961d466428a9ad9a5f69d20bf57. --- .travis/get-anaconda.sh | 2 +- conda/artiq/meta.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.travis/get-anaconda.sh b/.travis/get-anaconda.sh index bf76e2fa3..790a723ad 100755 --- a/.travis/get-anaconda.sh +++ b/.travis/get-anaconda.sh @@ -10,4 +10,4 @@ conda update -q conda conda info -a conda install conda-build jinja2 conda create -q -n py35 python=$TRAVIS_PYTHON_VERSION -conda config --add channels https://conda.anaconda.org/m-labs/channel/main +conda config --add channels https://conda.anaconda.org/m-labs/channel/dev diff --git a/conda/artiq/meta.yaml b/conda/artiq/meta.yaml index 6dacc0293..e96b8d4bb 100644 --- a/conda/artiq/meta.yaml +++ b/conda/artiq/meta.yaml @@ -31,7 +31,7 @@ requirements: - python >=3.5.0 - setuptools - numpy - - migen + - migen 0.0 - pyelftools - binutils-or1k-linux run: From a05d04b0167a399908959a25beecc0e044507d59 Mon Sep 17 00:00:00 2001 From: whitequark Date: Fri, 9 Oct 2015 22:50:51 +0300 Subject: [PATCH 36/53] conda: add missing recipes. [ci skip] --- conda/chardet/bld.bat | 2 ++ conda/chardet/build.sh | 1 + conda/chardet/meta.yaml | 33 +++++++++++++++++++++++++++++++++ conda/dateutil/bld.bat | 1 + conda/dateutil/build.sh | 1 + conda/dateutil/meta.yaml | 30 ++++++++++++++++++++++++++++++ conda/sphinx-argparse/bld.bat | 1 + conda/sphinx-argparse/build.sh | 1 + conda/sphinx-argparse/meta.yaml | 28 ++++++++++++++++++++++++++++ 9 files changed, 98 insertions(+) create mode 100644 conda/chardet/bld.bat create mode 100644 conda/chardet/build.sh create mode 100644 conda/chardet/meta.yaml create mode 100644 conda/dateutil/bld.bat create mode 100644 conda/dateutil/build.sh create mode 100644 conda/dateutil/meta.yaml create mode 100644 conda/sphinx-argparse/bld.bat create mode 100644 conda/sphinx-argparse/build.sh create mode 100644 conda/sphinx-argparse/meta.yaml diff --git a/conda/chardet/bld.bat b/conda/chardet/bld.bat new file mode 100644 index 000000000..c40a9bbef --- /dev/null +++ b/conda/chardet/bld.bat @@ -0,0 +1,2 @@ +"%PYTHON%" setup.py install +if errorlevel 1 exit 1 diff --git a/conda/chardet/build.sh b/conda/chardet/build.sh new file mode 100644 index 000000000..5a5aeeb48 --- /dev/null +++ b/conda/chardet/build.sh @@ -0,0 +1 @@ +$PYTHON setup.py install diff --git a/conda/chardet/meta.yaml b/conda/chardet/meta.yaml new file mode 100644 index 000000000..e9b7c795c --- /dev/null +++ b/conda/chardet/meta.yaml @@ -0,0 +1,33 @@ +package: + name: chardet + version: 2.2.1 + +source: + fn: chardet-2.2.1.tar.gz + url: https://pypi.python.org/packages/source/c/chardet/chardet-2.2.1.tar.gz + md5: 4a758402eaefd0331bdedc7ecb6f452c + +build: + entry_points: + - chardetect = chardet.chardetect:main + number: 0 + +requirements: + build: + - python + - setuptools + + run: + - python + +test: + # Python imports + imports: + - chardet + + commands: + - chardetect run_test.py + +about: + home: https://github.com/chardet/chardet + license: GNU Library or Lesser General Public License (LGPL) diff --git a/conda/dateutil/bld.bat b/conda/dateutil/bld.bat new file mode 100644 index 000000000..39b5e1fee --- /dev/null +++ b/conda/dateutil/bld.bat @@ -0,0 +1 @@ +%PYTHON% setup.py install diff --git a/conda/dateutil/build.sh b/conda/dateutil/build.sh new file mode 100644 index 000000000..5a5aeeb48 --- /dev/null +++ b/conda/dateutil/build.sh @@ -0,0 +1 @@ +$PYTHON setup.py install diff --git a/conda/dateutil/meta.yaml b/conda/dateutil/meta.yaml new file mode 100644 index 000000000..fd9d40a3e --- /dev/null +++ b/conda/dateutil/meta.yaml @@ -0,0 +1,30 @@ +package: + name: dateutil + version: 2.4.2 + +source: + fn: python-dateutil-2.4.2.tar.gz + url: https://pypi.python.org/packages/source/p/python-dateutil/python-dateutil-2.4.2.tar.gz + md5: 4ef68e1c485b09e9f034e10473e5add2 + +build: + number: 0 + +requirements: + build: + - python + - setuptools + - six >=1.5 + run: + - python + - six >=1.5 + +test: + imports: + - dateutil + - dateutil.zoneinfo + +about: + home: https://dateutil.readthedocs.org + license: BSD + summary: 'Extensions to the standard Python datetime module' diff --git a/conda/sphinx-argparse/bld.bat b/conda/sphinx-argparse/bld.bat new file mode 100644 index 000000000..39b5e1fee --- /dev/null +++ b/conda/sphinx-argparse/bld.bat @@ -0,0 +1 @@ +%PYTHON% setup.py install diff --git a/conda/sphinx-argparse/build.sh b/conda/sphinx-argparse/build.sh new file mode 100644 index 000000000..5a5aeeb48 --- /dev/null +++ b/conda/sphinx-argparse/build.sh @@ -0,0 +1 @@ +$PYTHON setup.py install diff --git a/conda/sphinx-argparse/meta.yaml b/conda/sphinx-argparse/meta.yaml new file mode 100644 index 000000000..6ead92292 --- /dev/null +++ b/conda/sphinx-argparse/meta.yaml @@ -0,0 +1,28 @@ +package: + name: sphinx-argparse + version: 0.1.13 + +source: + fn: sphinx-argparse-0.1.13.tar.gz + url: https://pypi.python.org/packages/source/s/sphinx-argparse/sphinx-argparse-0.1.13.tar.gz + md5: 5ec84e75e1c4b2ae7ca5fb92a6abd738 + +build: + number: 0 + +requirements: + build: + - python + - setuptools + - sphinx + run: + - python + - sphinx + +test: + imports: + - sphinxarg + +about: + license: MIT + summary: 'Sphinx extension that automatically documents argparse commands and options' From 6d0ec2f01d5bbdd55db0158edd1a1c03959e8f76 Mon Sep 17 00:00:00 2001 From: whitequark Date: Sat, 10 Oct 2015 03:50:58 +0300 Subject: [PATCH 37/53] travis: export llvm-or1k lib/ in LD_LIBRARY_PATH. The llvm-or1k build that is currently used is dynamically linked, and in case llvmlite-artiq is linked with this llvm-or1k, this is necessary to run ARTIQ. --- .travis/get-toolchain.sh | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/.travis/get-toolchain.sh b/.travis/get-toolchain.sh index 73c268d0a..9b62ad225 100755 --- a/.travis/get-toolchain.sh +++ b/.travis/get-toolchain.sh @@ -19,11 +19,10 @@ do done export PATH=$PWD/packages/usr/local/llvm-or1k/bin:$PWD/packages/usr/local/bin:$PWD/packages/usr/bin:$PATH -export LD_LIBRARY_PATH=$PWD/packages/usr/lib/x86_64-linux-gnu:$PWD/packages/usr/local/x86_64-unknown-linux-gnu/or1k-elf/lib:$LD_LIBRARY_PATH +export LD_LIBRARY_PATH=$PWD/packages/usr/local/llvm-or1k/lib:$PWD/packages/usr/lib/x86_64-linux-gnu:$PWD/packages/usr/local/x86_64-unknown-linux-gnu/or1k-elf/lib:$LD_LIBRARY_PATH -echo "export LD_LIBRARY_PATH=$PWD/packages/usr/lib/x86_64-linux-gnu:$PWD/packages/usr/local/x86_64-unknown-linux-gnu/or1k-elf/lib:\$LD_LIBRARY_PATH" >> $HOME/.mlabs/build_settings.sh echo "export PATH=$PWD/packages/usr/local/llvm-or1k/bin:$PWD/packages/usr/local/bin:$PWD/packages/usr/bin:\$PATH" >> $HOME/.mlabs/build_settings.sh +echo "export LD_LIBRARY_PATH=$PWD/packages/usr/local/llvm-or1k/lib:$PWD/packages/usr/lib/x86_64-linux-gnu:$PWD/packages/usr/local/x86_64-unknown-linux-gnu/or1k-elf/lib:\$LD_LIBRARY_PATH" >> $HOME/.mlabs/build_settings.sh -or1k-linux-as --version llc --version clang --version From cd3b590962ca0ee07d1201cb736f783e738fb124 Mon Sep 17 00:00:00 2001 From: Sebastien Bourdeauducq Date: Sun, 11 Oct 2015 10:05:27 +0800 Subject: [PATCH 38/53] language/scan: add missing attributes to RandomScan Reported by Joe --- artiq/language/scan.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/artiq/language/scan.py b/artiq/language/scan.py index dabd49af8..a5ffa27d6 100644 --- a/artiq/language/scan.py +++ b/artiq/language/scan.py @@ -79,6 +79,9 @@ class RandomScan(ScanObject): """A scan object that yields a fixed number of randomly ordered evenly spaced values in a range.""" def __init__(self, min, max, npoints, seed=0): + self.min = min + self.max = max + self.npoints = npoints self.sequence = list(LinearScan(min, max, npoints)) shuffle(self.sequence, Random(seed).random) From 7414b90da3775cc7fc8d45ad5872f9f1ceec3202 Mon Sep 17 00:00:00 2001 From: Sebastien Bourdeauducq Date: Mon, 12 Oct 2015 16:06:12 +0800 Subject: [PATCH 39/53] language/scan: add len() support --- artiq/language/scan.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/artiq/language/scan.py b/artiq/language/scan.py index a5ffa27d6..7bc5d5665 100644 --- a/artiq/language/scan.py +++ b/artiq/language/scan.py @@ -47,6 +47,9 @@ class NoScan(ScanObject): def __iter__(self): return self._gen() + def __len__(self): + return 1 + def describe(self): return {"ty": "NoScan", "value": self.value} @@ -70,6 +73,9 @@ class LinearScan(ScanObject): def __iter__(self): return self._gen() + def __len__(self): + return self.npoints + def describe(self): return {"ty": "LinearScan", "min": self.min, "max": self.max, "npoints": self.npoints} @@ -89,6 +95,9 @@ class RandomScan(ScanObject): def __iter__(self): return iter(self.sequence) + def __len__(self): + return self.npoints + def describe(self): return {"ty": "RandomScan", "min": self.min, "max": self.max, "npoints": self.npoints} @@ -103,6 +112,9 @@ class ExplicitScan(ScanObject): def __iter__(self): return iter(self.sequence) + def __len__(self): + return len(self.sequence) + def describe(self): return {"ty": "ExplicitScan", "sequence": self.sequence} From b5cc680fdb6f757e951ae7ad791e8f2c9653833c Mon Sep 17 00:00:00 2001 From: Joe Britton Date: Mon, 12 Oct 2015 16:13:30 +0800 Subject: [PATCH 40/53] devices/novatech409b: improve simultaneous update API --- artiq/devices/novatech409b/driver.py | 31 ++++------------------------ 1 file changed, 4 insertions(+), 27 deletions(-) diff --git a/artiq/devices/novatech409b/driver.py b/artiq/devices/novatech409b/driver.py index ffe971012..e4bea9804 100644 --- a/artiq/devices/novatech409b/driver.py +++ b/artiq/devices/novatech409b/driver.py @@ -137,6 +137,10 @@ class Novatech409B: else: self._ser_send("I a") + def do_simultaneous_update(self): + """Apply update in simultaneous update mode.""" + self._ser_send("I p") + def set_freq(self, ch_no, freq): """Set frequency of one channel.""" self.set_simultaneous_update(False) @@ -154,33 +158,6 @@ class Novatech409B: cmd = "P{:d} {:d}".format(ch_no, phase_word) self._ser_send(cmd) - def set_freq_all_phase_continuous(self, freq): - """Set frequency of all channels simultaneously. - - Set frequency of all channels simultaneously. - 1) all DDSs are set to phase continuous mode - 2) all DDSs are simultaneously set to new frequency - Together 1 and 2 ensure phase continuous frequency switching. - """ - self.set_simultaneous_update(True) - self.set_phase_continuous(True) - for i in range(4): - self.set_freq(i, freq) - # send command necessary to update all channels at the same time - self._ser_send("I p") - - def set_phase_all(self, phase): - """Set phase of all channels simultaneously.""" - - self.set_simultaneous_update(True) - # Note that this only works if the continuous - # phase switching is turned off. - self.set_phase_continuous(False) - for i in range(4): - self.set_phase(i, phase) - # send command necessary to update all channels at the same time - self._ser_send("I p") - def set_gain(self, ch_no, volts): """Set amplitude of one channel.""" From 36c3f022aa621d933fd6cde5238ba3a9f66c7cf8 Mon Sep 17 00:00:00 2001 From: Sebastien Bourdeauducq Date: Mon, 12 Oct 2015 16:15:04 +0800 Subject: [PATCH 41/53] devices/novatech409b: remove unwanted calls to set_simultaneous_update --- artiq/devices/novatech409b/driver.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/artiq/devices/novatech409b/driver.py b/artiq/devices/novatech409b/driver.py index e4bea9804..54f369ddd 100644 --- a/artiq/devices/novatech409b/driver.py +++ b/artiq/devices/novatech409b/driver.py @@ -143,14 +143,11 @@ class Novatech409B: def set_freq(self, ch_no, freq): """Set frequency of one channel.""" - self.set_simultaneous_update(False) # Novatech expects MHz self._ser_send("F{:d} {:f}".format(ch_no, freq/1e6)) def set_phase(self, ch_no, phase): """Set phase of one channel.""" - # do this immediately, disable SimultaneousUpdate mode - self.set_simultaneous_update(False) # phase word is required by device # N is an integer from 0 to 16383. Phase is set to # N*360/16384 deg; in ARTIQ represent phase in cycles [0, 1] @@ -168,7 +165,6 @@ class Novatech409B: s = "Amplitude out of range {v}".format(v=volts) raise ValueError(s) - self.set_simultaneous_update(False) s = "V{:d} {:d}".format(ch_no, dac_value) self._ser_send(s) From 97accd254042933a069000b892c31f7f0e312271 Mon Sep 17 00:00:00 2001 From: Sebastien Bourdeauducq Date: Mon, 12 Oct 2015 17:18:23 +0800 Subject: [PATCH 42/53] merge parameters and results into datasets --- artiq/frontend/artiq_client.py | 50 +++--- artiq/frontend/artiq_gui.py | 43 +++-- artiq/frontend/artiq_influxdb.py | 28 +++- artiq/frontend/artiq_master.py | 40 +++-- artiq/frontend/artiq_run.py | 49 +++--- artiq/gui/console.py | 14 +- artiq/gui/{results.py => datasets.py} | 30 ++-- artiq/language/environment.py | 111 ++++--------- artiq/master/databases.py | 62 +++++++ artiq/master/ddb.py | 24 --- artiq/master/worker_db.py | 154 +++++++++--------- artiq/master/worker_impl.py | 42 +++-- examples/master/dataset_db.pyon | 1 + examples/master/{ddb.pyon => device_db.pyon} | 0 examples/master/pdb.pyon | 1 - .../repository/flopping_f_simulation.py | 26 +-- .../master/repository/photon_histogram.py | 11 +- examples/master/repository/speed_benchmark.py | 12 +- examples/sim/al_spectroscopy.py | 2 +- examples/sim/{pdb.pyon => dataset_db.pyon} | 0 examples/sim/{ddb.pyon => device_db.pyon} | 0 21 files changed, 349 insertions(+), 351 deletions(-) rename artiq/gui/{results.py => datasets.py} (83%) create mode 100644 artiq/master/databases.py delete mode 100644 artiq/master/ddb.py create mode 100644 examples/master/dataset_db.pyon rename examples/master/{ddb.pyon => device_db.pyon} (100%) delete mode 100644 examples/master/pdb.pyon rename examples/sim/{pdb.pyon => dataset_db.pyon} (100%) rename examples/sim/{ddb.pyon => device_db.pyon} (100%) diff --git a/artiq/frontend/artiq_client.py b/artiq/frontend/artiq_client.py index c5e153302..b1bdd2bf3 100755 --- a/artiq/frontend/artiq_client.py +++ b/artiq/frontend/artiq_client.py @@ -63,24 +63,26 @@ def get_argparser(): parser_delete.add_argument("rid", type=int, help="run identifier (RID)") - parser_set_parameter = subparsers.add_parser( - "set-parameter", help="add or modify a parameter") - parser_set_parameter.add_argument("name", help="name of the parameter") - parser_set_parameter.add_argument("value", - help="value in PYON format") + parser_set_dataset = subparsers.add_parser( + "set-dataset", help="add or modify a dataset") + parser_set_dataset.add_argument("name", help="name of the dataset") + parser_set_dataset.add_argument("value", + help="value in PYON format") + parser_set_dataset.add_argument("-p", "--persist", action="store_true", + help="make the dataset persistent") - parser_del_parameter = subparsers.add_parser( - "del-parameter", help="delete a parameter") - parser_del_parameter.add_argument("name", help="name of the parameter") + parser_del_dataset = subparsers.add_parser( + "del-dataset", help="delete a dataset") + parser_del_dataset.add_argument("name", help="name of the dataset") parser_show = subparsers.add_parser( - "show", help="show schedule, log, devices or parameters") + "show", help="show schedule, log, devices or datasets") parser_show.add_argument( "what", - help="select object to show: schedule/log/devices/parameters") + help="select object to show: schedule/log/devices/datasets") subparsers.add_parser( - "scan-ddb", help="trigger a device database (re)scan") + "scan-devices", help="trigger a device database (re)scan") parser_scan_repos = subparsers.add_parser( "scan-repository", help="trigger a repository (re)scan") @@ -129,15 +131,15 @@ def _action_delete(remote, args): remote.delete(args.rid) -def _action_set_parameter(remote, args): - remote.set(args.name, pyon.decode(args.value)) +def _action_set_dataset(remote, args): + remote.set(args.name, pyon.decode(args.value), args.persist) -def _action_del_parameter(remote, args): +def _action_del_dataset(remote, args): remote.delete(args.name) -def _action_scan_ddb(remote, args): +def _action_scan_devices(remote, args): remote.scan() @@ -186,11 +188,11 @@ def _show_devices(devices): print(table) -def _show_parameters(parameters): +def _show_datasets(datasets): clear_screen() - table = PrettyTable(["Parameter", "Value"]) - for k, v in sorted(parameters.items(), key=itemgetter(0)): - table.add_row([k, str(v)]) + table = PrettyTable(["Dataset", "Persistent", "Value"]) + for k, (persist, value) in sorted(datasets.items(), key=itemgetter(0)): + table.add_row([k, "Y" if persist else "N", str(value)]) print(table) @@ -259,8 +261,8 @@ def main(): _show_log(args) elif args.what == "devices": _show_dict(args, "devices", _show_devices) - elif args.what == "parameters": - _show_dict(args, "parameters", _show_parameters) + elif args.what == "datasets": + _show_dict(args, "datasets", _show_datasets) else: print("Unknown object to show, use -h to list valid names.") sys.exit(1) @@ -269,9 +271,9 @@ def main(): target_name = { "submit": "master_schedule", "delete": "master_schedule", - "set_parameter": "master_pdb", - "del_parameter": "master_pdb", - "scan_ddb": "master_ddb", + "set_dataset": "master_dataset_db", + "del_dataset": "master_dataset_db", + "scan_devices": "master_device_db", "scan_repository": "master_repository" }[action] remote = Client(args.server, port, target_name) diff --git a/artiq/frontend/artiq_gui.py b/artiq/frontend/artiq_gui.py index f6f9db57c..10db8eb41 100755 --- a/artiq/frontend/artiq_gui.py +++ b/artiq/frontend/artiq_gui.py @@ -15,8 +15,7 @@ from artiq.protocols.pc_rpc import AsyncioClient from artiq.gui.state import StateManager from artiq.gui.explorer import ExplorerDock from artiq.gui.moninj import MonInj -from artiq.gui.results import ResultsDock -from artiq.gui.parameters import ParametersDock +from artiq.gui.datasets import DatasetsDock from artiq.gui.schedule import ScheduleDock from artiq.gui.log import LogDock from artiq.gui.console import ConsoleDock @@ -92,30 +91,24 @@ def main(): args.server, args.port_notify)) atexit.register(lambda: loop.run_until_complete(d_explorer.sub_close())) - d_results = ResultsDock(win, area) - smgr.register(d_results) - loop.run_until_complete(d_results.sub_connect( + d_datasets = DatasetsDock(win, area) + smgr.register(d_datasets) + loop.run_until_complete(d_datasets.sub_connect( args.server, args.port_notify)) - atexit.register(lambda: loop.run_until_complete(d_results.sub_close())) + atexit.register(lambda: loop.run_until_complete(d_datasets.sub_close())) if os.name != "nt": d_ttl_dds = MonInj() loop.run_until_complete(d_ttl_dds.start(args.server, args.port_notify)) atexit.register(lambda: loop.run_until_complete(d_ttl_dds.stop())) - d_params = ParametersDock() - loop.run_until_complete(d_params.sub_connect( - args.server, args.port_notify)) - atexit.register(lambda: loop.run_until_complete(d_params.sub_close())) - if os.name != "nt": area.addDock(d_ttl_dds.dds_dock, "top") area.addDock(d_ttl_dds.ttl_dock, "above", d_ttl_dds.dds_dock) - area.addDock(d_results, "above", d_ttl_dds.ttl_dock) + area.addDock(d_datasets, "above", d_ttl_dds.ttl_dock) else: - area.addDock(d_results, "top") - area.addDock(d_params, "above", d_results) - area.addDock(d_explorer, "above", d_params) + area.addDock(d_datasets, "top") + area.addDock(d_explorer, "above", d_datasets) d_schedule = ScheduleDock(status_bar, schedule_ctl) loop.run_until_complete(d_schedule.sub_connect( @@ -127,16 +120,18 @@ def main(): args.server, args.port_notify)) atexit.register(lambda: loop.run_until_complete(d_log.sub_close())) - pdb = AsyncioClient() - loop.run_until_complete(pdb.connect_rpc( - args.server, args.port_control, "master_pdb")) - atexit.register(lambda: pdb.close_rpc()) - def _get_parameter(k, v): - asyncio.ensure_future(pdb.set(k, v)) + dataset_db = AsyncioClient() + loop.run_until_complete(dataset_db.connect_rpc( + args.server, args.port_control, "master_dataset_db")) + atexit.register(lambda: dataset_db.close_rpc()) + def _set_dataset(k, v): + asyncio.ensure_future(dataset_db.set(k, v)) + def _del_dataset(k): + asyncio.ensure_future(dataset_db.delete(k)) d_console = ConsoleDock( - d_params.get_parameter, - _get_parameter, - d_results.get_result) + d_datasets.get_dataset, + _set_dataset, + _del_dataset) area.addDock(d_console, "bottom") area.addDock(d_log, "above", d_console) diff --git a/artiq/frontend/artiq_influxdb.py b/artiq/frontend/artiq_influxdb.py index f69da9b0e..36e7be2c3 100755 --- a/artiq/frontend/artiq_influxdb.py +++ b/artiq/frontend/artiq_influxdb.py @@ -93,7 +93,7 @@ class DBWriter(TaskObject): try: self._queue.put_nowait((k, v)) except asyncio.QueueFull: - logger.warning("failed to update parameter '%s': " + logger.warning("failed to update dataset '%s': " "too many pending updates", k) async def _do(self): @@ -103,7 +103,7 @@ class DBWriter(TaskObject): params = {"u": self.user, "p": self.password, "db": self.database, "consistency": "any", "precision": "n"} fmt_ty, fmt_v = format_influxdb(v) - data = "{},parameter={} {}={}".format(self.table, k, fmt_ty, fmt_v) + data = "{},dataset={} {}={}".format(self.table, k, fmt_ty, fmt_v) try: response = await aiohttp.request( "POST", url, params=params, data=data) @@ -121,15 +121,31 @@ class DBWriter(TaskObject): response.close() -class Parameters: +class _Mock: + def __setitem__(self, k, v): + pass + + def __getitem__(self, k): + return self + + def __delitem__(self, k): + pass + + +class Datasets: def __init__(self, filter_function, writer, init): self.filter_function = filter_function self.writer = writer def __setitem__(self, k, v): if self.filter_function(k): - self.writer.update(k, v) + self.writer.update(k, v[1]) + # ignore mutations + def __getitem__(self, k): + return _Mock() + + # ignore deletions def __delitem__(self, k): pass @@ -145,8 +161,8 @@ class MasterReader(TaskObject): async def _do(self): subscriber = Subscriber( - "parameters", - partial(Parameters, self.filter_function, self.writer)) + "datasets", + partial(Datasets, self.filter_function, self.writer)) while True: try: await subscriber.connect(self.server, self.port) diff --git a/artiq/frontend/artiq_master.py b/artiq/frontend/artiq_master.py index e0f3b740d..264658ef9 100755 --- a/artiq/frontend/artiq_master.py +++ b/artiq/frontend/artiq_master.py @@ -6,9 +6,8 @@ import atexit import os from artiq.protocols.pc_rpc import Server -from artiq.protocols.sync_struct import Notifier, Publisher, process_mod -from artiq.master.ddb import DDB -from artiq.protocols.file_db import FlatFileDB +from artiq.protocols.sync_struct import Notifier, Publisher +from artiq.master.databases import DeviceDB, DatasetDB from artiq.master.scheduler import Scheduler from artiq.master.worker_db import get_last_rid from artiq.master.repository import FilesystemBackend, GitBackend, Repository @@ -28,10 +27,10 @@ def get_argparser(): "--port-control", default=3251, type=int, help="TCP port to listen to for control (default: %(default)d)") group = parser.add_argument_group("databases") - group.add_argument("-d", "--ddb", default="ddb.pyon", - help="device database file") - group.add_argument("-p", "--pdb", default="pdb.pyon", - help="parameter database file") + group.add_argument("--device-db", default="device_db.pyon", + help="device database file (default: '%(default)s')") + group.add_argument("--dataset-db", default="dataset_db.pyon", + help="dataset file (default: '%(default)s')") group = parser.add_argument_group("repository") group.add_argument( "-g", "--git", default=False, action="store_true", @@ -65,25 +64,25 @@ def main(): loop = asyncio.get_event_loop() atexit.register(lambda: loop.close()) - ddb = DDB(args.ddb) - pdb = FlatFileDB(args.pdb) - rtr = Notifier(dict()) + device_db = DeviceDB(args.device_db) + dataset_db = DatasetDB(args.dataset_db) + dataset_db.start() + atexit.register(lambda: loop.run_until_complete(dataset_db.stop())) log = Log(1000) if args.git: repo_backend = GitBackend(args.repository) else: repo_backend = FilesystemBackend(args.repository) - repository = Repository(repo_backend, ddb.get_ddb, log.log) + repository = Repository(repo_backend, device_db.get_ddb, log.log) atexit.register(repository.close) repository.scan_async() worker_handlers = { - "get_ddb": ddb.get_ddb, - "get_device": ddb.get, - "get_parameter": pdb.get, - "set_parameter": pdb.set, - "update_rt_results": lambda mod: process_mod(rtr, mod), + "get_ddb": device_db.get_ddb, + "get_device": device_db.get, + "get_dataset": dataset_db.get, + "update_dataset": dataset_db.update, "log": log.log } scheduler = Scheduler(get_last_rid() + 1, worker_handlers, repo_backend) @@ -92,8 +91,8 @@ def main(): atexit.register(lambda: loop.run_until_complete(scheduler.stop())) server_control = Server({ - "master_ddb": ddb, - "master_pdb": pdb, + "master_device_db": device_db, + "master_dataset_db": dataset_db, "master_schedule": scheduler, "master_repository": repository }) @@ -103,9 +102,8 @@ def main(): server_notify = Publisher({ "schedule": scheduler.notifier, - "devices": ddb.data, - "parameters": pdb.data, - "rt_results": rtr, + "devices": device_db.data, + "datasets": dataset_db.data, "explist": repository.explist, "log": log.data }) diff --git a/artiq/frontend/artiq_run.py b/artiq/frontend/artiq_run.py index 3474f4740..b4477077d 100755 --- a/artiq/frontend/artiq_run.py +++ b/artiq/frontend/artiq_run.py @@ -4,17 +4,14 @@ import argparse import sys -import time from operator import itemgetter -from itertools import chain import logging import h5py from artiq.language.environment import EnvExperiment -from artiq.protocols.file_db import FlatFileDB -from artiq.master.ddb import DDB -from artiq.master.worker_db import DeviceManager, ResultDB +from artiq.master.databases import DeviceDB, DatasetDB +from artiq.master.worker_db import DeviceManager, DatasetManager from artiq.tools import * @@ -33,11 +30,6 @@ class ELFRunner(EnvExperiment): self.core.comm.serve(dict(), dict()) -class SimpleParamLogger: - def set(self, timestamp, name, value): - logger.info("Parameter change: {} = {}".format(name, value)) - - class DummyScheduler: def __init__(self): self.next_rid = 0 @@ -63,10 +55,10 @@ def get_argparser(with_file=True): description="Local experiment running tool") verbosity_args(parser) - parser.add_argument("-d", "--ddb", default="ddb.pyon", - help="device database file") - parser.add_argument("-p", "--pdb", default="pdb.pyon", - help="parameter database file") + parser.add_argument("--device-db", default="device_db.pyon", + help="device database file (default: '%(default)s')") + parser.add_argument("--dataset-db", default="dataset_db.pyon", + help="dataset file (default: '%(default)s')") parser.add_argument("-e", "--experiment", default=None, help="experiment to run") @@ -82,7 +74,7 @@ def get_argparser(with_file=True): return parser -def _build_experiment(dmgr, pdb, rdb, args): +def _build_experiment(device_mgr, dataset_mgr, args): if hasattr(args, "file"): if args.file.endswith(".elf"): if args.arguments: @@ -90,7 +82,7 @@ def _build_experiment(dmgr, pdb, rdb, args): if args.experiment: raise ValueError("experiment-by-name not supported " "for ELF kernels") - return ELFRunner(dmgr, pdb, rdb, file=args.file) + return ELFRunner(device_mgr, dataset_mgr, file=args.file) else: module = file_import(args.file) file = args.file @@ -104,35 +96,34 @@ def _build_experiment(dmgr, pdb, rdb, args): "experiment": args.experiment, "arguments": arguments } - dmgr.virtual_devices["scheduler"].expid = expid - return exp(dmgr, pdb, rdb, **arguments) + device_mgr.virtual_devices["scheduler"].expid = expid + return exp(device_mgr, dataset_mgr, **arguments) def run(with_file=False): args = get_argparser(with_file).parse_args() init_logger(args) - dmgr = DeviceManager(DDB(args.ddb), - virtual_devices={"scheduler": DummyScheduler()}) - pdb = FlatFileDB(args.pdb) - pdb.hooks.append(SimpleParamLogger()) - rdb = ResultDB() + device_mgr = DeviceManager(DeviceDB(args.device_db), + virtual_devices={"scheduler": DummyScheduler()}) + dataset_db = DatasetDB(args.dataset_db) + dataset_mgr = DatasetManager(dataset_db) try: - exp_inst = _build_experiment(dmgr, pdb, rdb, args) + exp_inst = _build_experiment(device_mgr, dataset_mgr, args) exp_inst.prepare() exp_inst.run() exp_inst.analyze() finally: - dmgr.close_devices() + device_mgr.close_devices() if args.hdf5 is not None: with h5py.File(args.hdf5, "w") as f: - rdb.write_hdf5(f) - elif rdb.rt.read or rdb.nrt: - r = chain(rdb.rt.read.items(), rdb.nrt.items()) - for k, v in sorted(r, key=itemgetter(0)): + dataset_mgr.write_hdf5(f) + else: + for k, v in sorted(dataset_mgr.local.items(), key=itemgetter(0)): print("{}: {}".format(k, v)) + dataset_db.save() def main(): diff --git a/artiq/gui/console.py b/artiq/gui/console.py index 38015ef29..b7b98e573 100644 --- a/artiq/gui/console.py +++ b/artiq/gui/console.py @@ -5,19 +5,19 @@ _help = """ This is an interactive Python console. The following functions are available: - get_parameter(key) - set_parameter(key, value) [asynchronous update] - get_result(key) [real-time results only] + get_dataset(key) + set_dataset(key, value, persist=False) [asynchronous update] + del_dataset(key) [asynchronous update] """ class ConsoleDock(dockarea.Dock): - def __init__(self, get_parameter, set_parameter, get_result): + def __init__(self, get_dataset, set_dataset, del_dataset): dockarea.Dock.__init__(self, "Console", size=(1000, 300)) ns = { - "get_parameter": get_parameter, - "set_parameter": set_parameter, - "get_result": get_result + "get_dataset": get_dataset, + "set_dataset": set_dataset, + "del_dataset": del_dataset } c = console.ConsoleWidget(namespace=ns, text=_help) self.addWidget(c) diff --git a/artiq/gui/results.py b/artiq/gui/datasets.py similarity index 83% rename from artiq/gui/results.py rename to artiq/gui/datasets.py index edd40e2ad..8d50f761d 100644 --- a/artiq/gui/results.py +++ b/artiq/gui/datasets.py @@ -15,9 +15,9 @@ from artiq.gui.displays import * logger = logging.getLogger(__name__) -class ResultsModel(DictSyncModel): +class DatasetsModel(DictSyncModel): def __init__(self, parent, init): - DictSyncModel.__init__(self, ["Result", "Value"], + DictSyncModel.__init__(self, ["Dataset", "Persistent", "Value"], parent, init) def sort_key(self, k, v): @@ -27,7 +27,9 @@ class ResultsModel(DictSyncModel): if column == 0: return k elif column == 1: - return short_format(v) + return "Y" if v[0] else "N" + elif column == 2: + return short_format(v[1]) else: raise ValueError @@ -38,9 +40,9 @@ def _get_display_type_name(display_cls): return name -class ResultsDock(dockarea.Dock): +class DatasetsDock(dockarea.Dock): def __init__(self, dialog_parent, dock_area): - dockarea.Dock.__init__(self, "Results", size=(1500, 500)) + dockarea.Dock.__init__(self, "Datasets", size=(1500, 500)) self.dialog_parent = dialog_parent self.dock_area = dock_area @@ -65,22 +67,26 @@ class ResultsDock(dockarea.Dock): self.displays = dict() - def get_result(self, key): - return self.table_model.backing_store[key] + def get_dataset(self, key): + return self.table_model.backing_store[key][1] async def sub_connect(self, host, port): - self.subscriber = Subscriber("rt_results", self.init_results_model, + self.subscriber = Subscriber("datasets", self.init_datasets_model, self.on_mod) await self.subscriber.connect(host, port) async def sub_close(self): await self.subscriber.close() - def init_results_model(self, init): - self.table_model = ResultsModel(self.table, init) + def init_datasets_model(self, init): + self.table_model = DatasetsModel(self.table, init) self.table.setModel(self.table_model) return self.table_model + def update_display_data(self, dsp): + dsp.update_data({k: self.table_model.backing_store[k][1] + for k in dsp.data_sources()}) + def on_mod(self, mod): if mod["action"] == "init": for display in self.displays.values(): @@ -96,7 +102,7 @@ class ResultsDock(dockarea.Dock): for display in self.displays.values(): if source in display.data_sources(): - display.update_data(self.table_model.backing_store) + self.update_display_data(display) def create_dialog(self, ty): dlg_class = display_types[ty][0] @@ -111,7 +117,7 @@ class ResultsDock(dockarea.Dock): dsp_class = display_types[ty][1] dsp = dsp_class(name, settings) self.displays[name] = dsp - dsp.update_data(self.table_model.backing_store) + self.update_display_data(dsp) def on_close(): del self.displays[name] diff --git a/artiq/language/environment.py b/artiq/language/environment.py index b2b1c2a1d..4801720f7 100644 --- a/artiq/language/environment.py +++ b/artiq/language/environment.py @@ -113,15 +113,13 @@ class StringValue(_SimpleArgProcessor): class HasEnvironment: """Provides methods to manage the environment of an experiment (devices, parameters, results, arguments).""" - def __init__(self, dmgr=None, pdb=None, rdb=None, *, parent=None, - param_override=dict(), default_arg_none=False, **kwargs): + def __init__(self, device_mgr=None, dataset_mgr=None, *, parent=None, + default_arg_none=False, **kwargs): self.requested_args = OrderedDict() - self.__dmgr = dmgr - self.__pdb = pdb - self.__rdb = rdb + self.__device_mgr = device_mgr + self.__dataset_mgr = dataset_mgr self.__parent = parent - self.__param_override = param_override self.__default_arg_none = default_arg_none self.__kwargs = kwargs @@ -143,17 +141,16 @@ class HasEnvironment: are set to ``None``.""" raise NotImplementedError - def dbs(self): - """Returns the device manager, the parameter database and the result - database, in this order. + def managers(self): + """Returns the device manager and the dataset manager, in this order. This is the same order that the constructor takes them, allowing sub-objects to be created with this idiom to pass the environment around: :: - sub_object = SomeLibrary(*self.dbs()) + sub_object = SomeLibrary(*self.managers()) """ - return self.__dmgr, self.__pdb, self.__rdb + return self.__device_mgr, self.__dataset_mgr def get_argument(self, key, processor=None, group=None): """Retrieves and returns the value of an argument. @@ -193,91 +190,45 @@ class HasEnvironment: """Returns the full contents of the device database.""" if self.__parent is not None: return self.__parent.get_ddb() - return self.__dmgr.get_ddb() + return self.__device_mgr.get_ddb() def get_device(self, key): """Creates and returns a device driver.""" if self.__parent is not None: return self.__parent.get_device(key) - if self.__dmgr is None: + if self.__device_mgr is None: raise ValueError("Device manager not present") - return self.__dmgr.get(key) + return self.__device_mgr.get(key) def setattr_device(self, key): """Sets a device driver as attribute. The names of the device driver and of the attribute are the same.""" setattr(self, key, self.get_device(key)) - def get_parameter(self, key, default=NoDefault): - """Retrieves and returns a parameter.""" - if self.__parent is not None and key not in self.__param_override: - return self.__parent.get_parameter(key, default) - if self.__pdb is None: - raise ValueError("Parameter database not present") - if key in self.__param_override: - return self.__param_override[key] + def set_dataset(self, key, value, + broadcast=False, persist=False, save=True): + if self.__parent is not None: + self.__parent.set_dataset(key, value, broadcast, persist, save) + return + if self.__dataset_mgr is None: + raise ValueError("Dataset manager not present") + return self.__dataset_mgr.set(key, value, broadcast, persist, save) + + def get_dataset(self, key, default=NoDefault): + if self.__parent is not None: + return self.__parent.get_dataset(key, default) + if self.__dataset_mgr is None: + raise ValueError("Dataset manager not present") try: - return self.__pdb.get(key) + return self.__dataset_mgr.get(key) except KeyError: - if default is not NoDefault: - return default - else: + if default is NoDefault: raise + else: + return default - def setattr_parameter(self, key, default=NoDefault): - """Sets a parameter as attribute. The names of the argument and of the - parameter are the same.""" - setattr(self, key, self.get_parameter(key, default)) - - def set_parameter(self, key, value): - """Writes the value of a parameter into the parameter database.""" - if self.__parent is not None: - self.__parent.set_parameter(key, value) - return - if self.__pdb is None: - raise ValueError("Parameter database not present") - self.__pdb.set(key, value) - - def set_result(self, key, value, realtime=False, store=True): - """Writes the value of a result. - - :param realtime: Marks the result as real-time, making it immediately - available to clients such as the user interface. Returns a - ``Notifier`` instance that can be used to modify mutable results - (such as lists) and synchronize the modifications with the clients. - :param store: Defines if the result should be stored permanently, - e.g. in HDF5 output. Default is to store. - """ - if self.__parent is not None: - self.__parent.set_result(key, value, realtime, store) - return - if self.__rdb is None: - raise ValueError("Result database not present") - if realtime: - if key in self.__rdb.nrt: - raise ValueError("Result is already non-realtime") - self.__rdb.rt[key] = value - notifier = self.__rdb.rt[key] - notifier.kernel_attr_init = False - self.__rdb.set_store(key, store) - return notifier - else: - if key in self.__rdb.rt.read: - raise ValueError("Result is already realtime") - self.__rdb.nrt[key] = value - self.__rdb.set_store(key, store) - - def get_result(self, key): - """Retrieves the value of a result. - - There is no difference between real-time and non-real-time results - (this function does not return ``Notifier`` instances). - """ - if self.__parent is not None: - return self.__parent.get_result(key) - if self.__rdb is None: - raise ValueError("Result database not present") - return self.__rdb.get(key) + def setattr_dataset(self, key, default=NoDefault): + setattr(self, key, self.get_dataset(key, default)) class Experiment: diff --git a/artiq/master/databases.py b/artiq/master/databases.py new file mode 100644 index 000000000..acbf1a37a --- /dev/null +++ b/artiq/master/databases.py @@ -0,0 +1,62 @@ +import asyncio + +from artiq.protocols.sync_struct import Notifier, process_mod +from artiq.protocols import pyon +from artiq.tools import TaskObject + + +class DeviceDB: + def __init__(self, backing_file): + self.backing_file = backing_file + self.data = Notifier(pyon.load_file(self.backing_file)) + + def scan(self): + new_data = pyon.load_file(self.backing_file) + + for k in list(self.data.read.keys()): + if k not in new_data: + del self.data[k] + for k in new_data.keys(): + if k not in self.data.read or self.data.read[k] != new_data[k]: + self.data[k] = new_data[k] + + def get_ddb(self): + return self.data.read + + def get(self, key): + return self.data.read[key] + + +class DatasetDB(TaskObject): + def __init__(self, persist_file, autosave_period=30): + self.persist_file = persist_file + self.autosave_period = autosave_period + + file_data = pyon.load_file(self.persist_file) + self.data = Notifier({k: (True, v) for k, v in file_data.items()}) + + def save(self): + data = {k: v[1] for k, v in self.data.read.items() if v[0]} + pyon.store_file(self.persist_file, data) + + async def _do(self): + try: + while True: + await asyncio.sleep(self.autosave_period) + self.save() + finally: + self.save() + + def get(self, key): + return self.data.read[key][1] + + def update(self, mod): + process_mod(self.data, mod) + + # convenience functions (update() can be used instead) + def set(self, key, value, persist=False): + self.data[key] = (persist, value) + + def delete(self, key): + del self.data[key] + # diff --git a/artiq/master/ddb.py b/artiq/master/ddb.py deleted file mode 100644 index b798422c8..000000000 --- a/artiq/master/ddb.py +++ /dev/null @@ -1,24 +0,0 @@ -from artiq.protocols.sync_struct import Notifier -from artiq.protocols import pyon - - -class DDB: - def __init__(self, backing_file): - self.backing_file = backing_file - self.data = Notifier(pyon.load_file(self.backing_file)) - - def scan(self): - new_data = pyon.load_file(self.backing_file) - - for k in list(self.data.read.keys()): - if k not in new_data: - del self.data[k] - for k in new_data.keys(): - if k not in self.data.read or self.data.read[k] != new_data[k]: - self.data[k] = new_data[k] - - def get_ddb(self): - return self.data.read - - def get(self, key): - return self.data.read[key] diff --git a/artiq/master/worker_db.py b/artiq/master/worker_db.py index fce88f3a6..f36371a1f 100644 --- a/artiq/master/worker_db.py +++ b/artiq/master/worker_db.py @@ -15,6 +15,64 @@ from artiq.protocols.pc_rpc import Client, BestEffortClient logger = logging.getLogger(__name__) +def _create_device(desc, device_mgr): + ty = desc["type"] + if ty == "local": + module = importlib.import_module(desc["module"]) + device_class = getattr(module, desc["class"]) + return device_class(device_mgr, **desc["arguments"]) + elif ty == "controller": + if desc["best_effort"]: + cl = BestEffortClient + else: + cl = Client + return cl(desc["host"], desc["port"], desc["target_name"]) + else: + raise ValueError("Unsupported type in device DB: " + ty) + + +class DeviceManager: + """Handles creation and destruction of local device drivers and controller + RPC clients.""" + def __init__(self, ddb, virtual_devices=dict()): + self.ddb = ddb + self.virtual_devices = virtual_devices + self.active_devices = OrderedDict() + + def get_ddb(self): + """Returns the full contents of the device database.""" + return self.ddb.get_ddb() + + def get(self, name): + """Get the device driver or controller client corresponding to a + device database entry.""" + if name in self.virtual_devices: + return self.virtual_devices[name] + if name in self.active_devices: + return self.active_devices[name] + else: + desc = self.ddb.get(name) + while isinstance(desc, str): + # alias + desc = self.ddb.get(desc) + dev = _create_device(desc, self) + self.active_devices[name] = dev + return dev + + def close_devices(self): + """Closes all active devices, in the opposite order as they were + requested.""" + for dev in reversed(list(self.active_devices.values())): + try: + if isinstance(dev, (Client, BestEffortClient)): + dev.close_rpc() + elif hasattr(dev, "close"): + dev.close() + except Exception as e: + logger.warning("Exception %r when closing device %r", e, dev) + self.active_devices.clear() + + def get_hdf5_output(start_time, rid, name): dirname = os.path.join("results", time.strftime("%Y-%m-%d", start_time), @@ -87,84 +145,30 @@ def result_dict_to_hdf5(f, rd): dataset[()] = data -class ResultDB: - def __init__(self): - self.rt = Notifier(dict()) - self.nrt = dict() - self.store = set() +class DatasetManager: + def __init__(self, ddb): + self.broadcast = Notifier(dict()) + self.local = dict() + + self.ddb = ddb + self.broadcast.publish = ddb.update + + def set(self, key, value, broadcast=False, persist=False, save=True): + if persist: + broadcast = True + r = None + if broadcast: + self.broadcast[key] = (persist, value) + r = self.broadcast[key][1] + if save: + self.local[key] = value + return r def get(self, key): try: - return self.nrt[key] + return self.local[key] except KeyError: - return self.rt[key].read - - def set_store(self, key, store): - if store: - self.store.add(key) - else: - self.store.discard(key) + return self.ddb.get(key) def write_hdf5(self, f): - result_dict_to_hdf5( - f, {k: v for k, v in self.rt.read.items() if k in self.store}) - result_dict_to_hdf5( - f, {k: v for k, v in self.nrt.items() if k in self.store}) - - -def _create_device(desc, dmgr): - ty = desc["type"] - if ty == "local": - module = importlib.import_module(desc["module"]) - device_class = getattr(module, desc["class"]) - return device_class(dmgr, **desc["arguments"]) - elif ty == "controller": - if desc["best_effort"]: - cl = BestEffortClient - else: - cl = Client - return cl(desc["host"], desc["port"], desc["target_name"]) - else: - raise ValueError("Unsupported type in device DB: " + ty) - - -class DeviceManager: - """Handles creation and destruction of local device drivers and controller - RPC clients.""" - def __init__(self, ddb, virtual_devices=dict()): - self.ddb = ddb - self.virtual_devices = virtual_devices - self.active_devices = OrderedDict() - - def get_ddb(self): - """Returns the full contents of the device database.""" - return self.ddb.get_ddb() - - def get(self, name): - """Get the device driver or controller client corresponding to a - device database entry.""" - if name in self.virtual_devices: - return self.virtual_devices[name] - if name in self.active_devices: - return self.active_devices[name] - else: - desc = self.ddb.get(name) - while isinstance(desc, str): - # alias - desc = self.ddb.get(desc) - dev = _create_device(desc, self) - self.active_devices[name] = dev - return dev - - def close_devices(self): - """Closes all active devices, in the opposite order as they were - requested.""" - for dev in reversed(list(self.active_devices.values())): - try: - if isinstance(dev, (Client, BestEffortClient)): - dev.close_rpc() - elif hasattr(dev, "close"): - dev.close() - except Exception as e: - logger.warning("Exception %r when closing device %r", e, dev) - self.active_devices.clear() + result_dict_to_hdf5(f, self.local) diff --git a/artiq/master/worker_impl.py b/artiq/master/worker_impl.py index db1229d2e..ff9a58a4c 100644 --- a/artiq/master/worker_impl.py +++ b/artiq/master/worker_impl.py @@ -4,7 +4,7 @@ import os from artiq.protocols import pyon from artiq.tools import file_import -from artiq.master.worker_db import DeviceManager, ResultDB, get_hdf5_output +from artiq.master.worker_db import DeviceManager, DatasetManager, get_hdf5_output from artiq.language.environment import is_experiment from artiq.language.core import set_watchdog_factory, TerminationRequested @@ -62,17 +62,14 @@ class LogForwarder: pass -class ParentDDB: +class ParentDeviceDB: get_ddb = make_parent_action("get_ddb", "") get = make_parent_action("get_device", "key", KeyError) -class ParentPDB: - get = make_parent_action("get_parameter", "key", KeyError) - set = make_parent_action("set_parameter", "key value") - - -update_rt_results = make_parent_action("update_rt_results", "mod") +class ParentDatasetDB: + get = make_parent_action("get_dataset", "key", KeyError) + update = make_parent_action("update_dataset", "mod") class Watchdog: @@ -126,22 +123,22 @@ register_experiment = make_parent_action("register_experiment", "class_name name arguments") -class ExamineDMGR: +class ExamineDeviceMgr: get_ddb = make_parent_action("get_ddb", "") def get(self, name): return None -class DummyPDB: - def get(self, name): +class DummyDatasetMgr: + def set(self, key, value, broadcast=False, persist=False, save=True): return None - def set(self, name, value): + def get(self, key): pass -def examine(dmgr, pdb, rdb, file): +def examine(device_mgr, dataset_mgr, file): module = file_import(file) for class_name, exp_class in module.__dict__.items(): if class_name[0] == "_": @@ -153,7 +150,7 @@ def examine(dmgr, pdb, rdb, file): name = exp_class.__doc__.splitlines()[0].strip() if name[-1] == ".": name = name[:-1] - exp_inst = exp_class(dmgr, pdb, rdb, default_arg_none=True) + exp_inst = exp_class(device_mgr, dataset_mgr, default_arg_none=True) arguments = [(k, (proc.describe(), group)) for k, (proc, group) in exp_inst.requested_args.items()] register_experiment(class_name, name, arguments) @@ -168,10 +165,9 @@ def main(): exp = None exp_inst = None - dmgr = DeviceManager(ParentDDB, - virtual_devices={"scheduler": Scheduler()}) - rdb = ResultDB() - rdb.rt.publish = update_rt_results + device_mgr = DeviceManager(ParentDeviceDB, + virtual_devices={"scheduler": Scheduler()}) + dataset_mgr = DatasetManager(ParentDatasetDB) try: while True: @@ -187,9 +183,9 @@ def main(): else: expf = expid["file"] exp = get_exp(expf, expid["class_name"]) - dmgr.virtual_devices["scheduler"].set_run_info( + device_mgr.virtual_devices["scheduler"].set_run_info( obj["pipeline_name"], expid, obj["priority"]) - exp_inst = exp(dmgr, ParentPDB, rdb, + exp_inst = exp(device_mgr, dataset_mgr, **expid["arguments"]) put_object({"action": "completed"}) elif action == "prepare": @@ -204,7 +200,7 @@ def main(): elif action == "write_results": f = get_hdf5_output(start_time, rid, exp.__name__) try: - rdb.write_hdf5(f) + dataset_mgr.write_hdf5(f) if "repo_rev" in expid: rr = expid["repo_rev"] dtype = "S{}".format(len(rr)) @@ -214,12 +210,12 @@ def main(): f.close() put_object({"action": "completed"}) elif action == "examine": - examine(ExamineDMGR(), DummyPDB(), ResultDB(), obj["file"]) + examine(ExamineDeviceMgr(), DummyDatasetMgr(), obj["file"]) put_object({"action": "completed"}) elif action == "terminate": break finally: - dmgr.close_devices() + device_mgr.close_devices() if __name__ == "__main__": main() diff --git a/examples/master/dataset_db.pyon b/examples/master/dataset_db.pyon new file mode 100644 index 000000000..00f15d739 --- /dev/null +++ b/examples/master/dataset_db.pyon @@ -0,0 +1 @@ +{"flopping_freq": 1499.9876804260716} diff --git a/examples/master/ddb.pyon b/examples/master/device_db.pyon similarity index 100% rename from examples/master/ddb.pyon rename to examples/master/device_db.pyon diff --git a/examples/master/pdb.pyon b/examples/master/pdb.pyon deleted file mode 100644 index 27f1e7665..000000000 --- a/examples/master/pdb.pyon +++ /dev/null @@ -1 +0,0 @@ -{"flopping_freq": 1500.0164816344934} diff --git a/examples/master/repository/flopping_f_simulation.py b/examples/master/repository/flopping_f_simulation.py index 90cb2468d..3624502c9 100644 --- a/examples/master/repository/flopping_f_simulation.py +++ b/examples/master/repository/flopping_f_simulation.py @@ -37,11 +37,11 @@ class FloppingF(EnvExperiment): self.setattr_device("scheduler") def run(self): - frequency = self.set_result("flopping_f_frequency", [], - realtime=True, store=False) - brightness = self.set_result("flopping_f_brightness", [], - realtime=True) - self.set_result("flopping_f_fit", [], realtime=True, store=False) + frequency = self.set_dataset("flopping_f_frequency", [], + broadcast=True, save=False) + brightness = self.set_dataset("flopping_f_brightness", [], + broadcast=True) + self.set_dataset("flopping_f_fit", [], broadcast=True, save=False) for f in self.frequency_scan: m_brightness = model(f, self.F0) + self.noise_amplitude*random.random() @@ -52,16 +52,16 @@ class FloppingF(EnvExperiment): self.scheduler.priority, time.time() + 20, False) def analyze(self): - # Use get_result so that analyze can be run stand-alone. - frequency = self.get_result("flopping_f_frequency") - brightness = self.get_result("flopping_f_brightness") + # Use get_dataset so that analyze can be run stand-alone. + frequency = self.get_dataset("flopping_f_frequency") + brightness = self.get_dataset("flopping_f_brightness") popt, pcov = curve_fit(model_numpy, frequency, brightness, - p0=[self.get_parameter("flopping_freq")]) + p0=[self.get_dataset("flopping_freq")]) perr = np.sqrt(np.diag(pcov)) if perr < 0.1: F0 = float(popt) - self.set_parameter("flopping_freq", F0) - self.set_result("flopping_f_fit", - [model(x, F0) for x in frequency], - realtime=True, store=False) + self.set_dataset("flopping_freq", F0, persist=True, save=False) + self.set_dataset("flopping_f_fit", + [model(x, F0) for x in frequency], + broadcast=True, save=False) diff --git a/examples/master/repository/photon_histogram.py b/examples/master/repository/photon_histogram.py index 99f66c388..a42552d53 100644 --- a/examples/master/repository/photon_histogram.py +++ b/examples/master/repository/photon_histogram.py @@ -16,9 +16,9 @@ class PhotonHistogram(EnvExperiment): self.setattr_argument("nbins", FreeValue(100)) self.setattr_argument("repeats", FreeValue(100)) - self.setattr_parameter("cool_f", 230*MHz) - self.setattr_parameter("detect_f", 220*MHz) - self.setattr_parameter("detect_t", 100*us) + self.setattr_dataset("cool_f", 230*MHz) + self.setattr_dataset("detect_f", 220*MHz) + self.setattr_dataset("detect_t", 100*us) @kernel def program_cooling(self): @@ -60,8 +60,9 @@ class PhotonHistogram(EnvExperiment): hist[n] += 1 total += n - self.set_result("cooling_photon_histogram", hist) - self.set_parameter("ion_present", total > 5*self.repeats) + self.set_dataset("cooling_photon_histogram", hist) + self.set_dataset("ion_present", total > 5*self.repeats, + broadcast=True) if __name__ == "__main__": diff --git a/examples/master/repository/speed_benchmark.py b/examples/master/repository/speed_benchmark.py index 8d8921bcb..11ad806b5 100644 --- a/examples/master/repository/speed_benchmark.py +++ b/examples/master/repository/speed_benchmark.py @@ -111,9 +111,9 @@ class SpeedBenchmark(EnvExperiment): self.scheduler.pause() end_time = time.monotonic() - self.set_result("benchmark_run_time", - (end_time-start_time)/self.nruns, - realtime=True) + self.set_dataset("benchmark_run_time", + (end_time-start_time)/self.nruns, + broadcast=True) def run(self): if self.mode == "Single experiment": @@ -133,6 +133,6 @@ class _Report(EnvExperiment): def run(self): end_time = time.monotonic() - self.set_result("benchmark_run_time", - (end_time-self.start_time)/self.nruns, - realtime=True) + self.set_dataset("benchmark_run_time", + (end_time-self.start_time)/self.nruns, + broadcast=True) diff --git a/examples/sim/al_spectroscopy.py b/examples/sim/al_spectroscopy.py index df038aa63..3cde9c72d 100644 --- a/examples/sim/al_spectroscopy.py +++ b/examples/sim/al_spectroscopy.py @@ -12,7 +12,7 @@ class AluminumSpectroscopy(EnvExperiment): self.setattr_device("spectroscopy_b") self.setattr_device("state_detection") self.setattr_device("pmt") - self.setattr_parameter("spectroscopy_freq", 432*MHz) + self.setattr_dataset("spectroscopy_freq", 432*MHz) self.setattr_argument("photon_limit_low", FreeValue(10)) self.setattr_argument("photon_limit_high", FreeValue(15)) diff --git a/examples/sim/pdb.pyon b/examples/sim/dataset_db.pyon similarity index 100% rename from examples/sim/pdb.pyon rename to examples/sim/dataset_db.pyon diff --git a/examples/sim/ddb.pyon b/examples/sim/device_db.pyon similarity index 100% rename from examples/sim/ddb.pyon rename to examples/sim/device_db.pyon From 3923dd83c2b260766c2b4f9e0b3274b1f5639075 Mon Sep 17 00:00:00 2001 From: Sebastien Bourdeauducq Date: Mon, 12 Oct 2015 17:20:01 +0800 Subject: [PATCH 43/53] protocols/file_db: remove unneeded hooks feature --- artiq/protocols/file_db.py | 9 --------- 1 file changed, 9 deletions(-) diff --git a/artiq/protocols/file_db.py b/artiq/protocols/file_db.py index 089763cc1..8a0da9aa0 100644 --- a/artiq/protocols/file_db.py +++ b/artiq/protocols/file_db.py @@ -1,5 +1,3 @@ -from time import time - from artiq.protocols import pyon from artiq.protocols.sync_struct import Notifier @@ -8,7 +6,6 @@ class FlatFileDB: def __init__(self, filename): self.filename = filename self.data = Notifier(pyon.load_file(self.filename)) - self.hooks = [] def save(self): pyon.store_file(self.filename, self.data.read) @@ -19,13 +16,7 @@ class FlatFileDB: def set(self, key, value): self.data[key] = value self.save() - timestamp = time() - for hook in self.hooks: - hook.set(timestamp, key, value) def delete(self, key): del self.data[key] self.save() - timestamp = time() - for hook in self.hooks: - hook.delete(timestamp, key) From e6e93ab6edf15f2b9d032f05f0eb2625cffdbbf0 Mon Sep 17 00:00:00 2001 From: Sebastien Bourdeauducq Date: Mon, 12 Oct 2015 17:31:55 +0800 Subject: [PATCH 44/53] gui: dataset search --- artiq/gui/datasets.py | 20 ++++++++++-- artiq/gui/parameters.py | 72 ----------------------------------------- 2 files changed, 18 insertions(+), 74 deletions(-) delete mode 100644 artiq/gui/parameters.py diff --git a/artiq/gui/datasets.py b/artiq/gui/datasets.py index 8d50f761d..d951143d3 100644 --- a/artiq/gui/datasets.py +++ b/artiq/gui/datasets.py @@ -49,14 +49,19 @@ class DatasetsDock(dockarea.Dock): grid = LayoutWidget() self.addWidget(grid) + self.search = QtGui.QLineEdit() + self.search.setPlaceholderText("search...") + self.search.editingFinished.connect(self._search_datasets) + grid.addWidget(self.search, 0, ) + self.table = QtGui.QTableView() self.table.setSelectionMode(QtGui.QAbstractItemView.NoSelection) self.table.horizontalHeader().setResizeMode( QtGui.QHeaderView.ResizeToContents) - grid.addWidget(self.table, 0, 0) + grid.addWidget(self.table, 1, 0) add_display_box = QtGui.QGroupBox("Add display") - grid.addWidget(add_display_box, 0, 1) + grid.addWidget(add_display_box, 1, 1) display_grid = QtGui.QGridLayout() add_display_box.setLayout(display_grid) @@ -67,6 +72,17 @@ class DatasetsDock(dockarea.Dock): self.displays = dict() + def _search_datasets(self): + model = self.table_model + search = self.search.displayText() + for row in range(model.rowCount(model.index(0, 0))): + index = model.index(row, 0) + dataset = model.data(index, QtCore.Qt.DisplayRole) + if search in dataset: + self.table.showRow(row) + else: + self.table.hideRow(row) + def get_dataset(self, key): return self.table_model.backing_store[key][1] diff --git a/artiq/gui/parameters.py b/artiq/gui/parameters.py deleted file mode 100644 index 21b91a499..000000000 --- a/artiq/gui/parameters.py +++ /dev/null @@ -1,72 +0,0 @@ -import asyncio - -from quamash import QtGui, QtCore -from pyqtgraph import dockarea -from pyqtgraph import LayoutWidget - -from artiq.protocols.sync_struct import Subscriber -from artiq.gui.tools import DictSyncModel, short_format - - -class ParametersModel(DictSyncModel): - def __init__(self, parent, init): - DictSyncModel.__init__(self, ["Parameter", "Value"], - parent, init) - - def sort_key(self, k, v): - return k - - def convert(self, k, v, column): - if column == 0: - return k - elif column == 1: - return short_format(v) - else: - raise ValueError - - -class ParametersDock(dockarea.Dock): - def __init__(self): - dockarea.Dock.__init__(self, "Parameters", size=(400, 300)) - - grid = LayoutWidget() - self.addWidget(grid) - - self.search = QtGui.QLineEdit() - self.search.setPlaceholderText("search...") - self.search.editingFinished.connect(self._search_parameters) - grid.addWidget(self.search, 0, 0) - - self.table = QtGui.QTableView() - self.table.setSelectionMode(QtGui.QAbstractItemView.NoSelection) - self.table.horizontalHeader().setResizeMode( - QtGui.QHeaderView.ResizeToContents) - grid.addWidget(self.table, 1, 0) - - def get_parameter(self, key): - return self.table_model.backing_store[key] - - def _search_parameters(self): - model = self.table.model() - parentIndex = model.index(0, 0) - numRows = model.rowCount(parentIndex) - - for row in range(numRows): - index = model.index(row, 0) - parameter = model.data(index, QtCore.Qt.DisplayRole) - if parameter.startswith(self.search.displayText()): - self.table.showRow(row) - else: - self.table.hideRow(row) - - async def sub_connect(self, host, port): - self.subscriber = Subscriber("parameters", self.init_parameters_model) - await self.subscriber.connect(host, port) - - async def sub_close(self): - await self.subscriber.close() - - def init_parameters_model(self, init): - self.table_model = ParametersModel(self.table, init) - self.table.setModel(self.table_model) - return self.table_model From 22bffa98b5da846b35c5cfe84b084c5fd822b513 Mon Sep 17 00:00:00 2001 From: Sebastien Bourdeauducq Date: Mon, 12 Oct 2015 18:10:58 +0800 Subject: [PATCH 45/53] client: use short_format in dataset display --- artiq/frontend/artiq_client.py | 3 ++- artiq/gui/datasets.py | 3 ++- artiq/gui/tools.py | 35 ---------------------------------- artiq/tools.py | 35 ++++++++++++++++++++++++++++++++++ 4 files changed, 39 insertions(+), 37 deletions(-) diff --git a/artiq/frontend/artiq_client.py b/artiq/frontend/artiq_client.py index b1bdd2bf3..2f8995482 100755 --- a/artiq/frontend/artiq_client.py +++ b/artiq/frontend/artiq_client.py @@ -12,6 +12,7 @@ from prettytable import PrettyTable from artiq.protocols.pc_rpc import Client from artiq.protocols.sync_struct import Subscriber from artiq.protocols import pyon +from artiq.tools import short_format def clear_screen(): @@ -192,7 +193,7 @@ def _show_datasets(datasets): clear_screen() table = PrettyTable(["Dataset", "Persistent", "Value"]) for k, (persist, value) in sorted(datasets.items(), key=itemgetter(0)): - table.add_row([k, "Y" if persist else "N", str(value)]) + table.add_row([k, "Y" if persist else "N", short_format(value)]) print(table) diff --git a/artiq/gui/datasets.py b/artiq/gui/datasets.py index d951143d3..ebe8d295c 100644 --- a/artiq/gui/datasets.py +++ b/artiq/gui/datasets.py @@ -8,7 +8,8 @@ from pyqtgraph import dockarea from pyqtgraph import LayoutWidget from artiq.protocols.sync_struct import Subscriber -from artiq.gui.tools import DictSyncModel, short_format +from artiq.tools import short_format +from artiq.gui.tools import DictSyncModel from artiq.gui.displays import * diff --git a/artiq/gui/tools.py b/artiq/gui/tools.py index 265a46491..242c66600 100644 --- a/artiq/gui/tools.py +++ b/artiq/gui/tools.py @@ -2,41 +2,6 @@ from quamash import QtCore import numpy as np -def elide(s, maxlen): - elided = False - if len(s) > maxlen: - s = s[:maxlen] - elided = True - try: - idx = s.index("\n") - except ValueError: - pass - else: - s = s[:idx] - elided = True - if elided: - maxlen -= 3 - if len(s) > maxlen: - s = s[:maxlen] - s += "..." - return s - - -def short_format(v): - if v is None: - return "None" - t = type(v) - if np.issubdtype(t, int) or np.issubdtype(t, float): - return str(v) - elif t is str: - return "\"" + elide(v, 15) + "\"" - else: - r = t.__name__ - if t is list or t is dict or t is set: - r += " ({})".format(len(v)) - return r - - def si_prefix(scale): try: return { diff --git a/artiq/tools.py b/artiq/tools.py index 56ae0fda1..7505081f8 100644 --- a/artiq/tools.py +++ b/artiq/tools.py @@ -23,6 +23,41 @@ def parse_arguments(arguments): return d +def elide(s, maxlen): + elided = False + if len(s) > maxlen: + s = s[:maxlen] + elided = True + try: + idx = s.index("\n") + except ValueError: + pass + else: + s = s[:idx] + elided = True + if elided: + maxlen -= 3 + if len(s) > maxlen: + s = s[:maxlen] + s += "..." + return s + + +def short_format(v): + if v is None: + return "None" + t = type(v) + if np.issubdtype(t, int) or np.issubdtype(t, float): + return str(v) + elif t is str: + return "\"" + elide(v, 15) + "\"" + else: + r = t.__name__ + if t is list or t is dict or t is set: + r += " ({})".format(len(v)) + return r + + def file_import(filename): linecache.checkcache(filename) From a83ffb3dce791e0ff7ecae241a4bcfad582ec014 Mon Sep 17 00:00:00 2001 From: Sebastien Bourdeauducq Date: Mon, 12 Oct 2015 18:19:28 +0800 Subject: [PATCH 46/53] protocols: move FlatFileDB into pyon --- artiq/frontend/artiq_compile.py | 2 +- artiq/protocols/file_db.py | 22 ---------------------- artiq/protocols/pyon.py | 20 ++++++++++++++++++++ artiq/test/hardware_testbench.py | 2 +- 4 files changed, 22 insertions(+), 24 deletions(-) delete mode 100644 artiq/protocols/file_db.py diff --git a/artiq/frontend/artiq_compile.py b/artiq/frontend/artiq_compile.py index c47360c51..a058a298a 100755 --- a/artiq/frontend/artiq_compile.py +++ b/artiq/frontend/artiq_compile.py @@ -3,7 +3,7 @@ import logging import argparse -from artiq.protocols.file_db import FlatFileDB +from artiq.protocols.pyon import FlatFileDB from artiq.master.worker_db import DeviceManager from artiq.tools import * diff --git a/artiq/protocols/file_db.py b/artiq/protocols/file_db.py deleted file mode 100644 index 8a0da9aa0..000000000 --- a/artiq/protocols/file_db.py +++ /dev/null @@ -1,22 +0,0 @@ -from artiq.protocols import pyon -from artiq.protocols.sync_struct import Notifier - - -class FlatFileDB: - def __init__(self, filename): - self.filename = filename - self.data = Notifier(pyon.load_file(self.filename)) - - def save(self): - pyon.store_file(self.filename, self.data.read) - - def get(self, key): - return self.data.read[key] - - def set(self, key, value): - self.data[key] = value - self.save() - - def delete(self, key): - del self.data[key] - self.save() diff --git a/artiq/protocols/pyon.py b/artiq/protocols/pyon.py index ff793945d..471025b43 100644 --- a/artiq/protocols/pyon.py +++ b/artiq/protocols/pyon.py @@ -187,3 +187,23 @@ def load_file(filename): """Parses the specified file and returns the decoded Python object.""" with open(filename, "r") as f: return decode(f.read()) + + +class FlatFileDB: + def __init__(self, filename): + self.filename = filename + self.data = pyon.load_file(self.filename) + + def save(self): + pyon.store_file(self.filename, self.data) + + def get(self, key): + return self.data[key] + + def set(self, key, value): + self.data[key] = value + self.save() + + def delete(self, key): + del self.data[key] + self.save() diff --git a/artiq/test/hardware_testbench.py b/artiq/test/hardware_testbench.py index 550e04371..cbb0dabe9 100644 --- a/artiq/test/hardware_testbench.py +++ b/artiq/test/hardware_testbench.py @@ -6,7 +6,7 @@ import unittest import logging from artiq.language import * -from artiq.protocols.file_db import FlatFileDB +from artiq.protocols.pyon import FlatFileDB from artiq.master.worker_db import DeviceManager, ResultDB from artiq.frontend.artiq_run import DummyScheduler From a754d4b5f5274fe32f409bc02a35651af2b8772b Mon Sep 17 00:00:00 2001 From: Sebastien Bourdeauducq Date: Mon, 12 Oct 2015 19:20:04 +0800 Subject: [PATCH 47/53] test: use new dataset API --- artiq/test/coredevice.py | 45 ++++++++++++++++---------------- artiq/test/coredevice_vs_host.py | 2 +- artiq/test/hardware_testbench.py | 24 +++++++++-------- artiq/test/scheduler.py | 13 +++++---- 4 files changed, 45 insertions(+), 39 deletions(-) diff --git a/artiq/test/coredevice.py b/artiq/test/coredevice.py index 33fad9098..aec923bd3 100644 --- a/artiq/test/coredevice.py +++ b/artiq/test/coredevice.py @@ -15,7 +15,7 @@ class RTT(EnvExperiment): self.setattr_device("ttl_inout") def set_rtt(self, rtt): - self.set_result("rtt", rtt) + self.set_dataset("rtt", rtt) @kernel def run(self): @@ -39,7 +39,7 @@ class Loopback(EnvExperiment): self.setattr_device("loop_out") def set_rtt(self, rtt): - self.set_result("rtt", rtt) + self.set_dataset("rtt", rtt) @kernel def run(self): @@ -61,7 +61,7 @@ class ClockGeneratorLoopback(EnvExperiment): self.setattr_device("loop_clock_out") def set_count(self, count): - self.set_result("count", count) + self.set_dataset("count", count) @kernel def run(self): @@ -82,7 +82,7 @@ class PulseRate(EnvExperiment): self.setattr_device("ttl_out") def set_pulse_rate(self, pulse_rate): - self.set_result("pulse_rate", pulse_rate) + self.set_dataset("pulse_rate", pulse_rate) @kernel def run(self): @@ -118,7 +118,7 @@ class LoopbackCount(EnvExperiment): self.setattr_argument("npulses") def set_count(self, count): - self.set_result("count", count) + self.set_dataset("count", count) @kernel def run(self): @@ -176,7 +176,7 @@ class TimeKeepsRunning(EnvExperiment): self.setattr_device("core") def set_time_at_start(self, time_at_start): - self.set_result("time_at_start", time_at_start) + self.set_dataset("time_at_start", time_at_start) @kernel def run(self): @@ -193,34 +193,34 @@ class Handover(EnvExperiment): def run(self): self.get_now() - self.set_result("t1", self.time_at_start) + self.set_dataset("t1", self.time_at_start) self.get_now() - self.set_result("t2", self.time_at_start) + self.set_dataset("t2", self.time_at_start) class CoredeviceTest(ExperimentCase): def test_rtt(self): self.execute(RTT) - rtt = self.rdb.get("rtt") + rtt = self.dataset_mgr.get("rtt") print(rtt) self.assertGreater(rtt, 0*ns) self.assertLess(rtt, 100*ns) def test_loopback(self): self.execute(Loopback) - rtt = self.rdb.get("rtt") + rtt = self.dataset_mgr.get("rtt") print(rtt) self.assertGreater(rtt, 0*ns) self.assertLess(rtt, 50*ns) def test_clock_generator_loopback(self): self.execute(ClockGeneratorLoopback) - count = self.rdb.get("count") + count = self.dataset_mgr.get("count") self.assertEqual(count, 10) def test_pulse_rate(self): self.execute(PulseRate) - rate = self.rdb.get("pulse_rate") + rate = self.dataset_mgr.get("pulse_rate") print(rate) self.assertGreater(rate, 100*ns) self.assertLess(rate, 2500*ns) @@ -228,7 +228,7 @@ class CoredeviceTest(ExperimentCase): def test_loopback_count(self): npulses = 2 self.execute(LoopbackCount, npulses=npulses) - count = self.rdb.get("count") + count = self.dataset_mgr.get("count") self.assertEqual(count, npulses) def test_underflow(self): @@ -250,17 +250,18 @@ class CoredeviceTest(ExperimentCase): def test_time_keeps_running(self): self.execute(TimeKeepsRunning) - t1 = self.rdb.get("time_at_start") + t1 = self.dataset_mgr.get("time_at_start") self.execute(TimeKeepsRunning) - t2 = self.rdb.get("time_at_start") - dead_time = mu_to_seconds(t2 - t1, self.dmgr.get("core")) + t2 = self.dataset_mgr.get("time_at_start") + dead_time = mu_to_seconds(t2 - t1, self.device_mgr.get("core")) print(dead_time) self.assertGreater(dead_time, 1*ms) self.assertLess(dead_time, 300*ms) def test_handover(self): self.execute(Handover) - self.assertEqual(self.rdb.get("t1"), self.rdb.get("t2")) + self.assertEqual(self.dataset_mgr.get("t1"), + self.dataset_mgr.get("t2")) class RPCTiming(EnvExperiment): @@ -283,14 +284,14 @@ class RPCTiming(EnvExperiment): def run(self): self.bench() mean = sum(self.ts)/self.repeats - self.set_result("rpc_time_stddev", sqrt( + self.set_dataset("rpc_time_stddev", sqrt( sum([(t - mean)**2 for t in self.ts])/self.repeats)) - self.set_result("rpc_time_mean", mean) + self.set_dataset("rpc_time_mean", mean) class RPCTest(ExperimentCase): def test_rpc_timing(self): self.execute(RPCTiming) - self.assertGreater(self.rdb.get("rpc_time_mean"), 100*ns) - self.assertLess(self.rdb.get("rpc_time_mean"), 15*ms) - self.assertLess(self.rdb.get("rpc_time_stddev"), 1*ms) + self.assertGreater(self.dataset_mgr.get("rpc_time_mean"), 100*ns) + self.assertLess(self.dataset_mgr.get("rpc_time_mean"), 15*ms) + self.assertLess(self.dataset_mgr.get("rpc_time_stddev"), 1*ms) diff --git a/artiq/test/coredevice_vs_host.py b/artiq/test/coredevice_vs_host.py index 2a412233f..aa9d8fd7d 100644 --- a/artiq/test/coredevice_vs_host.py +++ b/artiq/test/coredevice_vs_host.py @@ -85,7 +85,7 @@ class _Pulses(EnvExperiment): self.setattr_argument("output_list") for name in "a", "b", "c", "d": - pl = _PulseLogger(*self.dbs(), + pl = _PulseLogger(*self.managers(), output_list=self.output_list, name=name) setattr(self, name, pl) diff --git a/artiq/test/hardware_testbench.py b/artiq/test/hardware_testbench.py index cbb0dabe9..a9a9503af 100644 --- a/artiq/test/hardware_testbench.py +++ b/artiq/test/hardware_testbench.py @@ -1,3 +1,4 @@ +# Copyright (C) 2015 M-Labs Limited # Copyright (C) 2014, 2015 Robert Jordens import os @@ -6,8 +7,9 @@ import unittest import logging from artiq.language import * -from artiq.protocols.pyon import FlatFileDB -from artiq.master.worker_db import DeviceManager, ResultDB +from artiq.protocols import pyon +from artiq.master.databases import DeviceDB, DatasetDB +from artiq.master.worker_db import DeviceManager, DatasetManager from artiq.frontend.artiq_run import DummyScheduler @@ -18,14 +20,14 @@ logger = logging.getLogger(__name__) def get_from_ddb(*path, default="skip"): if not artiq_root: raise unittest.SkipTest("no ARTIQ_ROOT") - v = FlatFileDB(os.path.join(artiq_root, "ddb.pyon")).data + v = pyon.load_file(os.path.join(artiq_root, "device_db.pyon")) try: for p in path: v = v[p] return v.read except KeyError: if default == "skip": - raise unittest.SkipTest("ddb path {} not found".format(path)) + raise unittest.SkipTest("device db path {} not found".format(path)) else: return default @@ -33,11 +35,11 @@ def get_from_ddb(*path, default="skip"): @unittest.skipUnless(artiq_root, "no ARTIQ_ROOT") class ExperimentCase(unittest.TestCase): def setUp(self): - self.ddb = FlatFileDB(os.path.join(artiq_root, "ddb.pyon")) - self.dmgr = DeviceManager(self.ddb, + self.device_db = DeviceDB(os.path.join(artiq_root, "device_db.pyon")) + self.dataset_db = DatasetDB(os.path.join(artiq_root, "dataset_db.pyon")) + self.device_mgr = DeviceManager(self.device_db, virtual_devices={"scheduler": DummyScheduler()}) - self.pdb = FlatFileDB(os.path.join(artiq_root, "pdb.pyon")) - self.rdb = ResultDB() + self.dataset_mgr = DatasetManager(self.dataset_db) def execute(self, cls, **kwargs): expid = { @@ -45,10 +47,10 @@ class ExperimentCase(unittest.TestCase): "class_name": cls.__name__, "arguments": kwargs } - self.dmgr.virtual_devices["scheduler"].expid = expid + self.device_mgr.virtual_devices["scheduler"].expid = expid try: try: - exp = cls(self.dmgr, self.pdb, self.rdb, **kwargs) + exp = cls(self.device_mgr, self.dataset_mgr, **kwargs) except KeyError as e: # skip if ddb does not match requirements raise unittest.SkipTest(*e.args) @@ -57,4 +59,4 @@ class ExperimentCase(unittest.TestCase): exp.analyze() return exp finally: - self.dmgr.close_devices() + self.device_mgr.close_devices() diff --git a/artiq/test/scheduler.py b/artiq/test/scheduler.py index e92ac638c..c8d29759b 100644 --- a/artiq/test/scheduler.py +++ b/artiq/test/scheduler.py @@ -26,7 +26,8 @@ class BackgroundExperiment(EnvExperiment): self.scheduler.pause() sleep(0.2) except TerminationRequested: - self.set_parameter("termination_ok", True) + self.set_dataset("termination_ok", True, + broadcast=True, save=False) def _get_expid(name): @@ -108,13 +109,15 @@ class SchedulerCase(unittest.TestCase): loop = self.loop termination_ok = False - def check_termination(key, value): + def check_termination(mod): nonlocal termination_ok - self.assertEqual(key, "termination_ok") - self.assertEqual(value, True) + self.assertEqual( + mod, + {"action": "setitem", "key": "termination_ok", + "value": (False, True), "path": []}) termination_ok = True handlers = { - "set_parameter": check_termination + "update_dataset": check_termination } scheduler = Scheduler(0, handlers, None) From b51910fa42c31bfca18bb3ccd425727310479905 Mon Sep 17 00:00:00 2001 From: Sebastien Bourdeauducq Date: Mon, 12 Oct 2015 19:32:16 +0800 Subject: [PATCH 48/53] compile,coretool: use new dataset API --- artiq/frontend/artiq_compile.py | 20 ++++++++++---------- artiq/frontend/artiq_coretool.py | 12 ++++++------ 2 files changed, 16 insertions(+), 16 deletions(-) diff --git a/artiq/frontend/artiq_compile.py b/artiq/frontend/artiq_compile.py index a058a298a..e55b7ee1b 100755 --- a/artiq/frontend/artiq_compile.py +++ b/artiq/frontend/artiq_compile.py @@ -3,8 +3,8 @@ import logging import argparse -from artiq.protocols.pyon import FlatFileDB -from artiq.master.worker_db import DeviceManager +from artiq.master.databases import DeviceDB, DatasetDB +from artiq.master.worker_db import DeviceManager, DatasetManager from artiq.tools import * @@ -15,10 +15,10 @@ def get_argparser(): parser = argparse.ArgumentParser(description="ARTIQ static compiler") verbosity_args(parser) - parser.add_argument("-d", "--ddb", default="ddb.pyon", - help="device database file") - parser.add_argument("-p", "--pdb", default="pdb.pyon", - help="parameter database file") + parser.add_argument("--device-db", default="device_db.pyon", + help="device database file (default: '%(default)s')") + parser.add_argument("--dataset-db", default="dataset_db.pyon", + help="dataset file (default: '%(default)s')") parser.add_argument("-e", "--experiment", default=None, help="experiment to compile") @@ -36,14 +36,14 @@ def main(): args = get_argparser().parse_args() init_logger(args) - dmgr = DeviceManager(FlatFileDB(args.ddb)) - pdb = FlatFileDB(args.pdb) + device_mgr = DeviceManager(DeviceDB(args.device_db)) + dataset_mgr = DatasetManager(DatasetDB(args.dataset_db)) try: module = file_import(args.file) exp = get_experiment(module, args.experiment) arguments = parse_arguments(args.arguments) - exp_inst = exp(dmgr, pdb, **arguments) + exp_inst = exp(device_mgr, dataset_mgr, **arguments) if (not hasattr(exp.run, "k_function_info") or not exp.run.k_function_info): @@ -55,7 +55,7 @@ def main(): [exp_inst], {}, with_attr_writeback=False) finally: - dmgr.close_devices() + device_mgr.close_devices() if rpc_map: raise ValueError("Experiment must not use RPC") diff --git a/artiq/frontend/artiq_coretool.py b/artiq/frontend/artiq_coretool.py index 41649f03e..0574c53c1 100755 --- a/artiq/frontend/artiq_coretool.py +++ b/artiq/frontend/artiq_coretool.py @@ -2,8 +2,8 @@ import argparse +from artiq.master.databases import DeviceDB from artiq.master.worker_db import DeviceManager -from artiq.protocols.file_db import FlatFileDB def to_bytes(string): @@ -13,8 +13,8 @@ def to_bytes(string): def get_argparser(): parser = argparse.ArgumentParser(description="ARTIQ core device " "remote access tool") - parser.add_argument("--ddb", default="ddb.pyon", - help="device database file") + parser.add_argument("--device-db", default="device_db.pyon", + help="device database file (default: '%(default)s')") subparsers = parser.add_subparsers(dest="action") subparsers.required = True @@ -58,9 +58,9 @@ def get_argparser(): def main(): args = get_argparser().parse_args() - dmgr = DeviceManager(FlatFileDB(args.ddb)) + device_mgr = DeviceManager(DeviceDB(args.device_db)) try: - comm = dmgr.get("comm") + comm = device_mgr.get("comm") if args.action == "log": print(comm.get_log()) @@ -82,7 +82,7 @@ def main(): elif args.action == "cfg-erase": comm.flash_storage_erase() finally: - dmgr.close_devices() + device_mgr.close_devices() if __name__ == "__main__": main() From 9e64f7dc3ae5c4a223e732e5987ea065a5f539f8 Mon Sep 17 00:00:00 2001 From: whitequark Date: Mon, 12 Oct 2015 14:45:11 +0300 Subject: [PATCH 49/53] Adapt old compiler to Python 3.5 AST. --- artiq/transforms/inline.py | 6 ++---- artiq/transforms/interleave.py | 3 +-- artiq/transforms/lower_time.py | 6 ++---- artiq/transforms/quantize_time.py | 12 ++++------- artiq/transforms/tools.py | 6 ++---- artiq/transforms/unparse.py | 35 +++++-------------------------- 6 files changed, 16 insertions(+), 52 deletions(-) diff --git a/artiq/transforms/inline.py b/artiq/transforms/inline.py index dd67e0c69..4d444bbe8 100644 --- a/artiq/transforms/inline.py +++ b/artiq/transforms/inline.py @@ -356,8 +356,7 @@ class Function: exception_id = self.mappers.exception.encode(exception_class) return ast.copy_location( ast.Call(func=ast.Name("EncodedException", ast.Load()), - args=[value_to_ast(exception_id)], - keywords=[], starargs=None, kwargs=None), + args=[value_to_ast(exception_id)], keywords=[]), e) def code_visit_Raise(self, node): @@ -514,8 +513,7 @@ def get_attr_writeback(attribute_namespace, rpc_mapper, loc_node): arg3 = ast.copy_location( ast.Name(attr_info.mangled_name, ast.Load()), loc_node) call = ast.copy_location( - ast.Call(func=func, args=[arg1, arg2, arg3], - keywords=[], starargs=None, kwargs=None), + ast.Call(func=func, args=[arg1, arg2, arg3], keywords=[]), loc_node) expr = ast.copy_location(ast.Expr(call), loc_node) attr_writeback.append(expr) diff --git a/artiq/transforms/interleave.py b/artiq/transforms/interleave.py index 8aa9df838..7d1f733ff 100644 --- a/artiq/transforms/interleave.py +++ b/artiq/transforms/interleave.py @@ -71,8 +71,7 @@ def _interleave_timelines(timelines): delay_stmt = ast.copy_location( ast.Expr(ast.Call( func=ast.Name("delay_mu", ast.Load()), - args=[value_to_ast(dt)], - keywords=[], starargs=[], kwargs=[])), + args=[value_to_ast(dt)], keywords=[])), ref_stmt) r.append(delay_stmt) else: diff --git a/artiq/transforms/lower_time.py b/artiq/transforms/lower_time.py index 5b3df0245..904d73834 100644 --- a/artiq/transforms/lower_time.py +++ b/artiq/transforms/lower_time.py @@ -46,14 +46,12 @@ def lower_time(func_def): _TimeLowerer().visit(func_def) call_init = ast.Call( func=ast.Name("syscall", ast.Load()), - args=[ast.Str("now_init")], - keywords=[], starargs=None, kwargs=None) + args=[ast.Str("now_init")], keywords=[]) stmt_init = ast.Assign(targets=[ast.Name("now", ast.Store())], value=call_init) call_save = ast.Call( func=ast.Name("syscall", ast.Load()), - args=[ast.Str("now_save"), ast.Name("now", ast.Load())], - keywords=[], starargs=None, kwargs=None) + args=[ast.Str("now_save"), ast.Name("now", ast.Load())], keywords=[]) stmt_save = ast.Expr(call_save) func_def.body = [ stmt_init, diff --git a/artiq/transforms/quantize_time.py b/artiq/transforms/quantize_time.py index aad75069a..e4125d624 100644 --- a/artiq/transforms/quantize_time.py +++ b/artiq/transforms/quantize_time.py @@ -23,8 +23,7 @@ def _seconds_to_mu(ref_period, node): node) return ast.copy_location( ast.Call(func=ast.Name("round64", ast.Load()), - args=[divided], - keywords=[], starargs=[], kwargs=[]), + args=[divided], keywords=[]), divided) @@ -77,12 +76,10 @@ class _TimeQuantizer(ast.NodeTransformer): right=ast.Num(1000)) time_int = ast.Call( func=ast.Name("round", ast.Load()), - args=[time], - keywords=[], starargs=None, kwargs=None) + args=[time], keywords=[]) syscall_set = ast.Call( func=ast.Name("syscall", ast.Load()), - args=[ast.Str("watchdog_set"), time_int], - keywords=[], starargs=None, kwargs=None) + args=[ast.Str("watchdog_set"), time_int], keywords=[]) stmt_set = ast.copy_location( ast.Assign(targets=[ast.Name(idname, ast.Store())], value=syscall_set), @@ -91,8 +88,7 @@ class _TimeQuantizer(ast.NodeTransformer): syscall_clear = ast.Call( func=ast.Name("syscall", ast.Load()), args=[ast.Str("watchdog_clear"), - ast.Name(idname, ast.Load())], - keywords=[], starargs=None, kwargs=None) + ast.Name(idname, ast.Load())], keywords=[]) stmt_clear = ast.copy_location(ast.Expr(syscall_clear), node) node.items[0] = ast.withitem( diff --git a/artiq/transforms/tools.py b/artiq/transforms/tools.py index 97d596d2b..557a60221 100644 --- a/artiq/transforms/tools.py +++ b/artiq/transforms/tools.py @@ -40,8 +40,7 @@ def value_to_ast(value): if isinstance(value, core_language.int64): # must be before int return ast.Call( func=ast.Name("int64", ast.Load()), - args=[ast.Num(int(value))], - keywords=[], starargs=None, kwargs=None) + args=[ast.Num(int(value))], keywords=[]) elif isinstance(value, bool) or value is None: # must also be before int # isinstance(True/False, int) == True @@ -51,8 +50,7 @@ def value_to_ast(value): elif isinstance(value, Fraction): return ast.Call( func=ast.Name("Fraction", ast.Load()), - args=[ast.Num(value.numerator), ast.Num(value.denominator)], - keywords=[], starargs=None, kwargs=None) + args=[ast.Num(value.numerator), ast.Num(value.denominator)], keywords=[]) elif isinstance(value, str): return ast.Str(value) elif isinstance(value, list): diff --git a/artiq/transforms/unparse.py b/artiq/transforms/unparse.py index cacd6e73b..c796b58e8 100644 --- a/artiq/transforms/unparse.py +++ b/artiq/transforms/unparse.py @@ -211,20 +211,6 @@ class _Unparser: else: comma = True self.dispatch(e) - if t.starargs: - if comma: - self.write(", ") - else: - comma = True - self.write("*") - self.dispatch(t.starargs) - if t.kwargs: - if comma: - self.write(", ") - else: - comma = True - self.write("**") - self.dispatch(t.kwargs) self.write(")") self.enter() @@ -466,20 +452,6 @@ class _Unparser: else: comma = True self.dispatch(e) - if t.starargs: - if comma: - self.write(", ") - else: - comma = True - self.write("*") - self.dispatch(t.starargs) - if t.kwargs: - if comma: - self.write(", ") - else: - comma = True - self.write("**") - self.dispatch(t.kwargs) self.write(")") def _Subscript(self, t): @@ -571,8 +543,11 @@ class _Unparser: self.dispatch(t.kwarg.annotation) def _keyword(self, t): - self.write(t.arg) - self.write("=") + if t.arg is None: + self.write("**") + else: + self.write(t.arg) + self.write("=") self.dispatch(t.value) def _Lambda(self, t): From 5c4ed7a9bde6a5df559ff2638b057de6619e0ac7 Mon Sep 17 00:00:00 2001 From: Sebastien Bourdeauducq Date: Mon, 12 Oct 2015 19:46:14 +0800 Subject: [PATCH 50/53] fix imports --- artiq/gui/schedule.py | 3 ++- artiq/tools.py | 2 ++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/artiq/gui/schedule.py b/artiq/gui/schedule.py index 53d5197e9..1439dfa6d 100644 --- a/artiq/gui/schedule.py +++ b/artiq/gui/schedule.py @@ -6,7 +6,8 @@ from quamash import QtGui, QtCore from pyqtgraph import dockarea from artiq.protocols.sync_struct import Subscriber -from artiq.gui.tools import elide, DictSyncModel +from artiq.gui.tools import DictSyncModel +from artiq.tools import elide class _ScheduleModel(DictSyncModel): diff --git a/artiq/tools.py b/artiq/tools.py index 7505081f8..819979bb0 100644 --- a/artiq/tools.py +++ b/artiq/tools.py @@ -8,6 +8,8 @@ import time import collections import os.path +import numpy as np + from artiq.language.environment import is_experiment from artiq.protocols import pyon From 3cec1763184be9002b5e2b17b6f6c41e21272226 Mon Sep 17 00:00:00 2001 From: Sebastien Bourdeauducq Date: Mon, 12 Oct 2015 19:46:31 +0800 Subject: [PATCH 51/53] get_ddb -> get_device_db --- artiq/frontend/artiq_master.py | 4 ++-- artiq/language/environment.py | 6 +++--- artiq/master/databases.py | 2 +- artiq/master/repository.py | 11 ++++++----- artiq/master/worker_db.py | 4 ++-- artiq/master/worker_impl.py | 4 ++-- examples/master/repository/dds_setter.py | 4 ++-- 7 files changed, 18 insertions(+), 17 deletions(-) diff --git a/artiq/frontend/artiq_master.py b/artiq/frontend/artiq_master.py index 264658ef9..9a10e8c57 100755 --- a/artiq/frontend/artiq_master.py +++ b/artiq/frontend/artiq_master.py @@ -74,12 +74,12 @@ def main(): repo_backend = GitBackend(args.repository) else: repo_backend = FilesystemBackend(args.repository) - repository = Repository(repo_backend, device_db.get_ddb, log.log) + repository = Repository(repo_backend, device_db.get_device_db, log.log) atexit.register(repository.close) repository.scan_async() worker_handlers = { - "get_ddb": device_db.get_ddb, + "get_device_db": device_db.get_device_db, "get_device": device_db.get, "get_dataset": dataset_db.get, "update_dataset": dataset_db.update, diff --git a/artiq/language/environment.py b/artiq/language/environment.py index 4801720f7..7f378e092 100644 --- a/artiq/language/environment.py +++ b/artiq/language/environment.py @@ -186,11 +186,11 @@ class HasEnvironment: attribute are the same.""" setattr(self, key, self.get_argument(key, processor, group)) - def get_ddb(self): + def get_device_db(self): """Returns the full contents of the device database.""" if self.__parent is not None: - return self.__parent.get_ddb() - return self.__device_mgr.get_ddb() + return self.__parent.get_device_db() + return self.__device_mgr.get_device_db() def get_device(self, key): """Creates and returns a device driver.""" diff --git a/artiq/master/databases.py b/artiq/master/databases.py index acbf1a37a..40549683c 100644 --- a/artiq/master/databases.py +++ b/artiq/master/databases.py @@ -20,7 +20,7 @@ class DeviceDB: if k not in self.data.read or self.data.read[k] != new_data[k]: self.data[k] = new_data[k] - def get_ddb(self): + def get_device_db(self): return self.data.read def get(self, key): diff --git a/artiq/master/repository.py b/artiq/master/repository.py index c8024104d..f060fee87 100644 --- a/artiq/master/repository.py +++ b/artiq/master/repository.py @@ -12,13 +12,13 @@ from artiq.tools import exc_to_warning logger = logging.getLogger(__name__) -async def _scan_experiments(wd, get_ddb, log): +async def _scan_experiments(wd, get_device_db, log): r = dict() for f in os.listdir(wd): if f.endswith(".py"): try: worker = Worker({ - "get_ddb": get_ddb, + "get_device_db": get_device_db, "log": lambda message: log("scan", message) }) try: @@ -56,9 +56,9 @@ def _sync_explist(target, source): class Repository: - def __init__(self, backend, get_ddb_fn, log_fn): + def __init__(self, backend, get_device_db_fn, log_fn): self.backend = backend - self.get_ddb_fn = get_ddb_fn + self.get_device_db_fn = get_device_db_fn self.log_fn = log_fn self.cur_rev = self.backend.get_head_rev() @@ -81,7 +81,8 @@ class Repository: wd, _ = self.backend.request_rev(new_cur_rev) self.backend.release_rev(self.cur_rev) self.cur_rev = new_cur_rev - new_explist = await _scan_experiments(wd, self.get_ddb_fn, self.log_fn) + new_explist = await _scan_experiments(wd, self.get_device_db_fn, + self.log_fn) _sync_explist(self.explist, new_explist) finally: diff --git a/artiq/master/worker_db.py b/artiq/master/worker_db.py index f36371a1f..b9f76bae0 100644 --- a/artiq/master/worker_db.py +++ b/artiq/master/worker_db.py @@ -39,9 +39,9 @@ class DeviceManager: self.virtual_devices = virtual_devices self.active_devices = OrderedDict() - def get_ddb(self): + def get_device_db(self): """Returns the full contents of the device database.""" - return self.ddb.get_ddb() + return self.ddb.get_device_db() def get(self, name): """Get the device driver or controller client corresponding to a diff --git a/artiq/master/worker_impl.py b/artiq/master/worker_impl.py index ff9a58a4c..ad18d9149 100644 --- a/artiq/master/worker_impl.py +++ b/artiq/master/worker_impl.py @@ -63,7 +63,7 @@ class LogForwarder: class ParentDeviceDB: - get_ddb = make_parent_action("get_ddb", "") + get_device_db = make_parent_action("get_device_db", "") get = make_parent_action("get_device", "key", KeyError) @@ -124,7 +124,7 @@ register_experiment = make_parent_action("register_experiment", class ExamineDeviceMgr: - get_ddb = make_parent_action("get_ddb", "") + get_device_db = make_parent_action("get_device_db", "") def get(self, name): return None diff --git a/examples/master/repository/dds_setter.py b/examples/master/repository/dds_setter.py index ebcd72bc1..1e35496d8 100644 --- a/examples/master/repository/dds_setter.py +++ b/examples/master/repository/dds_setter.py @@ -8,8 +8,8 @@ class DDSSetter(EnvExperiment): def build(self): self.dds = dict() - ddb = self.get_ddb() - for k, v in sorted(ddb.items(), key=itemgetter(0)): + device_db = self.get_device_db() + for k, v in sorted(device_db.items(), key=itemgetter(0)): if (isinstance(v, dict) and v["type"] == "local" and v["module"] == "artiq.coredevice.dds" From 1d14975bd56c341c6d7e13d0ba3114b569495ba4 Mon Sep 17 00:00:00 2001 From: Sebastien Bourdeauducq Date: Tue, 13 Oct 2015 01:11:57 +0800 Subject: [PATCH 52/53] worker: cleaner termination on exception in user code, improve unittest --- artiq/master/worker.py | 6 ++++++ artiq/master/worker_impl.py | 4 ++++ artiq/test/worker.py | 27 +++++++++++++++++++++++++-- 3 files changed, 35 insertions(+), 2 deletions(-) diff --git a/artiq/master/worker.py b/artiq/master/worker.py index faa2dcf18..59f3f9b4a 100644 --- a/artiq/master/worker.py +++ b/artiq/master/worker.py @@ -21,6 +21,10 @@ class WorkerWatchdogTimeout(Exception): pass +class WorkerException(Exception): + pass + + class WorkerError(Exception): pass @@ -159,6 +163,8 @@ class Worker: return True elif action == "pause": return False + elif action == "exception": + raise WorkerException del obj["action"] if action == "create_watchdog": func = self.create_watchdog diff --git a/artiq/master/worker_impl.py b/artiq/master/worker_impl.py index ad18d9149..8b22d3acf 100644 --- a/artiq/master/worker_impl.py +++ b/artiq/master/worker_impl.py @@ -1,6 +1,7 @@ import sys import time import os +import traceback from artiq.protocols import pyon from artiq.tools import file_import @@ -214,6 +215,9 @@ def main(): put_object({"action": "completed"}) elif action == "terminate": break + except: + traceback.print_exc() + put_object({"action": "exception"}) finally: device_mgr.close_devices() diff --git a/artiq/test/worker.py b/artiq/test/worker.py index 7be4304a0..59847f6a5 100644 --- a/artiq/test/worker.py +++ b/artiq/test/worker.py @@ -8,6 +8,22 @@ from artiq import * from artiq.master.worker import * +class SimpleExperiment(EnvExperiment): + def build(self): + pass + + def run(self): + pass + + +class ExceptionTermination(EnvExperiment): + def build(self): + pass + + def run(self): + raise TypeError + + class WatchdogNoTimeout(EnvExperiment): def build(self): pass @@ -53,11 +69,11 @@ def _run_experiment(class_name): "arguments": dict() } loop = asyncio.get_event_loop() - worker = Worker() + worker = Worker(handlers={"log": lambda message: None}) loop.run_until_complete(_call_worker(worker, expid)) -class WatchdogCase(unittest.TestCase): +class WorkerCase(unittest.TestCase): def setUp(self): if os.name == "nt": self.loop = asyncio.ProactorEventLoop() @@ -65,6 +81,13 @@ class WatchdogCase(unittest.TestCase): self.loop = asyncio.new_event_loop() asyncio.set_event_loop(self.loop) + def test_simple_run(self): + _run_experiment("SimpleExperiment") + + def test_exception(self): + with self.assertRaises(WorkerException): + _run_experiment("ExceptionTermination") + def test_watchdog_no_timeout(self): _run_experiment("WatchdogNoTimeout") From ceb18d05b3ebf47bc2a02875b2b869fc0f6297ad Mon Sep 17 00:00:00 2001 From: Sebastien Bourdeauducq Date: Tue, 13 Oct 2015 01:47:18 +0800 Subject: [PATCH 53/53] manual/management: add warning about setups that resolve the current hostname to localhost. closes #134 --- doc/manual/management_system.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/doc/manual/management_system.rst b/doc/manual/management_system.rst index b52325945..b3072ad49 100644 --- a/doc/manual/management_system.rst +++ b/doc/manual/management_system.rst @@ -22,6 +22,8 @@ A controller manager connects to the master and uses the device database to dete Controller managers use the local network address of the connection to the master to filter the device database and run only those controllers that are allocated to the current node. Hostname resolution is supported. +.. warning:: With some network setups, the current machine's hostname without the domain name resolves to a localhost address (127.0.0.1 or even 127.0.1.1). If you wish to use controllers across a network, make sure that the hostname you provide resolves to an IP address visible on the network (e.g. try providing the full hostname including the domain name). + Command-line client -------------------