From 3e22fe86b5fca3ada4717b50a1a0f0f308270eec Mon Sep 17 00:00:00 2001 From: Sebastien Bourdeauducq Date: Sat, 17 Jan 2015 19:38:20 +0800 Subject: [PATCH] reorganize files as per discussion with Robert --- artiq/frontend/client.py | 12 +++-- artiq/frontend/ctlid.py | 2 +- artiq/frontend/gui.py | 2 +- artiq/frontend/lda_client.py | 2 +- artiq/frontend/lda_controller.py | 2 +- artiq/frontend/master.py | 6 +-- artiq/frontend/pdq2_client.py | 2 +- artiq/frontend/pdq2_controller.py | 2 +- artiq/frontend/run.py | 7 +-- artiq/gui/parameters.py | 2 +- artiq/gui/rt_results.py | 2 +- artiq/gui/scheduler.py | 4 +- artiq/management/__init__.py | 0 artiq/{management => master}/db.py | 48 +------------------ artiq/master/rt_results.py | 2 +- artiq/master/scheduler.py | 2 +- artiq/master/worker.py | 2 +- artiq/master/worker_impl.py | 6 +-- .../tools.py => protocols/asyncio_server.py} | 16 ------- artiq/protocols/file_db.py | 47 ++++++++++++++++++ artiq/{management => protocols}/pc_rpc.py | 8 ++-- artiq/{management => protocols}/pyon.py | 0 .../{management => protocols}/sync_struct.py | 4 +- artiq/test/pc_rpc.py | 2 +- artiq/test/serialization.py | 2 +- artiq/{management/file_import.py => tools.py} | 11 +++++ doc/manual/drivers_reference.rst | 2 +- doc/manual/index.rst | 2 +- doc/manual/management_reference.rst | 27 ----------- doc/manual/protocols_reference.rst | 27 +++++++++++ doc/manual/writing_a_driver.rst | 8 ++-- 31 files changed, 131 insertions(+), 130 deletions(-) delete mode 100644 artiq/management/__init__.py rename artiq/{management => master}/db.py (60%) rename artiq/{management/tools.py => protocols/asyncio_server.py} (82%) create mode 100644 artiq/protocols/file_db.py rename artiq/{management => protocols}/pc_rpc.py (97%) rename artiq/{management => protocols}/pyon.py (100%) rename artiq/{management => protocols}/sync_struct.py (98%) rename artiq/{management/file_import.py => tools.py} (60%) delete mode 100644 doc/manual/management_reference.rst create mode 100644 doc/manual/protocols_reference.rst diff --git a/artiq/frontend/client.py b/artiq/frontend/client.py index f6b3a08b4..e7224e924 100755 --- a/artiq/frontend/client.py +++ b/artiq/frontend/client.py @@ -8,10 +8,14 @@ from operator import itemgetter from prettytable import PrettyTable -from artiq.management.pc_rpc import Client -from artiq.management.sync_struct import Subscriber -from artiq.management.tools import clear_screen, format_run_arguments -from artiq.management import pyon +from artiq.protocols.pc_rpc import Client +from artiq.protocols.sync_struct import Subscriber +from artiq.protocols import pyon +from artiq.tools import format_run_arguments + + +def clear_screen(): + sys.stdout.write("\x1b[2J\x1b[H") def _get_args(): diff --git a/artiq/frontend/ctlid.py b/artiq/frontend/ctlid.py index d850fd69e..55f260055 100755 --- a/artiq/frontend/ctlid.py +++ b/artiq/frontend/ctlid.py @@ -2,7 +2,7 @@ import argparse -from artiq.management.pc_rpc import Client +from artiq.protocols.pc_rpc import Client def _get_args(): diff --git a/artiq/frontend/gui.py b/artiq/frontend/gui.py index 0ad585d9a..c2599fe95 100755 --- a/artiq/frontend/gui.py +++ b/artiq/frontend/gui.py @@ -7,7 +7,7 @@ import atexit import gbulb from gi.repository import Gtk -from artiq.management.pc_rpc import AsyncioClient +from artiq.protocols.pc_rpc import AsyncioClient from artiq.gui.scheduler import SchedulerWindow from artiq.gui.parameters import ParametersWindow from artiq.gui.rt_results import RTResults diff --git a/artiq/frontend/lda_client.py b/artiq/frontend/lda_client.py index 23317cd92..bcfc4712f 100755 --- a/artiq/frontend/lda_client.py +++ b/artiq/frontend/lda_client.py @@ -2,7 +2,7 @@ import argparse -from artiq.management.pc_rpc import Client +from artiq.protocols.pc_rpc import Client def main(): diff --git a/artiq/frontend/lda_controller.py b/artiq/frontend/lda_controller.py index 857dd4b73..03e94fd99 100755 --- a/artiq/frontend/lda_controller.py +++ b/artiq/frontend/lda_controller.py @@ -3,7 +3,7 @@ import argparse from artiq.devices.lda.driver import Lda, Ldasim -from artiq.management.pc_rpc import simple_server_loop +from artiq.protocols.pc_rpc import simple_server_loop def main(): diff --git a/artiq/frontend/master.py b/artiq/frontend/master.py index 60d9c1a99..972b237b5 100755 --- a/artiq/frontend/master.py +++ b/artiq/frontend/master.py @@ -4,9 +4,9 @@ import asyncio import argparse import atexit -from artiq.management.pc_rpc import Server -from artiq.management.sync_struct import Publisher -from artiq.management.db import FlatFileDB, SimpleHistory +from artiq.protocols.pc_rpc import Server +from artiq.protocols.sync_struct import Publisher +from artiq.protocols.file_db import FlatFileDB, SimpleHistory from artiq.master.scheduler import Scheduler from artiq.master.rt_results import RTResults diff --git a/artiq/frontend/pdq2_client.py b/artiq/frontend/pdq2_client.py index bacb2215b..3ebc027c7 100755 --- a/artiq/frontend/pdq2_client.py +++ b/artiq/frontend/pdq2_client.py @@ -8,7 +8,7 @@ import time from scipy import interpolate import numpy as np -from artiq.management.pc_rpc import Client +from artiq.protocols.pc_rpc import Client def _get_args(): diff --git a/artiq/frontend/pdq2_controller.py b/artiq/frontend/pdq2_controller.py index 867503f91..0c4aa5677 100755 --- a/artiq/frontend/pdq2_controller.py +++ b/artiq/frontend/pdq2_controller.py @@ -4,7 +4,7 @@ import argparse import logging from artiq.devices.pdq2.driver import Pdq2 -from artiq.management.pc_rpc import simple_server_loop +from artiq.protocols.pc_rpc import simple_server_loop def _get_args(): diff --git a/artiq/frontend/run.py b/artiq/frontend/run.py index b8855e9f9..4f5ff437c 100755 --- a/artiq/frontend/run.py +++ b/artiq/frontend/run.py @@ -6,10 +6,11 @@ from inspect import isclass from operator import itemgetter from itertools import chain -from artiq.management.file_import import file_import from artiq.language.db import * -from artiq.management import pyon -from artiq.management.db import * +from artiq.protocols import pyon +from artiq.protocols.file_db import FlatFileDB +from artiq.master.db import DBHub, ResultDB +from artiq.tools import file_import class ELFRunner(AutoDB): diff --git a/artiq/gui/parameters.py b/artiq/gui/parameters.py index 60cd4c263..a48a992ad 100644 --- a/artiq/gui/parameters.py +++ b/artiq/gui/parameters.py @@ -5,7 +5,7 @@ import time from gi.repository import Gtk from artiq.gui.tools import Window, ListSyncer, DictSyncer -from artiq.management.sync_struct import Subscriber +from artiq.protocols.sync_struct import Subscriber class _ParameterStoreSyncer(DictSyncer): diff --git a/artiq/gui/rt_results.py b/artiq/gui/rt_results.py index d35802da2..309107f8d 100644 --- a/artiq/gui/rt_results.py +++ b/artiq/gui/rt_results.py @@ -4,7 +4,7 @@ from collections import defaultdict from gi.repository import Gtk import cairoplot -from artiq.management.sync_struct import Subscriber +from artiq.protocols.sync_struct import Subscriber from artiq.gui.tools import Window diff --git a/artiq/gui/scheduler.py b/artiq/gui/scheduler.py index 0dd7d0bfd..3c45cd846 100644 --- a/artiq/gui/scheduler.py +++ b/artiq/gui/scheduler.py @@ -4,8 +4,8 @@ import asyncio from gi.repository import Gtk from artiq.gui.tools import Window, ListSyncer, DictSyncer -from artiq.management.sync_struct import Subscriber -from artiq.management.tools import format_run_arguments +from artiq.protocols.sync_struct import Subscriber +from artiq.tools import format_run_arguments class _QueueStoreSyncer(ListSyncer): diff --git a/artiq/management/__init__.py b/artiq/management/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/artiq/management/db.py b/artiq/master/db.py similarity index 60% rename from artiq/management/db.py rename to artiq/master/db.py index 8dac97e0b..ae475189d 100644 --- a/artiq/management/db.py +++ b/artiq/master/db.py @@ -1,53 +1,7 @@ from collections import OrderedDict import importlib -from time import time -from artiq.language.db import * -from artiq.management import pyon -from artiq.management.sync_struct import Notifier - - -class FlatFileDB: - def __init__(self, filename): - self.filename = filename - self.data = Notifier(pyon.load_file(self.filename)) - self.hooks = [] - - def save(self): - pyon.store_file(self.filename, self.data.read) - - def request(self, name): - return self.data.read[name] - - def set(self, name, value): - self.data[name] = value - self.save() - timestamp = time() - for hook in self.hooks: - hook.set(timestamp, name, value) - - def delete(self, name): - del self.data[name] - self.save() - timestamp = time() - for hook in self.hooks: - hook.delete(timestamp, name) - - -class SimpleHistory: - def __init__(self, depth): - self.depth = depth - self.history = Notifier([]) - - def set(self, timestamp, name, value): - if len(self.history.read) >= self.depth: - del self.history[0] - self.history.append((timestamp, name, value)) - - def delete(self, timestamp, name): - if len(self.history.read) >= self.depth: - del self.history[0] - self.history.append((timestamp, name)) +from artiq.protocols.sync_struct import Notifier class ResultDB: diff --git a/artiq/master/rt_results.py b/artiq/master/rt_results.py index 7d07851a1..f046c6273 100644 --- a/artiq/master/rt_results.py +++ b/artiq/master/rt_results.py @@ -1,4 +1,4 @@ -from artiq.management.sync_struct import Notifier, process_mod +from artiq.protocols.sync_struct import Notifier, process_mod class RTResults: diff --git a/artiq/master/scheduler.py b/artiq/master/scheduler.py index 94d6a00e0..ecc4215eb 100644 --- a/artiq/master/scheduler.py +++ b/artiq/master/scheduler.py @@ -1,7 +1,7 @@ import asyncio from time import time -from artiq.management.sync_struct import Notifier +from artiq.protocols.sync_struct import Notifier from artiq.master.worker import Worker diff --git a/artiq/master/worker.py b/artiq/master/worker.py index b2d611255..168c11050 100644 --- a/artiq/master/worker.py +++ b/artiq/master/worker.py @@ -4,7 +4,7 @@ import subprocess import signal import traceback -from artiq.management import pyon +from artiq.protocols import pyon class WorkerFailed(Exception): diff --git a/artiq/master/worker_impl.py b/artiq/master/worker_impl.py index 9bf7c1d6f..a7cfe8aa3 100644 --- a/artiq/master/worker_impl.py +++ b/artiq/master/worker_impl.py @@ -2,10 +2,10 @@ import sys from inspect import isclass import traceback -from artiq.management import pyon -from artiq.management.file_import import file_import +from artiq.protocols import pyon +from artiq.tools import file_import from artiq.language.db import AutoDB -from artiq.management.db import DBHub, ResultDB +from artiq.master.db import DBHub, ResultDB def get_object(): diff --git a/artiq/management/tools.py b/artiq/protocols/asyncio_server.py similarity index 82% rename from artiq/management/tools.py rename to artiq/protocols/asyncio_server.py index 4ee89895b..451bb272f 100644 --- a/artiq/management/tools.py +++ b/artiq/protocols/asyncio_server.py @@ -1,21 +1,5 @@ import asyncio -import sys from copy import copy -from operator import itemgetter - - -def clear_screen(): - sys.stdout.write("\x1b[2J\x1b[H") - - -def format_run_arguments(arguments): - fmtargs = [] - for k, v in sorted(arguments.items(), key=itemgetter(0)): - fmtargs.append(k + "=" + str(v)) - if fmtargs: - return " ".join(fmtargs) - else: - return "-" class AsyncioServer: diff --git a/artiq/protocols/file_db.py b/artiq/protocols/file_db.py new file mode 100644 index 000000000..edf6e7a33 --- /dev/null +++ b/artiq/protocols/file_db.py @@ -0,0 +1,47 @@ +from time import time + +from artiq.protocols import pyon +from artiq.protocols.sync_struct import Notifier + + +class FlatFileDB: + def __init__(self, filename): + self.filename = filename + self.data = Notifier(pyon.load_file(self.filename)) + self.hooks = [] + + def save(self): + pyon.store_file(self.filename, self.data.read) + + def request(self, name): + return self.data.read[name] + + def set(self, name, value): + self.data[name] = value + self.save() + timestamp = time() + for hook in self.hooks: + hook.set(timestamp, name, value) + + def delete(self, name): + del self.data[name] + self.save() + timestamp = time() + for hook in self.hooks: + hook.delete(timestamp, name) + + +class SimpleHistory: + def __init__(self, depth): + self.depth = depth + self.history = Notifier([]) + + def set(self, timestamp, name, value): + if len(self.history.read) >= self.depth: + del self.history[0] + self.history.append((timestamp, name, value)) + + def delete(self, timestamp, name): + if len(self.history.read) >= self.depth: + del self.history[0] + self.history.append((timestamp, name)) diff --git a/artiq/management/pc_rpc.py b/artiq/protocols/pc_rpc.py similarity index 97% rename from artiq/management/pc_rpc.py rename to artiq/protocols/pc_rpc.py index 6d979f982..4f852518a 100644 --- a/artiq/management/pc_rpc.py +++ b/artiq/protocols/pc_rpc.py @@ -1,7 +1,7 @@ """ This module provides a remote procedure call (RPC) mechanism over sockets between conventional computers (PCs) running Python. It strives to be -transparent and uses ``artiq.management.pyon`` internally so that e.g. Numpy +transparent and uses ``artiq.protocols.pyon`` internally so that e.g. Numpy arrays can be easily used. Note that the server operates on copies of objects provided by the client, @@ -16,8 +16,8 @@ import socket import asyncio import traceback -from artiq.management import pyon -from artiq.management.tools import AsyncioServer as _AsyncioServer +from artiq.protocols import pyon +from artiq.protocols.asyncio_server import AsyncioServer as _AsyncioServer class RemoteError(Exception): @@ -140,7 +140,7 @@ class Client: class AsyncioClient: - """This class is similar to :class:`artiq.management.pc_rpc.Client`, but + """This class is similar to :class:`artiq.protocols.pc_rpc.Client`, but uses ``asyncio`` instead of blocking calls. All RPC methods are coroutines. diff --git a/artiq/management/pyon.py b/artiq/protocols/pyon.py similarity index 100% rename from artiq/management/pyon.py rename to artiq/protocols/pyon.py diff --git a/artiq/management/sync_struct.py b/artiq/protocols/sync_struct.py similarity index 98% rename from artiq/management/sync_struct.py rename to artiq/protocols/sync_struct.py index 9c1a5f0b4..d850f947e 100644 --- a/artiq/management/sync_struct.py +++ b/artiq/protocols/sync_struct.py @@ -1,8 +1,8 @@ import asyncio from operator import getitem -from artiq.management import pyon -from artiq.management.tools import AsyncioServer +from artiq.protocols import pyon +from artiq.protocols.asyncio_server import AsyncioServer _init_string = b"ARTIQ sync_struct\n" diff --git a/artiq/test/pc_rpc.py b/artiq/test/pc_rpc.py index 60a07ed95..566b81cbb 100644 --- a/artiq/test/pc_rpc.py +++ b/artiq/test/pc_rpc.py @@ -6,7 +6,7 @@ import time import numpy as np -from artiq.management import pc_rpc +from artiq.protocols import pc_rpc test_address = "::1" diff --git a/artiq/test/serialization.py b/artiq/test/serialization.py index 4a73ac9bd..9587a7a77 100644 --- a/artiq/test/serialization.py +++ b/artiq/test/serialization.py @@ -5,7 +5,7 @@ from fractions import Fraction import numpy as np from artiq.language.units import * -from artiq.management import pyon +from artiq.protocols import pyon _pyon_test_object = { diff --git a/artiq/management/file_import.py b/artiq/tools.py similarity index 60% rename from artiq/management/file_import.py rename to artiq/tools.py index f183b142b..ea8276ee1 100644 --- a/artiq/management/file_import.py +++ b/artiq/tools.py @@ -1,7 +1,18 @@ +from operator import itemgetter import importlib.machinery import linecache +def format_run_arguments(arguments): + fmtargs = [] + for k, v in sorted(arguments.items(), key=itemgetter(0)): + fmtargs.append(k + "=" + str(v)) + if fmtargs: + return " ".join(fmtargs) + else: + return "-" + + def file_import(filename): linecache.checkcache(filename) diff --git a/doc/manual/drivers_reference.rst b/doc/manual/drivers_reference.rst index 96604e6bb..e620966d8 100644 --- a/doc/manual/drivers_reference.rst +++ b/doc/manual/drivers_reference.rst @@ -5,7 +5,7 @@ These drivers are for "slow" devices that are directly controlled by a PC, typic Certain devices (such as the PDQ2) may still perform real-time operations by having certain controls physically connected to the core device (for example, the trigger and frame selection signals on the PDQ2). For handling such cases, parts of the drivers may be kernels executed on the core device. -Each driver is run in a separate "controller" that exposes a RPC interface (based on :class:`artiq.management.pc_rpc`) to its functions. The master never does direct I/O to the devices, but issues RPCs to the controllers when needed. As opposed to running everything on the master, this architecture has those main advantages: +Each driver is run in a separate "controller" that exposes a RPC interface (based on :class:`artiq.protocols.pc_rpc`) to its functions. The master never does direct I/O to the devices, but issues RPCs to the controllers when needed. As opposed to running everything on the master, this architecture has those main advantages: * Each driver can be run on a different machine, which alleviates cabling issues and OS compatibility problems. * Reduces the impact of driver crashes. diff --git a/doc/manual/index.rst b/doc/manual/index.rst index 94c1c2d5b..d7e039d3b 100644 --- a/doc/manual/index.rst +++ b/doc/manual/index.rst @@ -11,6 +11,6 @@ Contents: writing_a_driver core_language_reference core_drivers_reference - management_reference + protocols_reference drivers_reference fpga_board_ports diff --git a/doc/manual/management_reference.rst b/doc/manual/management_reference.rst deleted file mode 100644 index f0712824b..000000000 --- a/doc/manual/management_reference.rst +++ /dev/null @@ -1,27 +0,0 @@ -Management reference -==================== - -:mod:`artiq.management.pyon` module ------------------------------------ - -.. automodule:: artiq.management.pyon - :members: - -:mod:`artiq.management.tools` module ------------------------------------- - -.. automodule:: artiq.management.tools - :members: - - -:mod:`artiq.management.pc_rpc` module -------------------------------------- - -.. automodule:: artiq.management.pc_rpc - :members: - -:mod:`artiq.management.sync_struct` module ------------------------------------------- - -.. automodule:: artiq.management.sync_struct - :members: diff --git a/doc/manual/protocols_reference.rst b/doc/manual/protocols_reference.rst new file mode 100644 index 000000000..1050a23b7 --- /dev/null +++ b/doc/manual/protocols_reference.rst @@ -0,0 +1,27 @@ +Management reference +==================== + +:mod:`artiq.protocols.pyon` module +---------------------------------- + +.. automodule:: artiq.protocols.pyon + :members: + +:mod:`artiq.protocols.tools` module +----------------------------------- + +.. automodule:: artiq.protocols.tools + :members: + + +:mod:`artiq.protocols.pc_rpc` module +------------------------------------ + +.. automodule:: artiq.protocols.pc_rpc + :members: + +:mod:`artiq.protocols.sync_struct` module +----------------------------------------- + +.. automodule:: artiq.protocols.sync_struct + :members: diff --git a/doc/manual/writing_a_driver.rst b/doc/manual/writing_a_driver.rst index 2a2a1a7ec..9687df1d8 100644 --- a/doc/manual/writing_a_driver.rst +++ b/doc/manual/writing_a_driver.rst @@ -6,7 +6,7 @@ These instructions cover writing a simple driver for a "slow" device, that uses The controller -------------- -A controller is a piece of software that receives commands from a client over the network (or the ``localhost`` interface), drives a device, and returns information about the device to the client. The mechanism used is remote procedure calls (RPCs) using :class:`artiq.management.pc_rpc`, which makes the network layers transparent for the driver's user. +A controller is a piece of software that receives commands from a client over the network (or the ``localhost`` interface), drives a device, and returns information about the device to the client. The mechanism used is remote procedure calls (RPCs) using :class:`artiq.protocols.pc_rpc`, which makes the network layers transparent for the driver's user. The controller we will develop is for a "device" that is very easy to work with: the console from which the controller is run. The operation that the driver will implement is writing a message to that console. @@ -16,9 +16,9 @@ For using RPC, the functions that a driver provides must be the methods of a sin def message(self, msg): print("message: " + msg) -To turn it into a server, we use :class:`artiq.management.pc_rpc`. Import the function we will use: :: +To turn it into a server, we use :class:`artiq.protocols.pc_rpc`. Import the function we will use: :: - from artiq.management.pc_rpc import simple_server_loop + from artiq.protocols.pc_rpc import simple_server_loop and add a ``main`` function that is run when the program is executed: :: @@ -63,7 +63,7 @@ Create a ``hello_client.py`` file with the following contents: :: #!/usr/bin/env python3 - from artiq.management.pc_rpc import Client + from artiq.protocols.pc_rpc import Client def main():