From 7d815208279a528f8893aed9e6f23200f93747ab Mon Sep 17 00:00:00 2001 From: Sebastien Bourdeauducq Date: Sat, 25 Jul 2015 00:30:36 +0800 Subject: [PATCH 01/20] protocols/pc_rpc: improve docstrings --- artiq/protocols/pc_rpc.py | 35 ++++++++--------------------------- 1 file changed, 8 insertions(+), 27 deletions(-) diff --git a/artiq/protocols/pc_rpc.py b/artiq/protocols/pc_rpc.py index a16c3dfd9..1f9151a4f 100644 --- a/artiq/protocols/pc_rpc.py +++ b/artiq/protocols/pc_rpc.py @@ -9,7 +9,6 @@ and modifications to mutable types are not written back. For example, if the client passes a list as a parameter of an RPC method, and that method ``append()s`` an element to the list, the element is not appended to the client's list. - """ import socket @@ -29,17 +28,13 @@ logger = logging.getLogger(__name__) class RemoteError(Exception): """Raised when a RPC failed or raised an exception on the remote (server) - side. - - """ + side.""" pass class IncompatibleServer(Exception): """Raised by the client when attempting to connect to a server that does - not have the expected target. - - """ + not have the expected target.""" pass @@ -74,7 +69,6 @@ class Client: Use ``None`` to skip selecting a target. The list of targets can then be retrieved using ``get_rpc_id`` and then one can be selected later using ``select_rpc_target``. - """ def __init__(self, host, port, target_name): self.__socket = socket.create_connection((host, port)) @@ -93,25 +87,20 @@ class Client: def select_rpc_target(self, target_name): """Selects a RPC target by name. This function should be called - exactly once if the object was created with ``target_name=None``. - - """ + exactly once if the object was created with ``target_name=None``.""" if target_name not in self.__target_names: raise IncompatibleServer self.__socket.sendall((target_name + "\n").encode()) def get_rpc_id(self): """Returns a tuple (target_names, id_parameters) containing the - identification information of the server. - - """ + identification information of the server.""" return (self.__target_names, self.__id_parameters) def close_rpc(self): """Closes the connection to the RPC server. No further method calls should be done after this method is called. - """ self.__socket.close() @@ -159,6 +148,8 @@ class AsyncioClient: All RPC methods are coroutines. + Concurrent access from different asyncio tasks is supported; all calls + use a single lock. """ def __init__(self): self.__lock = asyncio.Lock() @@ -171,9 +162,7 @@ class AsyncioClient: def connect_rpc(self, host, port, target_name): """Connects to the server. This cannot be done in __init__ because this method is a coroutine. See ``Client`` for a description of the - parameters. - - """ + parameters.""" self.__reader, self.__writer = \ yield from asyncio.open_connection(host, port) try: @@ -190,7 +179,6 @@ class AsyncioClient: def select_rpc_target(self, target_name): """Selects a RPC target by name. This function should be called exactly once if the connection was created with ``target_name=None``. - """ if target_name not in self.__target_names: raise IncompatibleServer @@ -198,16 +186,13 @@ class AsyncioClient: def get_rpc_id(self): """Returns a tuple (target_names, id_parameters) containing the - identification information of the server. - - """ + identification information of the server.""" return (self.__target_names, self.__id_parameters) def close_rpc(self): """Closes the connection to the RPC server. No further method calls should be done after this method is called. - """ self.__writer.close() self.__reader = None @@ -261,7 +246,6 @@ class BestEffortClient: connection attempt at object initialization. :param retry: Amount of time to wait between retries when reconnecting in the background. - """ def __init__(self, host, port, target_name, firstcon_timeout=0.5, retry=5.0): @@ -322,7 +306,6 @@ class BestEffortClient: """Closes the connection to the RPC server. No further method calls should be done after this method is called. - """ if self.__conretry_thread is None: if self.__socket is not None: @@ -415,7 +398,6 @@ class Server(_AsyncioServer): Clients select one of these objects using its name upon connection. :param id_parameters: An optional human-readable string giving more information about the parameters of the server. - """ def __init__(self, targets, id_parameters=None): _AsyncioServer.__init__(self) @@ -485,7 +467,6 @@ def simple_server_loop(targets, host, port, id_parameters=None): """Runs a server until an exception is raised (e.g. the user hits Ctrl-C). See ``Server`` for a description of the parameters. - """ loop = asyncio.get_event_loop() try: From 5b62b2452dcde9c077c41b714ebf1f691edf12bc Mon Sep 17 00:00:00 2001 From: Sebastien Bourdeauducq Date: Sat, 25 Jul 2015 00:32:18 +0800 Subject: [PATCH 02/20] gui: get spinboxes to behave --- artiq/gui/explorer.py | 4 ++-- artiq/gui/scan.py | 10 ++++++---- artiq/gui/tools.py | 8 ++++++++ 3 files changed, 16 insertions(+), 6 deletions(-) diff --git a/artiq/gui/explorer.py b/artiq/gui/explorer.py index 1d82b2a99..6e9a81334 100644 --- a/artiq/gui/explorer.py +++ b/artiq/gui/explorer.py @@ -7,7 +7,7 @@ from pyqtgraph import LayoutWidget from artiq.protocols.sync_struct import Subscriber from artiq.protocols import pyon -from artiq.gui.tools import DictSyncModel +from artiq.gui.tools import DictSyncModel, force_spinbox_value from artiq.gui.scan import ScanController @@ -73,7 +73,7 @@ class _NumberEntry(QtGui.QDoubleSpinBox): if procdesc["unit"]: self.setSuffix(" " + procdesc["unit"]) if "default" in procdesc: - self.setValue(procdesc["default"]) + force_spinbox_value(self, procdesc["default"]) def get_argument_value(self): return self.value() diff --git a/artiq/gui/scan.py b/artiq/gui/scan.py index 1ec7c2dcf..2edafa579 100644 --- a/artiq/gui/scan.py +++ b/artiq/gui/scan.py @@ -1,6 +1,8 @@ from quamash import QtGui from pyqtgraph import LayoutWidget +from artiq.gui.tools import force_spinbox_value + class _Range(LayoutWidget): def __init__(self, global_min, global_max, global_step, unit): @@ -33,9 +35,9 @@ class _Range(LayoutWidget): self.addWidget(self.npoints, 0, 5) def set_values(self, min, max, npoints): - self.min.setValue(min) - self.max.setValue(max) - self.npoints.setValue(npoints) + force_spinbox_value(self.min, min) + force_spinbox_value(self.max, max) + force_spinbox_value(self.npoints, npoints) def get_values(self): return { @@ -97,7 +99,7 @@ class ScanController(LayoutWidget): d = procdesc["default"] if d["ty"] == "NoScan": self.noscan.setChecked(True) - self.v_noscan.setValue(d["value"]) + force_spinbox_value(self.v_noscan, d["value"]) elif d["ty"] == "LinearScan": self.linear.setChecked(True) self.v_linear.set_values(d["min"], d["max"], d["step"]) diff --git a/artiq/gui/tools.py b/artiq/gui/tools.py index cab4e256d..536790549 100644 --- a/artiq/gui/tools.py +++ b/artiq/gui/tools.py @@ -1,6 +1,14 @@ from quamash import QtCore +def force_spinbox_value(spinbox, value): + if spinbox.minimum() > value: + spinbox.setMinimum(value) + if spinbox.maximum() < value: + spinbox.setMaximum(value) + spinbox.setValue(value) + + def short_format(v): t = type(v) if t is int or t is float: From 928775f6ac2daaca8e39feb07d46aa1c0370091d Mon Sep 17 00:00:00 2001 From: Sebastien Bourdeauducq Date: Sat, 25 Jul 2015 00:35:21 +0800 Subject: [PATCH 03/20] gui: fix default LinearScan/RandomScan --- artiq/gui/scan.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/artiq/gui/scan.py b/artiq/gui/scan.py index 2edafa579..8c319a925 100644 --- a/artiq/gui/scan.py +++ b/artiq/gui/scan.py @@ -102,10 +102,10 @@ class ScanController(LayoutWidget): force_spinbox_value(self.v_noscan, d["value"]) elif d["ty"] == "LinearScan": self.linear.setChecked(True) - self.v_linear.set_values(d["min"], d["max"], d["step"]) + self.v_linear.set_values(d["min"], d["max"], d["npoints"]) elif d["ty"] == "RandomScan": self.random.setChecked(True) - self.v_random.set_values(d["min"], d["max"], d["step"]) + self.v_random.set_values(d["min"], d["max"], d["npoints"]) elif d["ty"] == "ExplicitScan": self.explicit.setChecked(True) self.v_explicit.insert(" ".join( From f210e0dcd699aa51776df1746e913944277fa12e Mon Sep 17 00:00:00 2001 From: Sebastien Bourdeauducq Date: Sat, 25 Jul 2015 00:35:48 +0800 Subject: [PATCH 04/20] examples/flopping_f: use Scannable and NumberValue --- examples/master/repository/flopping_f_simulation.py | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/examples/master/repository/flopping_f_simulation.py b/examples/master/repository/flopping_f_simulation.py index 19a5127d4..5efe1d8e2 100644 --- a/examples/master/repository/flopping_f_simulation.py +++ b/examples/master/repository/flopping_f_simulation.py @@ -27,12 +27,11 @@ class FloppingF(EnvExperiment): """Flopping F simulation""" def build(self): - self.attr_argument("npoints", FreeValue(100)) - self.attr_argument("min_freq", FreeValue(1000)) - self.attr_argument("max_freq", FreeValue(2000)) + self.attr_argument("frequency_scan", Scannable( + default=LinearScan(1000, 2000, 100))) - self.attr_argument("F0", FreeValue(1500)) - self.attr_argument("noise_amplitude", FreeValue(0.1)) + self.attr_argument("F0", NumberValue(1500, min=1000, max=2000)) + self.attr_argument("noise_amplitude", NumberValue(0.1, min=0, max=100)) self.frequency = self.set_result("flopping_f_frequency", [], True) self.brightness = self.set_result("flopping_f_brightness", [], True) @@ -40,8 +39,7 @@ class FloppingF(EnvExperiment): self.attr_device("scheduler") def run(self): - for i in range(self.npoints): - frequency = (self.max_freq-self.min_freq)*i/(self.npoints - 1) + self.min_freq + for frequency in self.frequency_scan: brightness = model(frequency, self.F0) + self.noise_amplitude*random.random() self.frequency.append(frequency) self.brightness.append(brightness) From ef8b09d9bcddc8c7be149a311fc7a5f44046ff52 Mon Sep 17 00:00:00 2001 From: Sebastien Bourdeauducq Date: Sat, 25 Jul 2015 00:36:16 +0800 Subject: [PATCH 05/20] gui: add console --- artiq/frontend/artiq_gui.py | 15 ++++++++++++++- artiq/gui/console.py | 21 +++++++++++++++++++++ artiq/gui/parameters.py | 13 ++++++++----- artiq/gui/results.py | 3 +++ 4 files changed, 46 insertions(+), 6 deletions(-) create mode 100644 artiq/gui/console.py diff --git a/artiq/frontend/artiq_gui.py b/artiq/frontend/artiq_gui.py index f1f63bab4..ad1a19729 100755 --- a/artiq/frontend/artiq_gui.py +++ b/artiq/frontend/artiq_gui.py @@ -18,6 +18,7 @@ from artiq.gui.results import ResultsDock from artiq.gui.parameters import ParametersDock from artiq.gui.schedule import ScheduleDock from artiq.gui.log import LogDock +from artiq.gui.console import ConsoleDock data_dir = os.path.join(os.path.abspath(os.path.dirname(__file__)), @@ -109,7 +110,19 @@ def main(): args.server, args.port_notify)) atexit.register(lambda: loop.run_until_complete(d_log.sub_close())) - area.addDock(d_log, "bottom") + pdb = AsyncioClient() + loop.run_until_complete(pdb.connect_rpc( + args.server, args.port_control, "master_pdb")) + atexit.register(lambda: pdb.close_rpc()) + def _get_parameter(k, v): + asyncio.async(pdb.set(k, v)) + d_console = ConsoleDock( + d_params.get_parameter, + _get_parameter, + d_results.get_result) + + area.addDock(d_console, "bottom") + area.addDock(d_log, "above", d_console) area.addDock(d_schedule, "above", d_log) win.show() diff --git a/artiq/gui/console.py b/artiq/gui/console.py new file mode 100644 index 000000000..847a6eac9 --- /dev/null +++ b/artiq/gui/console.py @@ -0,0 +1,21 @@ +from pyqtgraph import console, dockarea + + +_help = """ +The following functions are available: + get_parameter(key) + set_parameter(key, value) [asynchronous update] + get_result(key) [real-time results only] + +""" + +class ConsoleDock(dockarea.Dock): + def __init__(self, get_parameter, set_parameter, get_result): + dockarea.Dock.__init__(self, "Console", size=(1000, 300)) + ns = { + "get_parameter": get_parameter, + "set_parameter": set_parameter, + "get_result": get_result + } + c = console.ConsoleWidget(namespace=ns, text=_help) + self.addWidget(c) diff --git a/artiq/gui/parameters.py b/artiq/gui/parameters.py index 5fbac5866..22f2addbd 100644 --- a/artiq/gui/parameters.py +++ b/artiq/gui/parameters.py @@ -34,7 +34,7 @@ class ParametersDock(dockarea.Dock): self.search = QtGui.QLineEdit() self.search.setPlaceholderText("search...") - self.search.editingFinished.connect(self.search_parameters) + self.search.editingFinished.connect(self._search_parameters) grid.addWidget(self.search, 0, 0) self.table = QtGui.QTableView() @@ -43,7 +43,10 @@ class ParametersDock(dockarea.Dock): QtGui.QHeaderView.ResizeToContents) grid.addWidget(self.table, 1, 0) - def search_parameters(self): + def get_parameter(self, key): + return self.table_model.backing_store[key] + + def _search_parameters(self): model = self.table.model() parentIndex = model.index(0, 0) numRows = model.rowCount(parentIndex) @@ -66,6 +69,6 @@ class ParametersDock(dockarea.Dock): yield from self.subscriber.close() def init_parameters_model(self, init): - table_model = ParametersModel(self.table, init) - self.table.setModel(table_model) - return table_model + self.table_model = ParametersModel(self.table, init) + self.table.setModel(self.table_model) + return self.table_model diff --git a/artiq/gui/results.py b/artiq/gui/results.py index 964856744..c7f47214f 100644 --- a/artiq/gui/results.py +++ b/artiq/gui/results.py @@ -55,6 +55,9 @@ class ResultsDock(dockarea.Dock): self.displays = dict() + def get_result(self, key): + return self.table_model.backing_store[key] + @asyncio.coroutine def sub_connect(self, host, port): self.subscriber = Subscriber("rt_results", self.init_results_model, From 5979f85c1c7851805eb0b9e827d4b05d3afe8fbd Mon Sep 17 00:00:00 2001 From: Sebastien Bourdeauducq Date: Sat, 25 Jul 2015 11:38:26 +0800 Subject: [PATCH 06/20] gui: use monospace font in log --- artiq/gui/log.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/artiq/gui/log.py b/artiq/gui/log.py index 7a0fb4894..8c7a99c24 100644 --- a/artiq/gui/log.py +++ b/artiq/gui/log.py @@ -12,6 +12,15 @@ class _LogModel(ListSyncModel): ListSyncModel.__init__(self, ["RID", "Message"], parent, init) + self.fixed_font = QtGui.QFont() + self.fixed_font.setFamily("Monospace") + + def data(self, index, role): + if (role == QtCore.Qt.FontRole and index.isValid() + and index.column() == 1): + return self.fixed_font + else: + return ListSyncModel.data(self, index, role) def convert(self, v, column): return v[column] From 9fe65769f2a7a1326b9669e17ad7cf97e676f695 Mon Sep 17 00:00:00 2001 From: Sebastien Bourdeauducq Date: Sat, 25 Jul 2015 12:01:47 +0800 Subject: [PATCH 07/20] gui: add console description --- artiq/gui/console.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/artiq/gui/console.py b/artiq/gui/console.py index 847a6eac9..38015ef29 100644 --- a/artiq/gui/console.py +++ b/artiq/gui/console.py @@ -2,6 +2,8 @@ from pyqtgraph import console, dockarea _help = """ +This is an interactive Python console. + The following functions are available: get_parameter(key) set_parameter(key, value) [asynchronous update] From 8bc1dd9f9cdd846d4a21b9329a891d1236a6dc0a Mon Sep 17 00:00:00 2001 From: Sebastien Bourdeauducq Date: Sat, 25 Jul 2015 12:23:43 +0800 Subject: [PATCH 08/20] test/serialization: remove redundant test since Quantity was removed --- artiq/test/serialization.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/artiq/test/serialization.py b/artiq/test/serialization.py index 9587a7a77..27e08bf30 100644 --- a/artiq/test/serialization.py +++ b/artiq/test/serialization.py @@ -4,14 +4,12 @@ from fractions import Fraction import numpy as np -from artiq.language.units import * from artiq.protocols import pyon _pyon_test_object = { (1, 2): [(3, 4.2), (2, )], Fraction(3, 4): np.linspace(5, 10, 1), - "frequency": 10*GHz } From 05dd11a60d214c74645fac73a775face9d741df2 Mon Sep 17 00:00:00 2001 From: Sebastien Bourdeauducq Date: Sat, 25 Jul 2015 12:28:41 +0800 Subject: [PATCH 09/20] protocols/pyon: support numpy scalars (closes #53) --- artiq/protocols/pyon.py | 24 +++++++++++++++++++++++- artiq/test/serialization.py | 3 +++ 2 files changed, 26 insertions(+), 1 deletion(-) diff --git a/artiq/protocols/pyon.py b/artiq/protocols/pyon.py index 3998d8bc8..13161094e 100644 --- a/artiq/protocols/pyon.py +++ b/artiq/protocols/pyon.py @@ -39,6 +39,16 @@ _encode_map = { numpy.ndarray: "nparray" } +_numpy_scalar = { + "int8", "int16", "int32", "int64", "uint8", "uint16", "uint32", "uint64", + "float16", "float32", "float64" +} + + +for _t in _numpy_scalar: + _encode_map[getattr(numpy, _t)] = "npscalar" + + class _Encoder: def __init__(self, pretty): self.pretty = pretty @@ -114,6 +124,13 @@ class _Encoder: r += ")" return r + def encode_npscalar(self, x): + r = "npscalar(" + r += "\"" + type(x).__name__ + "\", " + r += encode(base64.b64encode(x).decode()) + r += ")" + return r + def encode(self, x): return getattr(self, "encode_" + _encode_map[type(x)])(x) @@ -131,6 +148,10 @@ def _nparray(shape, dtype, data): return a.reshape(shape) +def _npscalar(ty, data): + return numpy.frombuffer(base64.b64decode(data), dtype=ty)[0] + + _eval_dict = { "__builtins__": None, @@ -139,7 +160,8 @@ _eval_dict = { "true": True, "Fraction": Fraction, - "nparray": _nparray + "nparray": _nparray, + "npscalar": _npscalar } def decode(s): diff --git a/artiq/test/serialization.py b/artiq/test/serialization.py index 27e08bf30..7bcbed5eb 100644 --- a/artiq/test/serialization.py +++ b/artiq/test/serialization.py @@ -10,6 +10,9 @@ from artiq.protocols import pyon _pyon_test_object = { (1, 2): [(3, 4.2), (2, )], Fraction(3, 4): np.linspace(5, 10, 1), + "a": np.int8(9), "b": np.int16(-98), "c": np.int32(42), "d": np.int64(-5), + "e": np.uint8(8), "f": np.uint16(5), "g": np.uint32(4), "h": np.uint64(9), + "x": np.float16(9.0), "y": np.float32(9.0), "z": np.float64(9.0), } From 61f45f505b1fa5a144965f30765a4a230357dbf9 Mon Sep 17 00:00:00 2001 From: Sebastien Bourdeauducq Date: Sat, 25 Jul 2015 14:37:19 +0800 Subject: [PATCH 10/20] gui/short_format: show string values --- artiq/gui/tools.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/artiq/gui/tools.py b/artiq/gui/tools.py index 536790549..f388521d8 100644 --- a/artiq/gui/tools.py +++ b/artiq/gui/tools.py @@ -13,6 +13,11 @@ def short_format(v): t = type(v) if t is int or t is float: return str(v) + elif t is str: + if len(v) < 15: + return "\"" + v + "\"" + else: + return "\"" + v[:12] + "\"..." else: r = t.__name__ if t is list or t is dict or t is set: From 696bceb406f683e60947d331b589492c83f4b710 Mon Sep 17 00:00:00 2001 From: Sebastien Bourdeauducq Date: Sat, 25 Jul 2015 15:08:27 +0800 Subject: [PATCH 11/20] gui: feedback on run deletion --- artiq/frontend/artiq_gui.py | 2 +- artiq/gui/schedule.py | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/artiq/frontend/artiq_gui.py b/artiq/frontend/artiq_gui.py index ad1a19729..fb6700377 100755 --- a/artiq/frontend/artiq_gui.py +++ b/artiq/frontend/artiq_gui.py @@ -100,7 +100,7 @@ def main(): area.addDock(d_params, "above", d_results) area.addDock(d_explorer, "above", d_params) - d_schedule = ScheduleDock(schedule_ctl) + d_schedule = ScheduleDock(status_bar, schedule_ctl) loop.run_until_complete(d_schedule.sub_connect( args.server, args.port_notify)) atexit.register(lambda: loop.run_until_complete(d_schedule.sub_close())) diff --git a/artiq/gui/schedule.py b/artiq/gui/schedule.py index d023c880b..65bcdc0cb 100644 --- a/artiq/gui/schedule.py +++ b/artiq/gui/schedule.py @@ -46,9 +46,10 @@ class _ScheduleModel(DictSyncModel): class ScheduleDock(dockarea.Dock): - def __init__(self, schedule_ctl): + def __init__(self, status_bar, schedule_ctl): dockarea.Dock.__init__(self, "Schedule", size=(1000, 300)) + self.status_bar = status_bar self.schedule_ctl = schedule_ctl self.table = QtGui.QTableView() @@ -86,4 +87,5 @@ class ScheduleDock(dockarea.Dock): if idx: row = idx[0].row() rid = self.table_model.row_to_key[row] + self.status_bar.showMessage("Deleted RID {}".format(rid)) asyncio.async(self.delete(rid)) From 3c6a4b81a312c6750f9e7676e4d91c401eb6556c Mon Sep 17 00:00:00 2001 From: Sebastien Bourdeauducq Date: Sat, 25 Jul 2015 15:37:02 +0800 Subject: [PATCH 12/20] examples/ddb: use LED channel for KC705 QC1 --- examples/master/ddb.pyon | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/master/ddb.pyon b/examples/master/ddb.pyon index ff945141a..e853675c5 100644 --- a/examples/master/ddb.pyon +++ b/examples/master/ddb.pyon @@ -53,7 +53,7 @@ "type": "local", "module": "artiq.coredevice.ttl", "class": "TTLOut", - "arguments": {"channel": 18} + "arguments": {"channel": 17} }, "dds_bus": { From d14a31f443f059df55e8683306376a96a3c56da4 Mon Sep 17 00:00:00 2001 From: Sebastien Bourdeauducq Date: Sat, 25 Jul 2015 15:52:38 +0800 Subject: [PATCH 13/20] artiq_run: fix ELF running --- artiq/frontend/artiq_run.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/artiq/frontend/artiq_run.py b/artiq/frontend/artiq_run.py index baace1d0e..06595746c 100755 --- a/artiq/frontend/artiq_run.py +++ b/artiq/frontend/artiq_run.py @@ -26,7 +26,7 @@ class ELFRunner(EnvExperiment): def run(self): with open(self.file, "rb") as f: self.core.comm.load(f.read()) - self.core.comm.run("run", False) + self.core.comm.run("run") self.core.comm.serve(dict(), dict()) From aba2d3f112ffbd0bfea1657ed535a95c05fc57a7 Mon Sep 17 00:00:00 2001 From: Sebastien Bourdeauducq Date: Sat, 25 Jul 2015 16:26:04 +0800 Subject: [PATCH 14/20] runtime: process essential kernel CPU messages at all time --- soc/runtime/kloader.c | 63 +++++++++++++++++++++++++++++++++++++++++++ soc/runtime/kloader.h | 6 +++++ soc/runtime/main.c | 5 +++- soc/runtime/session.c | 47 +++++--------------------------- 4 files changed, 80 insertions(+), 41 deletions(-) diff --git a/soc/runtime/kloader.c b/soc/runtime/kloader.c index f73723981..0cdf56634 100644 --- a/soc/runtime/kloader.c +++ b/soc/runtime/kloader.c @@ -4,6 +4,7 @@ #include "log.h" #include "flash_storage.h" #include "mailbox.h" +#include "messages.h" #include "elf_loader.h" #include "services.h" #include "kloader.h" @@ -122,3 +123,65 @@ void kloader_stop(void) kernel_cpu_reset_write(1); mailbox_acknowledge(); } + +int kloader_validate_kpointer(void *p) +{ + unsigned int v = (unsigned int)p; + if((v < 0x40400000) || (v > (0x4fffffff - 1024*1024))) { + log("Received invalid pointer from kernel CPU: 0x%08x", v); + return 0; + } + return 1; +} + +int kloader_is_essential_kmsg(int msgtype) +{ + switch(msgtype) { + case MESSAGE_TYPE_NOW_INIT_REQUEST: + case MESSAGE_TYPE_NOW_SAVE: + case MESSAGE_TYPE_LOG: + return 1; + default: + return 0; + } +} + +long long int now; + +void kloader_service_essential_kmsg(void) +{ + struct msg_base *umsg; + + umsg = mailbox_receive(); + if(umsg) { + if(!kloader_validate_kpointer(umsg)) + return; + switch(umsg->type) { + case MESSAGE_TYPE_NOW_INIT_REQUEST: { + struct msg_now_init_reply reply; + + reply.type = MESSAGE_TYPE_NOW_INIT_REPLY; + reply.now = now; + mailbox_send_and_wait(&reply); + break; + } + case MESSAGE_TYPE_NOW_SAVE: { + struct msg_now_save *msg = (struct msg_now_save *)umsg; + + now = msg->now; + mailbox_acknowledge(); + break; + } + case MESSAGE_TYPE_LOG: { + struct msg_log *msg = (struct msg_log *)umsg; + + log_va(msg->fmt, msg->args); + mailbox_acknowledge(); + break; + } + default: + /* handled elsewhere */ + break; + } + } +} diff --git a/soc/runtime/kloader.h b/soc/runtime/kloader.h index bf484b4e1..098a740b7 100644 --- a/soc/runtime/kloader.h +++ b/soc/runtime/kloader.h @@ -4,6 +4,8 @@ #define KERNELCPU_EXEC_ADDRESS 0x40400000 #define KERNELCPU_PAYLOAD_ADDRESS 0x40404000 +extern long long int now; + typedef void (*kernel_function)(void); int kloader_load(void *buffer, int length); @@ -14,4 +16,8 @@ void kloader_start_idle_kernel(void); void kloader_start_user_kernel(kernel_function k); void kloader_stop(void); +int kloader_validate_kpointer(void *p); +int kloader_is_essential_kmsg(int msgtype); +void kloader_service_essential_kmsg(void); + #endif /* __KLOADER_H */ diff --git a/soc/runtime/main.c b/soc/runtime/main.c index 4feb8c9a0..306f26b20 100644 --- a/soc/runtime/main.c +++ b/soc/runtime/main.c @@ -143,6 +143,7 @@ static void regular_main(void) session_end(); while(1) { lwip_service(); + kloader_service_essential_kmsg(); kserver_service(); } } @@ -201,8 +202,10 @@ static void regular_main(void) /* Open the session for the serial control. */ session_start(); - while(1) + while(1) { + kloader_service_essential_kmsg(); serial_service(); + } } #endif diff --git a/soc/runtime/session.c b/soc/runtime/session.c index 1a7f5d879..9b3591e30 100644 --- a/soc/runtime/session.c +++ b/soc/runtime/session.c @@ -11,8 +11,8 @@ #include "log.h" #include "kloader.h" #include "exceptions.h" -#include "session.h" #include "flash_storage.h" +#include "session.h" #define BUFFER_IN_SIZE (1024*1024) #define BUFFER_OUT_SIZE (1024*1024) @@ -55,7 +55,6 @@ static void submit_output(int len) } static int user_kernel_state; -static long long int now; enum { USER_KERNEL_NONE = 0, @@ -153,7 +152,7 @@ static int process_input(void) log("Attempted to switch RTIO clock while kernel running"); buffer_out[8] = REMOTEMSG_TYPE_CLOCK_SWITCH_FAILED; submit_output(9); - break; + break; } rtio_crg_clock_sel_write(buffer_in[9]); buffer_out[8] = REMOTEMSG_TYPE_CLOCK_SWITCH_COMPLETED; @@ -421,16 +420,6 @@ static int add_rpc_value(int bi, int type_tag, void *value) return bi - obi; } -static int validate_kpointer(void *p) -{ - unsigned int v = (unsigned int)p; - if((v < 0x40400000) || (v > (0x4fffffff - 1024*1024))) { - log("Received invalid pointer from kernel CPU: 0x%08x", v); - return 0; - } - return 1; -} - static int send_rpc_request(int rpc_num, va_list args) { int r; @@ -448,7 +437,7 @@ static int send_rpc_request(int rpc_num, va_list args) v = NULL; else { v = va_arg(args, void *); - if(!validate_kpointer(v)) + if(!kloader_validate_kpointer(v)) return 0; } r = add_rpc_value(bi, type_tag, v); @@ -467,31 +456,16 @@ static int send_rpc_request(int rpc_num, va_list args) /* assumes output buffer is empty when called */ static int process_kmsg(struct msg_base *umsg) { - if(!validate_kpointer(umsg)) + if(!kloader_validate_kpointer(umsg)) return 0; - if((user_kernel_state != USER_KERNEL_RUNNING) - && (umsg->type != MESSAGE_TYPE_NOW_INIT_REQUEST) - && (umsg->type != MESSAGE_TYPE_NOW_SAVE)) { + if(kloader_is_essential_kmsg(umsg->type)) + return 1; /* handled elsewhere */ + if(user_kernel_state != USER_KERNEL_RUNNING) { log("Received unexpected message from kernel CPU while not in running state"); return 0; } switch(umsg->type) { - case MESSAGE_TYPE_NOW_INIT_REQUEST: { - struct msg_now_init_reply reply; - - reply.type = MESSAGE_TYPE_NOW_INIT_REPLY; - reply.now = now; - mailbox_send_and_wait(&reply); - break; - } - case MESSAGE_TYPE_NOW_SAVE: { - struct msg_now_save *msg = (struct msg_now_save *)umsg; - - now = msg->now; - mailbox_acknowledge(); - break; - } case MESSAGE_TYPE_FINISHED: buffer_out[8] = REMOTEMSG_TYPE_KERNEL_FINISHED; submit_output(9); @@ -538,13 +512,6 @@ static int process_kmsg(struct msg_base *umsg) mailbox_acknowledge(); break; } - case MESSAGE_TYPE_LOG: { - struct msg_log *msg = (struct msg_log *)umsg; - - log_va(msg->fmt, msg->args); - mailbox_acknowledge(); - break; - } default: { log("Received invalid message type from kernel CPU"); return 0; From e5acdfe3fdb4c02d88c5b98946827991055d890e Mon Sep 17 00:00:00 2001 From: whitequark Date: Sun, 26 Jul 2015 09:28:11 +0300 Subject: [PATCH 15/20] Update manual install instructions. --- doc/manual/installing.rst | 55 ++++++++++++++++++--------------------- 1 file changed, 25 insertions(+), 30 deletions(-) diff --git a/doc/manual/installing.rst b/doc/manual/installing.rst index dec7eab80..84243811e 100644 --- a/doc/manual/installing.rst +++ b/doc/manual/installing.rst @@ -99,31 +99,40 @@ These steps are required to generate bitstream (``.bit``) files, build the MiSoC .. note:: The options ``develop`` and ``--user`` are for setup.py to install Migen in ``~/.local/lib/python3.4``. -* Install OpenRISC GCC/binutils toolchain (or1k-elf-...): :: +* Install OpenRISC binutils (or1k-linux-...): :: $ cd ~/artiq-dev - $ git clone https://github.com/openrisc/or1k-src - $ cd or1k-src - $ mkdir build - $ cd build - $ ../configure --target=or1k-elf --enable-shared --disable-itcl \ - --disable-tk --disable-tcl --disable-winsup \ - --disable-gdbtk --disable-libgui --disable-rda \ - --disable-sid --disable-sim --disable-gdb \ - --disable-newlib --disable-libgloss --disable-werror + $ wget https://ftp.gnu.org/gnu/binutils/binutils-2.25.1.tar.bz2 + $ tar xvf binutils-2.25.1.tar.bz2 + $ rm binutils-2.25.1.tar.bz2 + + $ mkdir binutils-2.25.1/build + $ cd binutils-2.25.1/build + $ ../configure --target=or1k-linux --prefix=/usr/local $ make -j4 $ sudo make install +.. note:: + We're using an ``or1k-linux`` target because it is necessary to enable + shared library support in ``ld``, not because Linux is involved. + +* Install LLVM and Clang: :: + $ cd ~/artiq-dev - $ git clone https://github.com/openrisc/or1k-gcc - $ cd or1k-gcc + $ git clone https://github.com/openrisc/llvm-or1k + $ cd llvm-or1k/tools + $ git clone https://github.com/openrisc/clang-or1k clang + $ cd .. + $ mkdir build $ cd build - $ ../configure --target=or1k-elf --enable-languages=c \ - --disable-shared --disable-libssp + $ cmake .. -DCMAKE_INSTALL_PREFIX=/usr/local/llvm-or1k -DLLVM_TARGETS_TO_BUILD="OR1K;X86" -DCMAKE_BUILD_TYPE=Rel -DLLVM_ENABLE_ASSERTIONS=ON $ make -j4 $ sudo make install +.. note:: + Compilation of LLVM can take more than 30 min on some machines. + .. _install-xc3sprog: * Install JTAG tools needed to program the Pipistrello and KC705: @@ -181,6 +190,7 @@ These steps are required to generate bitstream (``.bit``) files, build the MiSoC :: $ cd ~/artiq-dev/misoc + $ export PATH=$PATH:/usr/local/llvm-or1k/bin * For Pipistrello:: @@ -279,19 +289,7 @@ To flash the ``idle`` kernel: Installing the host-side software ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -* Install LLVM and the llvmlite Python bindings: :: - - $ cd ~/artiq-dev - $ git clone https://github.com/openrisc/llvm-or1k - $ cd llvm-or1k/tools - $ git clone https://github.com/openrisc/clang-or1k clang - - $ cd .. - $ mkdir build - $ cd build - $ cmake .. -DCMAKE_INSTALL_PREFIX=/usr/local/llvm-or1k -DLLVM_TARGETS_TO_BUILD="OR1K;X86" -DCMAKE_BUILD_TYPE=Debug - $ make -j4 - $ sudo make install +* Install the llvmlite Python bindings: :: $ cd ~/artiq-dev $ git clone https://github.com/numba/llvmlite @@ -304,9 +302,6 @@ Installing the host-side software .. note:: llvmlite is in development and its API is not stable yet. Commit ID ``11a8303d02e3d6dd2d1e0e9065701795cd8a979f`` is known to work. -.. note:: - Compilation of LLVM can take more than 30 min on some machines. - * Install ARTIQ: :: $ cd ~/artiq-dev From 95f7be0a8871385a6a8d03db2114696e732f1fc0 Mon Sep 17 00:00:00 2001 From: whitequark Date: Sun, 26 Jul 2015 09:38:43 +0300 Subject: [PATCH 16/20] Update install instructions to reflect that LLVM is always needed. --- doc/manual/installing.rst | 53 ++++++++++++++++++++++----------------- 1 file changed, 30 insertions(+), 23 deletions(-) diff --git a/doc/manual/installing.rst b/doc/manual/installing.rst index 84243811e..ff74515ba 100644 --- a/doc/manual/installing.rst +++ b/doc/manual/installing.rst @@ -70,35 +70,19 @@ Next step (for KC705) is to flash MAC and IP addresses to the board: Installing from source ---------------------- -You can skip this if you already installed from conda. +You can skip the first two steps if you already installed from conda. -Preparing the core device FPGA board -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +Preparing the build environment for the core device +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -These steps are required to generate bitstream (``.bit``) files, build the MiSoC BIOS and ARTIQ runtime, and flash FPGA boards. If the board is already flashed, you may skip those steps and go directly to `Installing the host-side software`. - -* Install the FPGA vendor tools (e.g. Xilinx ISE and/or Vivado): - - * Get Xilinx tools from http://www.xilinx.com/support/download/index.htm. ISE can build bitstreams both for boards using the Spartan-6 (Pipistrello) and 7-series devices (KC705), while Vivado supports only boards using 7-series devices. - - * The Pipistrello is supported by Webpack, the KC705 is not. - - * During the Xilinx toolchain installation, uncheck ``Install cable drivers`` (they are not required as we use better and open source alternatives). +These steps are required to generate code that can run on the core +device. They are necessary both for building the MiSoC BIOS +and the ARTIQ kernels. * Create a development directory: :: $ mkdir ~/artiq-dev -* Install Migen: :: - - $ cd ~/artiq-dev - $ git clone https://github.com/m-labs/migen - $ cd migen - $ python3 setup.py develop --user - -.. note:: - The options ``develop`` and ``--user`` are for setup.py to install Migen in ``~/.local/lib/python3.4``. - * Install OpenRISC binutils (or1k-linux-...): :: $ cd ~/artiq-dev @@ -133,6 +117,29 @@ These steps are required to generate bitstream (``.bit``) files, build the MiSoC .. note:: Compilation of LLVM can take more than 30 min on some machines. +Preparing the core device FPGA board +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +These steps are required to generate bitstream (``.bit``) files, build the MiSoC BIOS and ARTIQ runtime, and flash FPGA boards. If the board is already flashed, you may skip those steps and go directly to `Installing the host-side software`. + +* Install the FPGA vendor tools (e.g. Xilinx ISE and/or Vivado): + + * Get Xilinx tools from http://www.xilinx.com/support/download/index.htm. ISE can build bitstreams both for boards using the Spartan-6 (Pipistrello) and 7-series devices (KC705), while Vivado supports only boards using 7-series devices. + + * The Pipistrello is supported by Webpack, the KC705 is not. + + * During the Xilinx toolchain installation, uncheck ``Install cable drivers`` (they are not required as we use better and open source alternatives). + +* Install Migen: :: + + $ cd ~/artiq-dev + $ git clone https://github.com/m-labs/migen + $ cd migen + $ python3 setup.py develop --user + +.. note:: + The options ``develop`` and ``--user`` are for setup.py to install Migen in ``~/.local/lib/python3.4``. + .. _install-xc3sprog: * Install JTAG tools needed to program the Pipistrello and KC705: @@ -297,7 +304,7 @@ Installing the host-side software $ patch -p1 < ~/artiq-dev/artiq/misc/llvmlite-add-all-targets.patch $ patch -p1 < ~/artiq-dev/artiq/misc/llvmlite-rename.patch $ patch -p1 < ~/artiq-dev/artiq/misc/llvmlite-build-as-debug-on-windows.patch - $ PATH=/usr/local/llvm-or1k/bin:$PATH sudo -E python3 setup.py install + $ LLVM_CONFIG=/usr/local/llvm-or1k/bin/llvm-config python3 setup.py install --user .. note:: llvmlite is in development and its API is not stable yet. Commit ID ``11a8303d02e3d6dd2d1e0e9065701795cd8a979f`` is known to work. From 163edc02c6373dbb0335b02366b843714a70f671 Mon Sep 17 00:00:00 2001 From: whitequark Date: Sun, 26 Jul 2015 09:49:39 +0300 Subject: [PATCH 17/20] Fix Mock usage in sphinx configuration. --- doc/manual/conf.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/doc/manual/conf.py b/doc/manual/conf.py index 2df182943..ffb4f02f8 100644 --- a/doc/manual/conf.py +++ b/doc/manual/conf.py @@ -22,7 +22,9 @@ from unittest.mock import MagicMock class Mock(MagicMock): @classmethod def __getattr__(cls, name): - return Mock() + if name == "_mock_methods": + return None + return Mock() mock_modules = ["artiq.gui.moninj", "quamash", "pyqtgraph", "matplotlib"] From cf16da576306aa90033642be15f84f08966c125b Mon Sep 17 00:00:00 2001 From: whitequark Date: Sun, 26 Jul 2015 09:55:18 +0300 Subject: [PATCH 18/20] Add a setuptools command to rsync the documentation. --- setup.py | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index 8a8e3f476..e3ca507f6 100755 --- a/setup.py +++ b/setup.py @@ -1,12 +1,22 @@ #!/usr/bin/env python3 -from setuptools import setup, find_packages +from setuptools import setup, find_packages, Command import sys import os if sys.version_info[:3] < (3, 4, 3): raise Exception("You need at least Python 3.4.3 to run ARTIQ") +class PushDocCommand(Command): + description = "uploads the documentation to m-labs.hk" + user_options = [] + def initialize_options(self): + pass + def finalize_options(self): + pass + def run(self): + os.system("rsync -avz doc/manual/_build/html/ shell.serverraum.org:~/web/m-labs.hk/artiq/manual") + requirements = [ "sphinx", "sphinx-argparse", "pyserial", "numpy", "scipy", "python-dateutil", "prettytable", "h5py", "pydaqmx", "pyelftools", @@ -52,5 +62,6 @@ setup( ext_modules=[], entry_points={ "console_scripts": scripts, - } + }, + cmdclass={"push_doc":PushDocCommand} ) From d252a00b563090b4dd5ec3a78a42f3a1e416cf82 Mon Sep 17 00:00:00 2001 From: whitequark Date: Sun, 26 Jul 2015 14:08:34 +0300 Subject: [PATCH 19/20] Add prebuilt or1k-linux binutils and or1k llvm for travis. --- .travis/get-toolchain.sh | 21 +++++++++++++-------- 1 file changed, 13 insertions(+), 8 deletions(-) diff --git a/.travis/get-toolchain.sh b/.travis/get-toolchain.sh index 7f736aa23..361ecc5ee 100755 --- a/.travis/get-toolchain.sh +++ b/.travis/get-toolchain.sh @@ -1,8 +1,7 @@ #!/bin/sh -packages="https://people.phys.ethz.ch/~robertjo/artiq-dev/or1k-gcc_20141105-1_amd64.deb -https://people.phys.ethz.ch/~robertjo/artiq-dev/or1k-binutils_20141105-1_amd64.deb -http://us.archive.ubuntu.com/ubuntu/pool/universe/i/iverilog/iverilog_0.9.7-1_amd64.deb" +packages="http://us.archive.ubuntu.com/ubuntu/pool/universe/i/iverilog/iverilog_0.9.7-1_amd64.deb" +archives="http://fehu.whitequark.org/files/binutils-or1k.tbz2 http://fehu.whitequark.org/files/llvm-or1k.tbz2" mkdir -p packages @@ -13,12 +12,18 @@ do dpkg -x $pkg_name packages done -export PATH=$PWD/packages/usr/local/bin:$PWD/packages/usr/bin:$PATH +for a in $archives +do + wget $a + (cd packages && tar xvf ../$(basename $a)) +done + +export PATH=$PWD/packages/usr/local/llvm-or1k/bin:$PWD/packages/usr/local/bin:$PWD/packages/usr/bin:$PATH export LD_LIBRARY_PATH=$PWD/packages/usr/lib/x86_64-linux-gnu:$PWD/packages/usr/local/x86_64-unknown-linux-gnu/or1k-elf/lib:$LD_LIBRARY_PATH -echo -e "export LD_LIBRARY_PATH=$LD_LIBRARY_PATH" >> $HOME/.mlabs/build_settings.sh +echo "export LD_LIBRARY_PATH=$LD_LIBRARY_PATH" >> $HOME/.mlabs/build_settings.sh +echo "export PATH=$PWD/packages/usr/local/llvm-or1k/bin:$PATH" >> $HOME/.mlabs/build_settings.sh -or1k-elf-as --version -or1k-elf-gcc --version +or1k-linux-as --version +llc --version clang --version -llvm-as --version || true From 1d9f40833de8e92ac5684062c5845cb748f062be Mon Sep 17 00:00:00 2001 From: whitequark Date: Sun, 26 Jul 2015 16:16:48 +0300 Subject: [PATCH 20/20] Update ldscripts with -fPIC support. --- soc/runtime/ksupport.ld | 12 +++++++++++- soc/runtime/linker.ld | 12 +++++++++++- 2 files changed, 22 insertions(+), 2 deletions(-) diff --git a/soc/runtime/ksupport.ld b/soc/runtime/ksupport.ld index b4a35873b..4126b4a7b 100644 --- a/soc/runtime/ksupport.ld +++ b/soc/runtime/ksupport.ld @@ -24,6 +24,17 @@ SECTIONS _etext = .; } > ksupport + .got : + { + _GLOBAL_OFFSET_TABLE_ = .; + *(.got) + } > ksupport + + .got.plt : + { + *(.got.plt) + } > ksupport + .rodata : { . = ALIGN(4); @@ -39,7 +50,6 @@ SECTIONS _fdata = .; *(.data .data.* .gnu.linkonce.d.*) *(.data1) - _gp = ALIGN(16); *(.sdata .sdata.* .gnu.linkonce.s.*) _edata = .; } > ksupport diff --git a/soc/runtime/linker.ld b/soc/runtime/linker.ld index 5fc12b610..d0c3a6558 100644 --- a/soc/runtime/linker.ld +++ b/soc/runtime/linker.ld @@ -26,6 +26,17 @@ SECTIONS _etext = .; } > runtime + .got : + { + _GLOBAL_OFFSET_TABLE_ = .; + *(.got) + } > runtime + + .got.plt : + { + *(.got.plt) + } > runtime + .rodata : { . = ALIGN(4); @@ -41,7 +52,6 @@ SECTIONS _fdata = .; *(.data .data.* .gnu.linkonce.d.*) *(.data1) - _gp = ALIGN(16); *(.sdata .sdata.* .gnu.linkonce.s.*) _edata = .; } > runtime