From fc4791bbbe64416f10422b2db02323e665bf7e2a Mon Sep 17 00:00:00 2001 From: Sebastien Bourdeauducq Date: Tue, 13 Oct 2015 18:11:17 +0800 Subject: [PATCH 01/14] doc: new dataset API --- artiq/language/environment.py | 19 +++++++++++++++++++ doc/manual/environment.rst | 22 ++++++++++++---------- doc/manual/getting_started_core.rst | 2 +- doc/manual/getting_started_mgmt.rst | 2 +- doc/manual/utilities.rst | 4 +--- 5 files changed, 34 insertions(+), 15 deletions(-) diff --git a/artiq/language/environment.py b/artiq/language/environment.py index 7f378e092..8c8f11b82 100644 --- a/artiq/language/environment.py +++ b/artiq/language/environment.py @@ -207,6 +207,14 @@ class HasEnvironment: def set_dataset(self, key, value, broadcast=False, persist=False, save=True): + """Sets the contents and handling modes of a dataset. + + :param broadcast: the data is sent in real-time to the master, which + dispatches it. Returns a Notifier that can be used to mutate the dataset. + :param persist: the master should store the data on-disk. Implies broadcast. + :param save: the data is saved into the local storage of the current + run (archived as a HDF5 file). + """ if self.__parent is not None: self.__parent.set_dataset(key, value, broadcast, persist, save) return @@ -215,6 +223,15 @@ class HasEnvironment: return self.__dataset_mgr.set(key, value, broadcast, persist, save) def get_dataset(self, key, default=NoDefault): + """Returns the contents of a dataset. + + The local storage is searched first, followed by the master storage + (which contains the broadcasted datasets from all experiments) if the + key was not found initially. + + If the dataset does not exist, returns the default value. If no default + is provided, raises ``KeyError``. + """ if self.__parent is not None: return self.__parent.get_dataset(key, default) if self.__dataset_mgr is None: @@ -228,6 +245,8 @@ class HasEnvironment: return default def setattr_dataset(self, key, default=NoDefault): + """Sets the contents of a dataset as attribute. The names of the + dataset and of the attribute are the same.""" setattr(self, key, self.get_dataset(key, default)) diff --git a/doc/manual/environment.rst b/doc/manual/environment.rst index 8fbcda495..853c05ee2 100644 --- a/doc/manual/environment.rst +++ b/doc/manual/environment.rst @@ -3,7 +3,7 @@ The environment Experiments interact with an environment that consists of devices, parameters, arguments and results. Access to the environment is handled by the class :class:`artiq.language.environment.EnvExperiment` that experiments should derive from. -.. _ddb: +.. _device-db: The device database ------------------- @@ -12,7 +12,7 @@ The device database contains information about the devices available in a ARTIQ The master (or ``artiq_run``) instantiates the device drivers (and the RPC clients in the case of controllers) for the experiments based on the contents of the device database. -The device database is stored in the memory of the master and is backed by a PYON file typically called ``ddb.pyon``. +The device database is stored in the memory of the master and is backed by a PYON file typically called ``device_db.pyon``. The device database is a Python dictionary whose keys are the device names, and values can have several types. @@ -33,11 +33,6 @@ Aliases If an entry is a string, that string is used as a key for another lookup in the device database. -The parameter database ----------------------- - -The parameter database is a key-value store that is global to all experiments. It is stored in the memory of the master and is backed by a PYON file typically called ``pdb.pyon``. It may be used to communicate values across experiments; for example, a periodic calibration experiment may update a parameter read by payload experiments. - Arguments --------- @@ -45,7 +40,14 @@ Arguments are values that parameterize the behavior of an experiment and are set Requesting the values of arguments can only be done in the build phase of an experiment. The value requests are also used to define the GUI widgets shown in the explorer when the experiment is selected. -Results -------- -Results are the output of an experiment. They are archived after in the HDF5 format after the experiment is run. Experiments may define real-time results that are (additionally) distributed to all clients connected to the master; for example, the ARTIQ GUI may plot them while the experiment is in progress to give rapid feedback to the user. Real-time results are a global key-value store (similar to the parameter database); experiments should use distinctive real-time result names in order to avoid conflicts. +Datasets +-------- + +Datasets are values (possibly arrays) that are read and written by experiments and live in a key-value store. + +A dataset may be broadcasted, that is, distributed to all clients connected to the master. For example, the ARTIQ GUI may plot it while the experiment is in progress to give rapid feedback to the user. Broadcasted datasets live in a global key-value store; experiments should use distinctive real-time result names in order to avoid conflicts. Broadcasted datasets may be used to communicate values across experiments; for example, a periodic calibration experiment may update a dataset read by payload experiments. Broadcasted datasets are replaced when a new dataset with the same key (name) is produced. + +Broadcasted datasets may be persistent: the master stores them in a file typically called ``dataset_db.pyon`` so they are saved across master restarts. + +Datasets produced by an experiment run may be archived in the HDF5 output for that run. diff --git a/doc/manual/getting_started_core.rst b/doc/manual/getting_started_core.rst index b7a9c6f5e..a9e78518c 100644 --- a/doc/manual/getting_started_core.rst +++ b/doc/manual/getting_started_core.rst @@ -22,7 +22,7 @@ As a very first step, we will turn on a LED on the core device. Create a file `` The central part of our code is our ``LED`` class, that derives from :class:`artiq.language.environment.EnvExperiment`. Among other features, ``EnvExperiment`` calls our ``build`` method and provides the ``setattr_device`` method that interfaces to the device database to create the appropriate device drivers and make those drivers accessible as ``self.core`` and ``self.led``. The ``@kernel`` decorator tells the system that the ``run`` method must be executed on the core device (instead of the host). The decorator uses ``self.core`` internally, which is why we request the core device using ``setattr_device`` like any other. -Copy the files ``ddb.pyon`` and ``pdb.pyon`` (containing the device and parameter databases) from the ``examples/master`` folder of ARTIQ into the same directory as ``led.py`` (alternatively, you can use the ``-d`` and ``-p`` options of ``artiq_run``). You can open the database files using a text editor - their contents are in a human-readable format. You will probably want to set the IP address of the core device in ``ddb.pyon`` so that the computer can connect to it (it is the ``host`` parameter of the ``comm`` entry). See :ref:`ddb` for more information. The example device database is designed for the NIST QC1 hardware on the KC705; see :ref:`board-ports` for RTIO channel assignments if you need to adapt the device database to a different hardware platform. +Copy the files ``device_db.pyon`` and ``dataset_db.pyon`` (containing the device and dataset databases) from the ``examples/master`` folder of ARTIQ into the same directory as ``led.py`` (alternatively, you can use the ``--device-db`` and ``--dataset-db`` options of ``artiq_run``). You can open the database files using a text editor - their contents are in a human-readable format. You will probably want to set the IP address of the core device in ``device_db.pyon`` so that the computer can connect to it (it is the ``host`` parameter of the ``comm`` entry). See :ref:`device-db` for more information. The example device database is designed for the NIST QC1 hardware on the KC705; see :ref:`board-ports` for RTIO channel assignments if you need to adapt the device database to a different hardware platform. .. note:: If the ``led`` device is a bidirectional TTL (i.e. ``TTLInOut`` instead of ``TTLOut``), you need to put it in output (driving) mode. Add the following at the beginning of ``run``: :: diff --git a/doc/manual/getting_started_mgmt.rst b/doc/manual/getting_started_mgmt.rst index 40c8a2054..8152295c5 100644 --- a/doc/manual/getting_started_mgmt.rst +++ b/doc/manual/getting_started_mgmt.rst @@ -10,7 +10,7 @@ Starting your first experiment with the master In the previous tutorial, we used the ``artiq_run`` utility to execute our experiments, which is a simple stand-alone tool that bypasses the ARTIQ management system. We will now see how to run an experiment using the master (the central program in the management system that schedules and executes experiments) and the GUI client (that connects to the master and controls it). -First, create a folder ``~/artiq-master`` and copy the ``ddb.pyon`` and ``pdb.pyon`` files (device and parameter databases) found in the ``examples/master`` directory from the ARTIQ sources. The master uses those files in the same way as ``artiq_run``. +First, create a folder ``~/artiq-master`` and copy the ``device_db.pyon`` and ``dataset_db.pyon`` (containing the device and dataset databases) found in the ``examples/master`` directory from the ARTIQ sources. The master uses those files in the same way as ``artiq_run``. Then create a ``~/artiq-master/repository`` sub-folder to contain experiments. The master scans this ``repository`` folder to determine what experiments are available (the name of the folder can be changed using ``-r``). diff --git a/doc/manual/utilities.rst b/doc/manual/utilities.rst index 306ebe0f9..1f97c98ca 100644 --- a/doc/manual/utilities.rst +++ b/doc/manual/utilities.rst @@ -104,9 +104,7 @@ The artiq_coretool utility allows to perform maintenance on the core device: * as well as read, write and remove key-value records from the :ref:`core-device-flash-storage`; * erase the entire flash storage area. -To use this tool, you need to specify a ``ddb.pyon`` DDB file which contains a ``comm`` device (an example is provided in ``artiq/examples/master/ddb.pyon``). -This tells the tool how to connect to the core device (via serial or via TCP) and with which parameters (baudrate, serial device, IP address, TCP port). -When not specified, the artiq_coretool utility will assume that there is a file named ``ddb.pyon`` in the current directory. +To use this tool, you need to specify a ``device_db.pyon`` device database file which contains a ``comm`` device (an example is provided in ``artiq/examples/master/device_db.pyon``). This tells the tool how to connect to the core device (via serial or via TCP) and with which parameters (baudrate, serial device, IP address, TCP port). When not specified, the artiq_coretool utility will assume that there is a file named ``device_db.pyon`` in the current directory. To read the record whose key is ``mac``:: From ef487ee8b043e81063eefefa2fb6625051f337da Mon Sep 17 00:00:00 2001 From: Sebastien Bourdeauducq Date: Tue, 13 Oct 2015 18:45:04 +0800 Subject: [PATCH 02/14] doc: more updates for dataset API --- doc/manual/core_device.rst | 2 +- doc/manual/environment.rst | 2 +- doc/manual/getting_started_mgmt.rst | 8 ++++---- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/doc/manual/core_device.rst b/doc/manual/core_device.rst index c99ff7862..d01a992a6 100644 --- a/doc/manual/core_device.rst +++ b/doc/manual/core_device.rst @@ -3,7 +3,7 @@ Core device The core device is a FPGA-based hardware component that contains a softcore CPU tightly coupled with the so-called RTIO core that provides precision timing. The CPU executes Python code that is statically compiled by the ARTIQ compiler, and communicates with the core device peripherals (TTL, DDS, etc.) over the RTIO core. This architecture provides high timing resolution, low latency, low jitter, high level programming capabilities, and good integration with the rest of the Python experiment code. -While it is possible to use all the other parts of ARTIQ (controllers, master, GUI, result management, etc.) without a core device, many experiments require it. +While it is possible to use all the other parts of ARTIQ (controllers, master, GUI, dataset management, etc.) without a core device, many experiments require it. .. _core-device-flash-storage: diff --git a/doc/manual/environment.rst b/doc/manual/environment.rst index 853c05ee2..d5d78d323 100644 --- a/doc/manual/environment.rst +++ b/doc/manual/environment.rst @@ -1,7 +1,7 @@ The environment =============== -Experiments interact with an environment that consists of devices, parameters, arguments and results. Access to the environment is handled by the class :class:`artiq.language.environment.EnvExperiment` that experiments should derive from. +Experiments interact with an environment that consists of devices, arguments and datasets. Access to the environment is handled by the class :class:`artiq.language.environment.EnvExperiment` that experiments should derive from. .. _device-db: diff --git a/doc/manual/getting_started_mgmt.rst b/doc/manual/getting_started_mgmt.rst index 8152295c5..9ddf82f1f 100644 --- a/doc/manual/getting_started_mgmt.rst +++ b/doc/manual/getting_started_mgmt.rst @@ -136,20 +136,20 @@ The master should now run the new version from its repository. As an exercise, add another argument to the experiment, commit and push the result, and verify that the new control is added in the GUI. -Results -------- +Datasets +-------- Modify the ``run()`` method of the experiment as follows: :: def run(self): - parabola = self.set_result("parabola", [], realtime=True) + parabola = self.set_dataset("parabola", [], broadcast=True) for i in range(int(self.count)): parabola.append(i*i) time.sleep(0.5) .. note:: You need to import the ``time`` module. -Commit, push and submit the experiment as before. While it is running, go to the "Results" dock of the GUI and create a new XY plot showing the new result. Observe how the points are added one by one to the plot. +Commit, push and submit the experiment as before. While it is running, go to the "Datasets" dock of the GUI and create a new XY plot showing the new result. Observe how the points are added one by one to the plot. After the experiment has finished executing, the results are written to a HDF5 file that resides in ``~/artiq-master/results//