generate device database from executable python file

README_PHASER.rst

@@ -94,7 +94,7 @@ Setup
 
     cd artiq/examples/phaser
 
-* Edit ``device_db.pyon`` to match the hostname or IP address of the core device.
+* Edit ``device_db.py`` to match the hostname or IP address of the core device.
 * Use ``ping`` and ``flterm`` to verify that the core device starts up and boots correctly.
 
 Usage

RELEASE_NOTES.rst

@@ -31,6 +31,10 @@ Release notes
 * ``LinearScan`` and ``RandomScan`` have been consolidated into RangeScan.
 * The Pipistrello is no longer supported. For a low-cost ARTIQ setup, use either
   ARTIQ 2.x with Pipistrello, or the future ARTIQ 4.x with Kasli.
+* The device database is now generated by an executable Python script. To migrate
+  an existing database, add ``device_db = `` at the beginning, and replace any PYON
+  identifiers (``true``, ``null``, ...) with their Python equivalents
+  (``True``, ``None` ...).
 
 
 2.3

artiq/compiler/testbench/embedding.py

@@ -24,7 +24,7 @@ def main():
     else:
         compile_only = False
 
-    ddb_path = os.path.join(os.path.dirname(sys.argv[1]), "device_db.pyon")
+    ddb_path = os.path.join(os.path.dirname(sys.argv[1]), "device_db.py")
     dmgr = DeviceManager(DeviceDB(ddb_path))
 
     with open(sys.argv[1]) as f:

artiq/compiler/testbench/perf_embedding.py

@@ -8,6 +8,7 @@ from ..embedding import Stitcher
 from ..targets import OR1KTarget
 from . import benchmark
 
+
 def main():
     if not len(sys.argv) == 2:
         print("Expected exactly one module filename", file=sys.stderr)
@@ -26,7 +27,7 @@ def main():
         testcase_vars = {'__name__': 'testbench'}
         exec(testcase_code, testcase_vars)
 
-    device_db_path = os.path.join(os.path.dirname(sys.argv[1]), "device_db.pyon")
+    device_db_path = os.path.join(os.path.dirname(sys.argv[1]), "device_db.py")
     device_mgr = DeviceManager(DeviceDB(device_db_path))
 
     dataset_db_path = os.path.join(os.path.dirname(sys.argv[1]), "dataset_db.pyon")

artiq/examples/artiq_ipython_notebook.ipynb

@@ -65,7 +65,7 @@
     "# and access any artiq device\n",
     "\n",
     "# we can have artiq prepare that connection for us:\n",
-    "ddb = DeviceDB(\"device_db.pyon\")\n",
+    "ddb = DeviceDB(\"device_db.py\")\n",
     "devmgr = DeviceManager(ddb)\n",
     "lda = devmgr.get(\"lda\")\n",
     "lda.set_attenuation(42)\n",

artiq/examples/drtio/device_db.py

@@ -2,7 +2,7 @@
 # The RTIO channel numbers here are for NIST CLOCK on KC705.
 # The list of devices here is not exhaustive.
 
-{
+device_db = {
     "comm": {
         "type": "local",
         "module": "artiq.coredevice.comm_kernel",
@@ -144,5 +144,4 @@
         "class": "TTLInOut",
         "arguments": {"channel": 0x010009}
     },
-
 }

artiq/examples/master/device_db.py

@@ -2,7 +2,7 @@
 # The RTIO channel numbers here are for NIST CLOCK on KC705.
 # The list of devices here is not exhaustive.
 
-{
+device_db = {
     "comm": {
         "type": "local",
         "module": "artiq.coredevice.comm_kernel",
@@ -203,7 +203,7 @@
 
     "lda": {
         "type": "controller",
-        "best_effort": true,
+        "best_effort": True,
         "host": "::1",
         "port": 3253,
         "command": "lda_controller -p {port} --bind {bind} --simulation"

artiq/examples/phaser/device_db.py

@@ -1,6 +1,6 @@
 # The RTIO channel numbers here are for Phaser on KC705.
 
-{
+device_db = {
     "comm": {
         "type": "local",
         "module": "artiq.coredevice.comm_kernel",

artiq/examples/sim/device_db.py

@@ -1,4 +1,4 @@
-{
+device_db = {
     "core": {
         "type": "local",
         "module": "artiq.sim.devices",

artiq/frontend/artiq_compile.py

@@ -16,7 +16,7 @@ def get_argparser():
     parser = argparse.ArgumentParser(description="ARTIQ static compiler")
 
     verbosity_args(parser)
-    parser.add_argument("--device-db", default="device_db.pyon",
+    parser.add_argument("--device-db", default="device_db.py",
                         help="device database file (default: '%(default)s')")
     parser.add_argument("--dataset-db", default="dataset_db.pyon",
                         help="dataset file (default: '%(default)s')")

artiq/frontend/artiq_coreanalyzer.py

@@ -15,7 +15,7 @@ def get_argparser():
                                                  "RTIO analysis tool")
 
     verbosity_args(parser)
-    parser.add_argument("--device-db", default="device_db.pyon",
+    parser.add_argument("--device-db", default="device_db.py",
                        help="device database file (default: '%(default)s')")
 
     parser.add_argument("-r", "--read-dump", type=str, default=None,

artiq/frontend/artiq_coreboot.py

@@ -14,7 +14,7 @@ def get_argparser():
     parser = argparse.ArgumentParser(description="ARTIQ core device boot tool")
 
     verbosity_args(parser)
-    parser.add_argument("--device-db", default="device_db.pyon",
+    parser.add_argument("--device-db", default="device_db.py",
                        help="device database file (default: '%(default)s')")
 
     subparsers = parser.add_subparsers(dest="action")
@@ -25,7 +25,7 @@ def get_argparser():
     p_hotswap = subparsers.add_parser("hotswap",
                                       help="load the specified firmware in RAM")
 
-    p_hotswap.add_argument("image", metavar="IMAGE", type=argparse.FileType('rb'),
+    p_hotswap.add_argument("image", metavar="IMAGE", type=argparse.FileType("rb"),
                            help="runtime image to be executed")
 
     return parser

artiq/frontend/artiq_coreconfig.py

@@ -13,7 +13,7 @@ def get_argparser():
                                                  "configuration tool")
 
     verbosity_args(parser)
-    parser.add_argument("--device-db", default="device_db.pyon",
+    parser.add_argument("--device-db", default="device_db.py",
                        help="device database file (default: '%(default)s')")
 
     subparsers = parser.add_subparsers(dest="action")

artiq/frontend/artiq_corelog.py

@@ -12,7 +12,7 @@ def get_argparser():
     parser = argparse.ArgumentParser(description="ARTIQ core device "
                                                  "log download tool")
     verbosity_args(parser)
-    parser.add_argument("--device-db", default="device_db.pyon",
+    parser.add_argument("--device-db", default="device_db.py",
                        help="device database file (default: '%(default)s')")
 
     subparsers = parser.add_subparsers(dest="action")

artiq/frontend/artiq_master.py

@@ -33,7 +33,7 @@ def get_argparser():
     ])
 
     group = parser.add_argument_group("databases")
-    group.add_argument("--device-db", default="device_db.pyon",
+    group.add_argument("--device-db", default="device_db.py",
                        help="device database file (default: '%(default)s')")
     group.add_argument("--dataset-db", default="dataset_db.pyon",
                        help="dataset file (default: '%(default)s')")

artiq/frontend/artiq_run.py

@@ -127,7 +127,7 @@ def get_argparser(with_file=True):
         description="Local experiment running tool")
 
     verbosity_args(parser)
-    parser.add_argument("--device-db", default="device_db.pyon",
+    parser.add_argument("--device-db", default="device_db.py",
                         help="device database file (default: '%(default)s')")
     parser.add_argument("--dataset-db", default="dataset_db.pyon",
                         help="dataset file (default: '%(default)s')")

artiq/master/databases.py

@@ -5,13 +5,20 @@ from artiq.protocols import pyon
 from artiq.tools import TaskObject
 
 
+def device_db_from_file(filename):
+    glbs = dict()
+    with open(filename, "r") as f:
+        exec(f.read(), glbs)
+    return glbs["device_db"]
+
+
 class DeviceDB:
     def __init__(self, backing_file):
         self.backing_file = backing_file
-        self.data = Notifier(pyon.load_file(self.backing_file))
+        self.data = Notifier(device_db_from_file(self.backing_file))
 
     def scan(self):
-        new_data = pyon.load_file(self.backing_file)
+        new_data = device_db_from_file(self.backing_file)
 
         for k in list(self.data.read.keys()):
             if k not in new_data:

artiq/test/hardware_testbench.py

@@ -92,13 +92,13 @@ class GenericControllerCase(unittest.TestCase):
 @unittest.skipUnless(artiq_root, "no ARTIQ_ROOT")
 class ControllerCase(GenericControllerCase):
     def get_device_db(self):
-        return DeviceDB(os.path.join(artiq_root, "device_db.pyon"))
+        return DeviceDB(os.path.join(artiq_root, "device_db.py"))
 
 
 @unittest.skipUnless(artiq_root, "no ARTIQ_ROOT")
 class ExperimentCase(unittest.TestCase):
     def setUp(self):
-        self.device_db = DeviceDB(os.path.join(artiq_root, "device_db.pyon"))
+        self.device_db = DeviceDB(os.path.join(artiq_root, "device_db.py"))
         self.dataset_db = DatasetDB(
             os.path.join(artiq_root, "dataset_db.pyon"))
         self.device_mgr = DeviceManager(

artiq/test/lit/devirtualization/device_db.py

@@ -1,4 +1,4 @@
-{
+device_db = {
     "comm": {
         "type": "local",
         "module": "artiq.coredevice.comm_kernel_dummy",

artiq/test/lit/embedding/device_db.py

@@ -1,4 +1,4 @@
-{
+device_db = {
     "comm": {
         "type": "local",
         "module": "artiq.coredevice.comm_kernel_dummy",

artiq/test/lit/escape/device_db.py

@@ -1,4 +1,4 @@
-{
+device_db = {
     "comm": {
         "type": "local",
         "module": "artiq.coredevice.comm_kernel_dummy",

artiq/test/lit/regression/device_db.py

@@ -1,4 +1,4 @@
-{
+device_db = {
     "comm": {
         "type": "local",
         "module": "artiq.coredevice.comm_kernel_dummy",

doc/manual/environment.rst

@@ -12,9 +12,7 @@ The device database contains information about the devices available in a ARTIQ
 
 The master (or ``artiq_run``) instantiates the device drivers (and the RPC clients in the case of controllers) for the experiments based on the contents of the device database.
 
-The device database is stored in the memory of the master and is backed by a PYON file typically called ``device_db.pyon``.
-
-The device database is a Python dictionary whose keys are the device names, and values can have several types.
+The device database is stored in the memory of the master and is generated by a Python script typically called ``device_db.py``. That script must define a global variable ``device_db`` with the contents of the database. The device database is a Python dictionary whose keys are the device names, and values can have several types.
 
 Local devices
 +++++++++++++

doc/manual/getting_started_core.rst

@@ -23,7 +23,7 @@ As a very first step, we will turn on a LED on the core device. Create a file ``
 
 The central part of our code is our ``LED`` class, that derives from :class:`artiq.language.environment.EnvExperiment`. Among other features, ``EnvExperiment`` calls our ``build`` method and provides the ``setattr_device`` method that interfaces to the device database to create the appropriate device drivers and make those drivers accessible as ``self.core`` and ``self.led``. The ``@kernel`` decorator tells the system that the ``run`` method must be compiled for and executed on the core device (instead of being interpreted and executed as regular Python code on the host). The decorator uses ``self.core`` internally, which is why we request the core device using ``setattr_device`` like any other.
 
-Copy the file ``device_db.pyon`` (containing the device database) from the ``examples/master`` folder of ARTIQ into the same directory as ``led.py`` (alternatively, you can use the ``--device-db`` option of ``artiq_run``). You can open PYON database files using a text editor - their contents are in a human-readable format. You will probably want to set the IP address of the core device in ``device_db.pyon`` so that the computer can connect to it (it is the ``host`` parameter of the ``comm`` entry). See :ref:`device-db` for more information. The example device database is designed for the ``nist_clock`` hardware adapter on the KC705; see :ref:`board-ports` for RTIO channel assignments if you need to adapt the device database to a different hardware platform.
+Copy the file ``device_db.py`` (containing the device database) from the ``examples/master`` folder of ARTIQ into the same directory as ``led.py`` (alternatively, you can use the ``--device-db`` option of ``artiq_run``). You will probably want to set the IP address of the core device in ``device_db.py`` so that the computer can connect to it (it is the ``host`` parameter of the ``comm`` entry). See :ref:`device-db` for more information. The example device database is designed for the ``nist_clock`` hardware adapter on the KC705; see :ref:`board-ports` for RTIO channel assignments if you need to adapt the device database to a different hardware platform.
 
 .. note::
     To obtain the examples, you can find where the ARTIQ package is installed on your machine with: ::

doc/manual/getting_started_mgmt.rst

@@ -10,7 +10,7 @@ Starting your first experiment with the master
 
 In the previous tutorial, we used the ``artiq_run`` utility to execute our experiments, which is a simple stand-alone tool that bypasses the ARTIQ management system. We will now see how to run an experiment using the master (the central program in the management system that schedules and executes experiments) and the dashboard (that connects to the master and controls it).
 
-First, create a folder ``~/artiq-master`` and copy the file ``device_db.pyon`` (containing the device database) found in the ``examples/master`` directory from the ARTIQ sources. The master uses those files in the same way as ``artiq_run``.
+First, create a folder ``~/artiq-master`` and copy the file ``device_db.py`` (containing the device database) found in the ``examples/master`` directory from the ARTIQ sources. The master uses those files in the same way as ``artiq_run``.
 
 Then create a ``~/artiq-master/repository`` sub-folder to contain experiments. The master scans this ``repository`` folder to determine what experiments are available (the name of the folder can be changed using ``-r``).
 

doc/manual/utilities.rst

@@ -111,7 +111,7 @@ Core device configuration tool
 
 The artiq_coreconfig utility gives remote access to the :ref:`core-device-flash-storage`.
 
-To use this tool, you need to specify a ``device_db.pyon`` device database file which contains a ``comm`` device (an example is provided in ``examples/master/device_db.pyon``). This tells the tool how to connect to the core device and with which parameters (e.g. IP address, TCP port). When not specified, the artiq_coreconfig utility will assume that there is a file named ``device_db.pyon`` in the current directory.
+To use this tool, you need to specify a ``device_db.py`` device database file which contains a ``comm`` device (an example is provided in ``examples/master/device_db.py``). This tells the tool how to connect to the core device and with which parameters (e.g. IP address, TCP port). When not specified, the artiq_coreconfig utility will assume that there is a file named ``device_db.py`` in the current directory.
 
 To read the record whose key is ``mac``::