From cd757c0f160a3e5661624b9a624e54a4b6df18dc Mon Sep 17 00:00:00 2001
From: Sebastien Bourdeauducq <sb@m-labs.hk>
Date: Thu, 18 May 2017 23:14:20 +0800
Subject: [PATCH] generate device database from executable python file

---
 README_PHASER.rst                                     |  2 +-
 RELEASE_NOTES.rst                                     |  4 ++++
 artiq/compiler/testbench/embedding.py                 |  2 +-
 artiq/compiler/testbench/perf_embedding.py            |  3 ++-
 artiq/examples/artiq_ipython_notebook.ipynb           |  2 +-
 artiq/examples/drtio/{device_db.pyon => device_db.py} |  3 +--
 .../examples/master/{device_db.pyon => device_db.py}  |  4 ++--
 .../examples/phaser/{device_db.pyon => device_db.py}  |  2 +-
 artiq/examples/sim/{device_db.pyon => device_db.py}   |  2 +-
 artiq/frontend/artiq_compile.py                       |  2 +-
 artiq/frontend/artiq_coreanalyzer.py                  |  2 +-
 artiq/frontend/artiq_coreboot.py                      |  4 ++--
 artiq/frontend/artiq_coreconfig.py                    |  2 +-
 artiq/frontend/artiq_corelog.py                       |  2 +-
 artiq/frontend/artiq_master.py                        |  2 +-
 artiq/frontend/artiq_run.py                           |  2 +-
 artiq/master/databases.py                             | 11 +++++++++--
 artiq/test/hardware_testbench.py                      |  4 ++--
 .../devirtualization/{device_db.pyon => device_db.py} |  2 +-
 .../lit/embedding/{device_db.pyon => device_db.py}    |  2 +-
 .../test/lit/escape/{device_db.pyon => device_db.py}  |  2 +-
 .../lit/regression/{device_db.pyon => device_db.py}   |  2 +-
 doc/manual/environment.rst                            |  4 +---
 doc/manual/getting_started_core.rst                   |  2 +-
 doc/manual/getting_started_mgmt.rst                   |  2 +-
 doc/manual/utilities.rst                              |  2 +-
 26 files changed, 41 insertions(+), 32 deletions(-)
 rename artiq/examples/drtio/{device_db.pyon => device_db.py} (99%)
 rename artiq/examples/master/{device_db.pyon => device_db.py} (99%)
 rename artiq/examples/phaser/{device_db.pyon => device_db.py} (99%)
 rename artiq/examples/sim/{device_db.pyon => device_db.py} (98%)
 rename artiq/test/lit/devirtualization/{device_db.pyon => device_db.py} (95%)
 rename artiq/test/lit/embedding/{device_db.pyon => device_db.py} (95%)
 rename artiq/test/lit/escape/{device_db.pyon => device_db.py} (95%)
 rename artiq/test/lit/regression/{device_db.pyon => device_db.py} (95%)

diff --git a/README_PHASER.rst b/README_PHASER.rst
index 97065b834..369d91877 100644
--- a/README_PHASER.rst
+++ b/README_PHASER.rst
@@ -94,7 +94,7 @@ Setup
 
     cd artiq/examples/phaser
 
-* Edit ``device_db.pyon`` to match the hostname or IP address of the core device.
+* Edit ``device_db.py`` to match the hostname or IP address of the core device.
 * Use ``ping`` and ``flterm`` to verify that the core device starts up and boots correctly.
 
 Usage
diff --git a/RELEASE_NOTES.rst b/RELEASE_NOTES.rst
index e9866c2ba..f87de2534 100644
--- a/RELEASE_NOTES.rst
+++ b/RELEASE_NOTES.rst
@@ -31,6 +31,10 @@ Release notes
 * ``LinearScan`` and ``RandomScan`` have been consolidated into RangeScan.
 * The Pipistrello is no longer supported. For a low-cost ARTIQ setup, use either
   ARTIQ 2.x with Pipistrello, or the future ARTIQ 4.x with Kasli.
+* The device database is now generated by an executable Python script. To migrate
+  an existing database, add ``device_db = `` at the beginning, and replace any PYON
+  identifiers (``true``, ``null``, ...) with their Python equivalents
+  (``True``, ``None` ...).
 
 
 2.3
diff --git a/artiq/compiler/testbench/embedding.py b/artiq/compiler/testbench/embedding.py
index 01300c62a..65435a7db 100644
--- a/artiq/compiler/testbench/embedding.py
+++ b/artiq/compiler/testbench/embedding.py
@@ -24,7 +24,7 @@ def main():
     else:
         compile_only = False
 
-    ddb_path = os.path.join(os.path.dirname(sys.argv[1]), "device_db.pyon")
+    ddb_path = os.path.join(os.path.dirname(sys.argv[1]), "device_db.py")
     dmgr = DeviceManager(DeviceDB(ddb_path))
 
     with open(sys.argv[1]) as f:
diff --git a/artiq/compiler/testbench/perf_embedding.py b/artiq/compiler/testbench/perf_embedding.py
index 87d35ee04..b94b88735 100644
--- a/artiq/compiler/testbench/perf_embedding.py
+++ b/artiq/compiler/testbench/perf_embedding.py
@@ -8,6 +8,7 @@ from ..embedding import Stitcher
 from ..targets import OR1KTarget
 from . import benchmark
 
+
 def main():
     if not len(sys.argv) == 2:
         print("Expected exactly one module filename", file=sys.stderr)
@@ -26,7 +27,7 @@ def main():
         testcase_vars = {'__name__': 'testbench'}
         exec(testcase_code, testcase_vars)
 
-    device_db_path = os.path.join(os.path.dirname(sys.argv[1]), "device_db.pyon")
+    device_db_path = os.path.join(os.path.dirname(sys.argv[1]), "device_db.py")
     device_mgr = DeviceManager(DeviceDB(device_db_path))
 
     dataset_db_path = os.path.join(os.path.dirname(sys.argv[1]), "dataset_db.pyon")
diff --git a/artiq/examples/artiq_ipython_notebook.ipynb b/artiq/examples/artiq_ipython_notebook.ipynb
index 6a37fa819..6bab291fe 100644
--- a/artiq/examples/artiq_ipython_notebook.ipynb
+++ b/artiq/examples/artiq_ipython_notebook.ipynb
@@ -65,7 +65,7 @@
     "# and access any artiq device\n",
     "\n",
     "# we can have artiq prepare that connection for us:\n",
-    "ddb = DeviceDB(\"device_db.pyon\")\n",
+    "ddb = DeviceDB(\"device_db.py\")\n",
     "devmgr = DeviceManager(ddb)\n",
     "lda = devmgr.get(\"lda\")\n",
     "lda.set_attenuation(42)\n",
diff --git a/artiq/examples/drtio/device_db.pyon b/artiq/examples/drtio/device_db.py
similarity index 99%
rename from artiq/examples/drtio/device_db.pyon
rename to artiq/examples/drtio/device_db.py
index af2f6a49b..8171fda1b 100644
--- a/artiq/examples/drtio/device_db.pyon
+++ b/artiq/examples/drtio/device_db.py
@@ -2,7 +2,7 @@
 # The RTIO channel numbers here are for NIST CLOCK on KC705.
 # The list of devices here is not exhaustive.
 
-{
+device_db = {
     "comm": {
         "type": "local",
         "module": "artiq.coredevice.comm_kernel",
@@ -144,5 +144,4 @@
         "class": "TTLInOut",
         "arguments": {"channel": 0x010009}
     },
-
 }
diff --git a/artiq/examples/master/device_db.pyon b/artiq/examples/master/device_db.py
similarity index 99%
rename from artiq/examples/master/device_db.pyon
rename to artiq/examples/master/device_db.py
index 71dede55d..e111f9b82 100644
--- a/artiq/examples/master/device_db.pyon
+++ b/artiq/examples/master/device_db.py
@@ -2,7 +2,7 @@
 # The RTIO channel numbers here are for NIST CLOCK on KC705.
 # The list of devices here is not exhaustive.
 
-{
+device_db = {
     "comm": {
         "type": "local",
         "module": "artiq.coredevice.comm_kernel",
@@ -203,7 +203,7 @@
 
     "lda": {
         "type": "controller",
-        "best_effort": true,
+        "best_effort": True,
         "host": "::1",
         "port": 3253,
         "command": "lda_controller -p {port} --bind {bind} --simulation"
diff --git a/artiq/examples/phaser/device_db.pyon b/artiq/examples/phaser/device_db.py
similarity index 99%
rename from artiq/examples/phaser/device_db.pyon
rename to artiq/examples/phaser/device_db.py
index 208762478..0f740fae9 100644
--- a/artiq/examples/phaser/device_db.pyon
+++ b/artiq/examples/phaser/device_db.py
@@ -1,6 +1,6 @@
 # The RTIO channel numbers here are for Phaser on KC705.
 
-{
+device_db = {
     "comm": {
         "type": "local",
         "module": "artiq.coredevice.comm_kernel",
diff --git a/artiq/examples/sim/device_db.pyon b/artiq/examples/sim/device_db.py
similarity index 98%
rename from artiq/examples/sim/device_db.pyon
rename to artiq/examples/sim/device_db.py
index 7b749e6fd..2fe351a5b 100644
--- a/artiq/examples/sim/device_db.pyon
+++ b/artiq/examples/sim/device_db.py
@@ -1,4 +1,4 @@
-{
+device_db = {
     "core": {
         "type": "local",
         "module": "artiq.sim.devices",
diff --git a/artiq/frontend/artiq_compile.py b/artiq/frontend/artiq_compile.py
index f64d76ac6..83cf3c079 100755
--- a/artiq/frontend/artiq_compile.py
+++ b/artiq/frontend/artiq_compile.py
@@ -16,7 +16,7 @@ def get_argparser():
     parser = argparse.ArgumentParser(description="ARTIQ static compiler")
 
     verbosity_args(parser)
-    parser.add_argument("--device-db", default="device_db.pyon",
+    parser.add_argument("--device-db", default="device_db.py",
                         help="device database file (default: '%(default)s')")
     parser.add_argument("--dataset-db", default="dataset_db.pyon",
                         help="dataset file (default: '%(default)s')")
diff --git a/artiq/frontend/artiq_coreanalyzer.py b/artiq/frontend/artiq_coreanalyzer.py
index 4f7626ea9..22cafc5b7 100755
--- a/artiq/frontend/artiq_coreanalyzer.py
+++ b/artiq/frontend/artiq_coreanalyzer.py
@@ -15,7 +15,7 @@ def get_argparser():
                                                  "RTIO analysis tool")
 
     verbosity_args(parser)
-    parser.add_argument("--device-db", default="device_db.pyon",
+    parser.add_argument("--device-db", default="device_db.py",
                        help="device database file (default: '%(default)s')")
 
     parser.add_argument("-r", "--read-dump", type=str, default=None,
diff --git a/artiq/frontend/artiq_coreboot.py b/artiq/frontend/artiq_coreboot.py
index 45c317247..e619bc69a 100755
--- a/artiq/frontend/artiq_coreboot.py
+++ b/artiq/frontend/artiq_coreboot.py
@@ -14,7 +14,7 @@ def get_argparser():
     parser = argparse.ArgumentParser(description="ARTIQ core device boot tool")
 
     verbosity_args(parser)
-    parser.add_argument("--device-db", default="device_db.pyon",
+    parser.add_argument("--device-db", default="device_db.py",
                        help="device database file (default: '%(default)s')")
 
     subparsers = parser.add_subparsers(dest="action")
@@ -25,7 +25,7 @@ def get_argparser():
     p_hotswap = subparsers.add_parser("hotswap",
                                       help="load the specified firmware in RAM")
 
-    p_hotswap.add_argument("image", metavar="IMAGE", type=argparse.FileType('rb'),
+    p_hotswap.add_argument("image", metavar="IMAGE", type=argparse.FileType("rb"),
                            help="runtime image to be executed")
 
     return parser
diff --git a/artiq/frontend/artiq_coreconfig.py b/artiq/frontend/artiq_coreconfig.py
index cc8aa44be..43aa02739 100755
--- a/artiq/frontend/artiq_coreconfig.py
+++ b/artiq/frontend/artiq_coreconfig.py
@@ -13,7 +13,7 @@ def get_argparser():
                                                  "configuration tool")
 
     verbosity_args(parser)
-    parser.add_argument("--device-db", default="device_db.pyon",
+    parser.add_argument("--device-db", default="device_db.py",
                        help="device database file (default: '%(default)s')")
 
     subparsers = parser.add_subparsers(dest="action")
diff --git a/artiq/frontend/artiq_corelog.py b/artiq/frontend/artiq_corelog.py
index 76732c4bf..88bc996c7 100755
--- a/artiq/frontend/artiq_corelog.py
+++ b/artiq/frontend/artiq_corelog.py
@@ -12,7 +12,7 @@ def get_argparser():
     parser = argparse.ArgumentParser(description="ARTIQ core device "
                                                  "log download tool")
     verbosity_args(parser)
-    parser.add_argument("--device-db", default="device_db.pyon",
+    parser.add_argument("--device-db", default="device_db.py",
                        help="device database file (default: '%(default)s')")
 
     subparsers = parser.add_subparsers(dest="action")
diff --git a/artiq/frontend/artiq_master.py b/artiq/frontend/artiq_master.py
index 47d421ce2..d749fb4fa 100755
--- a/artiq/frontend/artiq_master.py
+++ b/artiq/frontend/artiq_master.py
@@ -33,7 +33,7 @@ def get_argparser():
     ])
 
     group = parser.add_argument_group("databases")
-    group.add_argument("--device-db", default="device_db.pyon",
+    group.add_argument("--device-db", default="device_db.py",
                        help="device database file (default: '%(default)s')")
     group.add_argument("--dataset-db", default="dataset_db.pyon",
                        help="dataset file (default: '%(default)s')")
diff --git a/artiq/frontend/artiq_run.py b/artiq/frontend/artiq_run.py
index 11419ea68..962e7a5e5 100755
--- a/artiq/frontend/artiq_run.py
+++ b/artiq/frontend/artiq_run.py
@@ -127,7 +127,7 @@ def get_argparser(with_file=True):
         description="Local experiment running tool")
 
     verbosity_args(parser)
-    parser.add_argument("--device-db", default="device_db.pyon",
+    parser.add_argument("--device-db", default="device_db.py",
                         help="device database file (default: '%(default)s')")
     parser.add_argument("--dataset-db", default="dataset_db.pyon",
                         help="dataset file (default: '%(default)s')")
diff --git a/artiq/master/databases.py b/artiq/master/databases.py
index 9ad0c1be3..e71862d44 100644
--- a/artiq/master/databases.py
+++ b/artiq/master/databases.py
@@ -5,13 +5,20 @@ from artiq.protocols import pyon
 from artiq.tools import TaskObject
 
 
+def device_db_from_file(filename):
+    glbs = dict()
+    with open(filename, "r") as f:
+        exec(f.read(), glbs)
+    return glbs["device_db"]
+
+
 class DeviceDB:
     def __init__(self, backing_file):
         self.backing_file = backing_file
-        self.data = Notifier(pyon.load_file(self.backing_file))
+        self.data = Notifier(device_db_from_file(self.backing_file))
 
     def scan(self):
-        new_data = pyon.load_file(self.backing_file)
+        new_data = device_db_from_file(self.backing_file)
 
         for k in list(self.data.read.keys()):
             if k not in new_data:
diff --git a/artiq/test/hardware_testbench.py b/artiq/test/hardware_testbench.py
index ff06e5c81..4ce3114c3 100644
--- a/artiq/test/hardware_testbench.py
+++ b/artiq/test/hardware_testbench.py
@@ -92,13 +92,13 @@ class GenericControllerCase(unittest.TestCase):
 @unittest.skipUnless(artiq_root, "no ARTIQ_ROOT")
 class ControllerCase(GenericControllerCase):
     def get_device_db(self):
-        return DeviceDB(os.path.join(artiq_root, "device_db.pyon"))
+        return DeviceDB(os.path.join(artiq_root, "device_db.py"))
 
 
 @unittest.skipUnless(artiq_root, "no ARTIQ_ROOT")
 class ExperimentCase(unittest.TestCase):
     def setUp(self):
-        self.device_db = DeviceDB(os.path.join(artiq_root, "device_db.pyon"))
+        self.device_db = DeviceDB(os.path.join(artiq_root, "device_db.py"))
         self.dataset_db = DatasetDB(
             os.path.join(artiq_root, "dataset_db.pyon"))
         self.device_mgr = DeviceManager(
diff --git a/artiq/test/lit/devirtualization/device_db.pyon b/artiq/test/lit/devirtualization/device_db.py
similarity index 95%
rename from artiq/test/lit/devirtualization/device_db.pyon
rename to artiq/test/lit/devirtualization/device_db.py
index eabdbcc0a..99fa148fd 100644
--- a/artiq/test/lit/devirtualization/device_db.pyon
+++ b/artiq/test/lit/devirtualization/device_db.py
@@ -1,4 +1,4 @@
-{
+device_db = {
     "comm": {
         "type": "local",
         "module": "artiq.coredevice.comm_kernel_dummy",
diff --git a/artiq/test/lit/embedding/device_db.pyon b/artiq/test/lit/embedding/device_db.py
similarity index 95%
rename from artiq/test/lit/embedding/device_db.pyon
rename to artiq/test/lit/embedding/device_db.py
index eabdbcc0a..99fa148fd 100644
--- a/artiq/test/lit/embedding/device_db.pyon
+++ b/artiq/test/lit/embedding/device_db.py
@@ -1,4 +1,4 @@
-{
+device_db = {
     "comm": {
         "type": "local",
         "module": "artiq.coredevice.comm_kernel_dummy",
diff --git a/artiq/test/lit/escape/device_db.pyon b/artiq/test/lit/escape/device_db.py
similarity index 95%
rename from artiq/test/lit/escape/device_db.pyon
rename to artiq/test/lit/escape/device_db.py
index eabdbcc0a..99fa148fd 100644
--- a/artiq/test/lit/escape/device_db.pyon
+++ b/artiq/test/lit/escape/device_db.py
@@ -1,4 +1,4 @@
-{
+device_db = {
     "comm": {
         "type": "local",
         "module": "artiq.coredevice.comm_kernel_dummy",
diff --git a/artiq/test/lit/regression/device_db.pyon b/artiq/test/lit/regression/device_db.py
similarity index 95%
rename from artiq/test/lit/regression/device_db.pyon
rename to artiq/test/lit/regression/device_db.py
index eabdbcc0a..99fa148fd 100644
--- a/artiq/test/lit/regression/device_db.pyon
+++ b/artiq/test/lit/regression/device_db.py
@@ -1,4 +1,4 @@
-{
+device_db = {
     "comm": {
         "type": "local",
         "module": "artiq.coredevice.comm_kernel_dummy",
diff --git a/doc/manual/environment.rst b/doc/manual/environment.rst
index d5d78d323..924c54509 100644
--- a/doc/manual/environment.rst
+++ b/doc/manual/environment.rst
@@ -12,9 +12,7 @@ The device database contains information about the devices available in a ARTIQ
 
 The master (or ``artiq_run``) instantiates the device drivers (and the RPC clients in the case of controllers) for the experiments based on the contents of the device database.
 
-The device database is stored in the memory of the master and is backed by a PYON file typically called ``device_db.pyon``.
-
-The device database is a Python dictionary whose keys are the device names, and values can have several types.
+The device database is stored in the memory of the master and is generated by a Python script typically called ``device_db.py``. That script must define a global variable ``device_db`` with the contents of the database. The device database is a Python dictionary whose keys are the device names, and values can have several types.
 
 Local devices
 +++++++++++++
diff --git a/doc/manual/getting_started_core.rst b/doc/manual/getting_started_core.rst
index 60fd301a7..eca62fd03 100644
--- a/doc/manual/getting_started_core.rst
+++ b/doc/manual/getting_started_core.rst
@@ -23,7 +23,7 @@ As a very first step, we will turn on a LED on the core device. Create a file ``
 
 The central part of our code is our ``LED`` class, that derives from :class:`artiq.language.environment.EnvExperiment`. Among other features, ``EnvExperiment`` calls our ``build`` method and provides the ``setattr_device`` method that interfaces to the device database to create the appropriate device drivers and make those drivers accessible as ``self.core`` and ``self.led``. The ``@kernel`` decorator tells the system that the ``run`` method must be compiled for and executed on the core device (instead of being interpreted and executed as regular Python code on the host). The decorator uses ``self.core`` internally, which is why we request the core device using ``setattr_device`` like any other.
 
-Copy the file ``device_db.pyon`` (containing the device database) from the ``examples/master`` folder of ARTIQ into the same directory as ``led.py`` (alternatively, you can use the ``--device-db`` option of ``artiq_run``). You can open PYON database files using a text editor - their contents are in a human-readable format. You will probably want to set the IP address of the core device in ``device_db.pyon`` so that the computer can connect to it (it is the ``host`` parameter of the ``comm`` entry). See :ref:`device-db` for more information. The example device database is designed for the ``nist_clock`` hardware adapter on the KC705; see :ref:`board-ports` for RTIO channel assignments if you need to adapt the device database to a different hardware platform.
+Copy the file ``device_db.py`` (containing the device database) from the ``examples/master`` folder of ARTIQ into the same directory as ``led.py`` (alternatively, you can use the ``--device-db`` option of ``artiq_run``). You will probably want to set the IP address of the core device in ``device_db.py`` so that the computer can connect to it (it is the ``host`` parameter of the ``comm`` entry). See :ref:`device-db` for more information. The example device database is designed for the ``nist_clock`` hardware adapter on the KC705; see :ref:`board-ports` for RTIO channel assignments if you need to adapt the device database to a different hardware platform.
 
 .. note::
     To obtain the examples, you can find where the ARTIQ package is installed on your machine with: ::
diff --git a/doc/manual/getting_started_mgmt.rst b/doc/manual/getting_started_mgmt.rst
index 154626e14..e667abe02 100644
--- a/doc/manual/getting_started_mgmt.rst
+++ b/doc/manual/getting_started_mgmt.rst
@@ -10,7 +10,7 @@ Starting your first experiment with the master
 
 In the previous tutorial, we used the ``artiq_run`` utility to execute our experiments, which is a simple stand-alone tool that bypasses the ARTIQ management system. We will now see how to run an experiment using the master (the central program in the management system that schedules and executes experiments) and the dashboard (that connects to the master and controls it).
 
-First, create a folder ``~/artiq-master`` and copy the file ``device_db.pyon`` (containing the device database) found in the ``examples/master`` directory from the ARTIQ sources. The master uses those files in the same way as ``artiq_run``.
+First, create a folder ``~/artiq-master`` and copy the file ``device_db.py`` (containing the device database) found in the ``examples/master`` directory from the ARTIQ sources. The master uses those files in the same way as ``artiq_run``.
 
 Then create a ``~/artiq-master/repository`` sub-folder to contain experiments. The master scans this ``repository`` folder to determine what experiments are available (the name of the folder can be changed using ``-r``).
 
diff --git a/doc/manual/utilities.rst b/doc/manual/utilities.rst
index 403919154..1a049b94f 100644
--- a/doc/manual/utilities.rst
+++ b/doc/manual/utilities.rst
@@ -111,7 +111,7 @@ Core device configuration tool
 
 The artiq_coreconfig utility gives remote access to the :ref:`core-device-flash-storage`.
 
-To use this tool, you need to specify a ``device_db.pyon`` device database file which contains a ``comm`` device (an example is provided in ``examples/master/device_db.pyon``). This tells the tool how to connect to the core device and with which parameters (e.g. IP address, TCP port). When not specified, the artiq_coreconfig utility will assume that there is a file named ``device_db.pyon`` in the current directory.
+To use this tool, you need to specify a ``device_db.py`` device database file which contains a ``comm`` device (an example is provided in ``examples/master/device_db.py``). This tells the tool how to connect to the core device and with which parameters (e.g. IP address, TCP port). When not specified, the artiq_coreconfig utility will assume that there is a file named ``device_db.py`` in the current directory.
 
 To read the record whose key is ``mac``::