From 00f2e3ae93c244c6cc58c73112c8e41bc580b83f Mon Sep 17 00:00:00 2001 From: architeuthidae Date: Tue, 27 Aug 2024 17:38:26 +0800 Subject: [PATCH 001/111] doc: Dashboard 'Load HDF5' button + nitpicks --- doc/manual/environment.rst | 2 +- doc/manual/using_data_interfaces.rst | 12 +++++++----- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/doc/manual/environment.rst b/doc/manual/environment.rst index b6df68280..c006cbf53 100644 --- a/doc/manual/environment.rst +++ b/doc/manual/environment.rst @@ -86,7 +86,7 @@ A dataset may be broadcast (``broadcast=True``), that is, distributed to all cli Broadcasted datasets are replaced when a new dataset with the same key (name) is produced. By default, they are erased when the master halts. Broadcasted datasets may be made persistent (``persistent=True``, which also implies ``broadcast=True``), in which case the master stores them in a LMDB database typically called ``dataset_db.mdb``, where they are saved across master restarts. -By default, datasets are archived in the ``results`` HDF5 output for that run, although this can be opted against (``archive=False``). +By default, datasets are archived in the ``results`` HDF5 output for that run, although this can be opted against (``archive=False``). They can be viewed and analyzed with the ARTIQ browser, or with an HDF5 viewer of your choice. Datasets and units ^^^^^^^^^^^^^^^^^^ diff --git a/doc/manual/using_data_interfaces.rst b/doc/manual/using_data_interfaces.rst index e303dc814..d0a6a671c 100644 --- a/doc/manual/using_data_interfaces.rst +++ b/doc/manual/using_data_interfaces.rst @@ -41,13 +41,15 @@ You can open the result file for this experiment with HDFView, h5dump, or any si .. tip:: If you are not familiar with Git, try running ``git log`` in either of your connected Git repositories to see a history of commits in the repository which includes their respective hashes. As long as this history remains intact, you can use a hash of this kind of to uniquely identify, and even retrieve, the state of the files in the repository at the time this experiment was run. In other words, when running experiments from a Git repository, it's always possible to retrieve the code that led to a particular set of results. +A last interesting feature of the result files is that, for experiments with arguments, they also store the values of the arguments used for that iteration of the experiment. Again, this is for reproducibility: if it's ever necessary to find what arguments produced certain results, that information is preserved in the HDF5 file. To repeat an experiment with the exact same arguments as in a previous run, the 'Load HDF5' button in the submission window can be used to take them directly from a result file. + Applets ^^^^^^^ -Most of the time, rather than the HDF dump, we would like to see our result datasets in a readable graphical form, preferably without opening any third-party applications. In the ARTIQ dashboard, this is achieved by programs called "applets". Applets provide simple, modular GUI features; are run independently from the dashboard as separate processes to achieve goals of modularity and resilience. ARTIQ supplies several applets for basic plotting in the :mod:`artiq.applets` module, and provides interfaces so users can write their own. +Most of the time, rather than the HDF dump, we would like to see our result datasets in a readable graphical form, preferably without opening any third-party applications. In the ARTIQ dashboard, this is achieved by programs called "applets". Applets provide simple, modular GUI features, and are run independently from the dashboard as separate processes for modularity and resilience. ARTIQ supplies several applets for basic plotting in the :mod:`artiq.applets` module, and provides interfaces so users can write their own. .. seealso:: - When developing your own applets, see also the references provided on the :ref:`Management system reference` page of this manual. + Resources for writing your own applets are detailed on the :ref:`Management system reference` page. For our ``parabola`` dataset, we will create an XY plot using the provided :mod:`artiq.applets.plot_xy`. Applets are configured with simple command line options. To figure out what configurations are accepted, use the ``-h`` flag, as in: :: @@ -57,7 +59,7 @@ In our case, we only need to supply our dataset to the applet to be plotted. Nav ${artiq_applet}plot_xy parabola -Run the experiment again, and observe how the points are added as they are generated to the plot in the applet window. +Run the experiment again, and observe how the points are added to the plot in the applet window as they are generated. .. tip:: Datasets and applets can both be arranged in groups for organizational purposes. (In fact, so can arguments; see the reference of :meth:`~artiq.language.environment.HasEnvironment.setattr_argument`). For datasets, use a dot (``.``) in names to separate folders. For applets, left-click in the applet list to see the option 'Create Group'. You can drag and drop to move applets in and out of groups, or select a particular group with a click to create new applets in that group. Deselect applets or groups with CTRL+click. @@ -68,7 +70,7 @@ Run the experiment again, and observe how the points are added as they are gener The ARTIQ browser ^^^^^^^^^^^^^^^^^ -ARTIQ also possesses a second GUI, specifically targeted for the manipulation and analysis of datasets, called the ARTIQ browser. It is independent, and does not require either a running master or a core device to operate; a connection to the master is only necessary if you want to upload edited datasets back to the main management system. Open ``results`` in the browser by running: :: +ARTIQ also possesses a second GUI, specifically targeted for the manipulation and analysis of datasets, called the ARTIQ browser. It is standalone, and does not require either a running master or a core device to operate; a connection to the master is only necessary if you want to upload edited datasets back to the main management system. Open ``results`` in the browser by running: :: $ cd ~/artiq-master $ artiq_browser ./results @@ -79,7 +81,7 @@ To open an experiment, click on 'Experiment' at the top left. Observe that inste As described later in :ref:`experiment-scheduling`, only :meth:`~artiq.language.environment.Experiment.run` is obligatory for experiments to implement, and only :meth:`~artiq.language.environment.Experiment.run` is permitted to access hardware; the preparation and analysis stages occur before and after, and are limited to the host machine. The browser allows for re-running the post-experiment :meth:`~artiq.language.environment.Experiment.analyze`, potentially with different arguments or an edited algorithm, while accessing the datasets from opened ``results`` files. -Notably, the browser does not merely act as an HD5 viewer, but also allows the use of ARTIQ applets to plot and view the data. For this, see the lower left dock; applets can be opened, closed, and managed just as they are in the dashboard, once again accessing datasets from ``results``. +Notably, the browser does not merely act as an HDF5 viewer, but also allows the use of ARTIQ applets to plot and view the data. For this, see the lower left dock; applets can be opened, closed, and managed just as they are in the dashboard, once again accessing datasets from ``results``. .. _mgmt-ctlmgr: From a1fb6e1b709fef1089663a7bc207fa4f09174378 Mon Sep 17 00:00:00 2001 From: David Nadlinger Date: Fri, 30 Aug 2024 15:46:56 +0100 Subject: [PATCH 002/111] setup.py: Add missing platformdirs dependency This was introduced in 583a4ceadd, but only added to the Nix flake there. --- setup.py | 1 + 1 file changed, 1 insertion(+) diff --git a/setup.py b/setup.py index f76401ecc..23ff6707c 100755 --- a/setup.py +++ b/setup.py @@ -16,6 +16,7 @@ requirements = [ "python-dateutil", "prettytable", "h5py", "lmdb", "qasync", "pyqtgraph", "pygit2", "llvmlite", "pythonparser", "levenshtein", + "platformdirs", ] console_scripts = [ From 11bab7dadb791bc4dd33ac4f26b929c1824ca3d6 Mon Sep 17 00:00:00 2001 From: David Nadlinger Date: Sat, 31 Aug 2024 21:29:30 +0100 Subject: [PATCH 003/111] gui: Update logo_ver.svg to major version 9 Opened in Inkscape, manually replaced the "8" with a text object showing "9" (Intro, 40pt), converted object to paths, and saved again with default settings. Only committed the part of the file change corresponding to the text. --- artiq/gui/logo_ver.svg | 17 +++++------------ 1 file changed, 5 insertions(+), 12 deletions(-) diff --git a/artiq/gui/logo_ver.svg b/artiq/gui/logo_ver.svg index b98176a18..94f829914 100644 --- a/artiq/gui/logo_ver.svg +++ b/artiq/gui/logo_ver.svg @@ -151,15 +151,8 @@ inkscape:connector-curvature="0" /> + style="fill:#ffffff;fill-opacity:1" /> From 0c1ffa9f4f6a3e7864459923ec4b9cc45f16327a Mon Sep 17 00:00:00 2001 From: architeuthidae Date: Mon, 26 Aug 2024 13:46:48 +0800 Subject: [PATCH 004/111] doc: Add 'Adding custom EEM' section --- doc/manual/extending_rtio.rst | 141 +++++++++++++++++++++++++++++++++- 1 file changed, 138 insertions(+), 3 deletions(-) diff --git a/doc/manual/extending_rtio.rst b/doc/manual/extending_rtio.rst index 770b7c0f2..9c53c6dbe 100644 --- a/doc/manual/extending_rtio.rst +++ b/doc/manual/extending_rtio.rst @@ -5,7 +5,7 @@ Extending RTIO This page is for users who want to extend or modify ARTIQ RTIO. Broadly speaking, one of the core intentions of ARTIQ is to provide a high-level, easy-to-use interface for experimentation, while the infrastructure handles the technological challenges of the high-resolution, timing-critical operations required. Rather than worrying about the details of timing in hardware, users can outline tasks quickly and efficiently in ARTIQ Python, and trust the system to carry out those tasks in real time. It is not normally, or indeed ever, necessary to make modifications on a gateware level. - However, ARTIQ is an open-source project, and welcomes interest and agency from its users, as well as from experienced developers. This page is intended to serve firstly as a broad introduction to the internal structure of ARTIQ, and secondly as a tutorial for how RTIO extensions in ARTIQ can be made. Experience with FPGAs or hardware description languages is not strictly necessary, but additional research on the topic will likely be required to make serious modifications of your own. + However, ARTIQ is an open-source project, and welcomes innovation and contribution from its users, as well as from experienced developers. This page is intended to serve firstly as a broad introduction to the internal structure of ARTIQ, and secondly as a tutorial for how RTIO extensions in ARTIQ can be made. Experience with FPGAs or hardware description languages is not strictly necessary, but additional research on the topic will likely be required to make serious modifications of your own. For instructions on setting up the ARTIQ development environment and on building gateware and firmware binaries, first see :doc:`building_developing` in the main part of the manual. @@ -49,6 +49,8 @@ Experiment kernels themselves -- ARTIQ Python, processed by the ARTIQ compiler a These frameworks are built to be self-contained and extensible. To make additions to the gateware and software, for example, we do not need to make changes to the firmware; we can interact purely with the interfaces provided on either side. +.. _extending-gateware-logic: + Extending gateware logic ------------------------ @@ -61,6 +63,8 @@ Gateware in ARTIQ is housed in ``artiq/gateware`` on the main ARTIQ repository a To change parameters related to particular peripherals, see also the files ``eem.py`` and ``eem_7series.py``, which describe the core device's interface with other EEM cards in Migen terms, and contain ``add_std`` methods that in turn reference specific gateware modules and assign RTIO channels. +.. _adding-phy: + Adding a module to gateware ^^^^^^^^^^^^^^^^^^^^^^^^^^^ @@ -154,6 +158,8 @@ The output ``pad0`` is continuously connected to the value of the ``pad0_o`` reg The module is now capable of accepting RTIO output events and applying them to the hardware outputs. What we can't yet do is generate these output events in an ARTIQ kernel. To do that, we need to add a core device driver. +.. _adding-core-driver: + Adding a core device driver ^^^^^^^^^^^^^^^^^^^^^^^^^^^ @@ -234,7 +240,7 @@ Now, before you can access your new core device driver from a kernel, it must be .. warning:: Channel numbers are assigned sequentially each time ``rtio_channels.append()`` is called. Since we assigned the channel for our linked LEDs in the same location as the old user LEDs, the correct channel number is likely simply the one previously used in your device database for the first LED. In any other case, however, the ``print()`` statement we added to the target file should tell us the exact canonical channel. Search through the console logs produced when generating the gateware to find the line starting with ``Linked LEDs at:``. - Depending on how your device database was written, note that the channel numbers for other peripherals, if they are present, *will have changed*, and :meth:`~artiq.frontend.artiq_ddb_template` will not generate their numbers correctly unless it is edited to match the new assignments of the user LEDs. For a longer-term gateware change, especially the addition of a new EEM card, ``artiq/frontend/artiq_ddb_template.py`` and ``artiq/coredevice/coredevice_generic.schema`` should be edited accordingly, so that system descriptions and device databases can continue to be parsed and generated correctly. + Depending on how your device database was written, note that the channel numbers for other peripherals, if they are present, *will have changed*, and :meth:`~artiq.frontend.artiq_ddb_template` will not generate their numbers correctly unless it is edited to match the new assignments of the user LEDs. For a more long-term gateware change, ``artiq/frontend/artiq_ddb_template.py`` and ``artiq/coredevice/coredevice_generic.schema`` should be edited accordingly, so that system descriptions and device databases can continue to be parsed and generated correctly. See also :ref:`extending-system-description` below. Test experiments ^^^^^^^^^^^^^^^^ @@ -269,4 +275,133 @@ and ``linkup.py``: :: Run these and observe the results. Congratulations! You have successfully constructed an extension to the ARTIQ RTIO. -.. + 'Adding custom EEMs' and 'Merging support' \ No newline at end of file +Adding a custom EEM +------------------- + +.. note:: + Adding a custom EEM to a Kasli or Kasli-SoC system is not much more difficult than adding new gateware logic for existing hardware, and may in some cases be simpler, if no custom PHY is required. On the other hand, modifying hardware in KC705 or ZC706-based systems is a different process, and gateware generation for these boards does not use the files and modules described below. Creating new KC705 or ZC706 variants is not directly addressed in this tutorial. That said, it would begin and end largely in the respective target file, where the variants are defined. + + Non-realtime hardware which does not need to connect directly to the core device or require gateware support should instead be handled through an NDSP, see :doc:`developing_a_ndsp`. This is a more accessible process in general and does not vary based on core device. + +Extending gateware support +^^^^^^^^^^^^^^^^^^^^^^^^^^ + +The first and most important file to look into is ``eem.py``, found in ``artiq/gateware``. This is where the classes for ARTIQ-supported EEM peripherals are defined, and where you can add your own class for a new EEM, following the model of the preexisting classes. + +Your custom EEM class should subclass :class:`artiq.gateware.eem._EEM` and provide the two methods ``io()`` and ``add_std()``. The second, ``add_std()``, will be called to add this EEM to a gateware build. The first is called by ``add_extension()`` in :class:`~artiq.gateware.eem._EEM` itself. Your class should look something like: :: + + class CustomEEM(_EEM): + @staticmethod + def io(*args, **kwargs iostandard=default_iostandard): + io = [ ... ] # A sequence of pad assignments + return io + + @classmethod + def add_std(cls, target, *args, **kwargs): + cls.add_extension(target, *args, **kwargs) # calls CustomEEM.io(*args, **kwargs) + + # Request IO pads that were added in CustomEEM.io() + target.platform.request(...) + + # Add submodule for PHY (pass IO pads in arguments) + phy = ... + phys.append(phy) + target.submodules += phy + + # Add RTIO channel(s) for PHY + target.rtio_channels.append(rtio.Channel.from_phy(...)) + +Note that the pad assignments ``io()`` returns should be in Migen, usually comprised out of Migen ``Subsignal`` and ``Pin`` constructs. The predefined :func:`~artiq.gateware.eem._eem_signal` and :func:`~artiq.gateware.eem._eem_pin` functions (also provided in ``eem.py``) may be useful. Note also that ``add_std()`` covers essentially the same territory as the modifications we simply made directly to the target file for the LED tutorial. Depending on your use case, you may need to write a custom PHY for your hardware, or you may be able to make use of the PHYs ARTIQ already makes available. See :ref:`adding-phy`, if you haven't already. A single EEM may also generate several PHYs and/or claim several RTIO channels. + +Now find the file ``eem_7series.py``, also in ``artiq/gateware``. The functions defined in this file mostly serve as wrappers for ``add_std()``, with some additional interpretation and checks on the parameters. Your own ``peripheral`` function should look something like: :: + + def peripheral_custom(module, peripheral): + ... # (interpret peripheral arguments) + CustomEEM.add_std(module, *args, **kwargs) + +Once you have written this function, add it to the ``peripheral_processors`` dictionary at the end of the file, as: :: + + peripheral_processors["custom_eem"] = peripheral_custom + +Now your EEM is fully supported by the ARTIQ gateware infrastructure. All that remains is to add it to a build configuration. + +.. _extending-system-description: + +Target file and system description +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +In the :ref:`extending-gateware-logic` tutorial above, we made modifications directly to the target file, to hardcode a certain PHY for a certain set of pads. This is reasonable to do in the case of the core device LEDs, which are always present and cannot be rearranged. It is theoretically possible to hardcode the addition of your new EEM in the same way. In this case it would not be necessary to make modifications to ``eem.py`` and ``eem_7series.py``; the pad assignments, requisite PHYs, and RTIO channels could all be defined directly in the target file. This is essentially how things are done for KC705 and ZC706 variants. + +However, with EEM cards, which can be present in different numbers and rearranged at will, it is preferable to be more flexible. This is the reason system description files are used. Assuming you have added your EEM to ``eem.py`` and the ``peripheral_processors`` dictionary, no modifications to the target file are actually necessarily. All Kasli and Kasli-SoC targets already contain the line: :: + + eem_7series.add_peripherals(self, description["peripherals"], iostandard=eem_iostandard) + +In other words, your custom EEM will be automatically included if it is in the ``description`` dictionary, which is interpreted directly from the JSON system description. Simply add an entry to your system description: :: + + { + "type": "custom_eem", + "ports": [0] + # any other args to pass to add_std or io later: + ... + } + +Note however that before a build system descriptions are always checked against the corresponding JSON schema, which you can find as ``coredevice_generic_schema.json`` in ``artiq/coredevice``. Add the new format for your entry here as well, under ``definition``, ``peripheral``, and ``allOf``: :: + + { + "title": "CustomEEM", + "if": { + "properties": { + "type": { + "const": "custom_eem" + } + } + }, + "then": { + "properties": { + "ports": { + "type": "array", + "items": { + "type": "integer" + }, + "minItems": ..., + "maxItems": ... + }, + ... + }, + "required": ["ports", ...] + } + }, + +Now it should be possible to :doc:`build the binaries `, using your system description and its custom entry. + +Device database and driver +^^^^^^^^^^^^^^^^^^^^^^^^^^ + +As usual, before you can use your hardware from a kernel, you will need to add an entry to your device database. You can use one of the existing ARTIQ core drivers, if applicable, or you can write your own custom driver, as we did in :ref:`adding-core-driver`. + +There are a few options to determine the correct channel number. You can figure it out from the structure of your system description; you can add a print statement to ``add_std()``; or, most preferably, you can add support for your custom EEM in :mod:`~artiq.frontend.artiq_ddb_template`, so that the channel number can be handled automatically as it is for other peripherals. + +The relevant file is in ``artiq/frontend``, named simply ``artiq_ddb_template.py``. You will want to add a method within ``PeripheralManager``, in the format: :: + + def process_custom_eem(self, rtio_offset, peripheral): + self.gen(""" + device_db["{name}"] = {{ + "type": "local", + "module": "artiq.coredevice.custom_eem", + "class": "CustomDriver", + "arguments": {{"channel": 0x{channel:06x}}} + }}""", + name=self.get_name("custom_eem"), + channel=rtio_offset + next(channel)) + return next(channel) + +Further arguments can be passed on through ``arguments`` if necessary. Note that the peripheral manager's ``process`` method chooses which method to use by performing a simple string check, so your ``process_`` method *must* use the same name for your custom hardware as given in the system description's ``"type"``. + +You should now be able to use :mod:`~artiq.frontend.artiq_ddb_template` to generate your device database, and from there, compile and run experiments with your new hardware. Congratulations! + +Merging support +--------------- + +Being an open-source project, ARTIQ welcomes contributions from outside sources. If you have successfully integrated additional gateware or new hardware into ARTIQ, and you think this might be useful to other ARTIQ users in the community, you might consider merging support -- having your additions incorporated into the canonical ARTIQ codebase. See `this pull request `_ for one example of such a community addition. + +Merging support also means the opportunity to have your code reviewed by experts, and if your addition is accepted, that maintaining these additions and keeping them up-to-date through new ARTIQ versions may be handled by the developers of ARTIQ directly, instead of being solely your responsibility. Clean up your code, test it well, be sure that it plays well with existing ARTIQ features and interfaces, and follow the `contribution guidelines `_. Your effort is appreciated! From 59121428362197384e22a46217b917af33eb371b Mon Sep 17 00:00:00 2001 From: Charles Baynham Date: Fri, 6 Sep 2024 13:33:58 +0100 Subject: [PATCH 005/111] nix: add paramiko to boards environment for lightweight flashing --- flake.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flake.nix b/flake.nix index 50aea1352..528663cd2 100644 --- a/flake.nix +++ b/flake.nix @@ -428,7 +428,7 @@ devShells.x86_64-linux.boards = pkgs.mkShell { name = "artiq-boards-shell"; buildInputs = [ - (pkgs.python3.withPackages(ps: with packages.x86_64-linux; [ migen misoc artiq ps.packaging ])) + (pkgs.python3.withPackages(ps: with packages.x86_64-linux; [ migen misoc artiq ps.packaging ps.paramiko ])) rust pkgs.llvmPackages_15.clang-unwrapped pkgs.llvm_15 From 4e654011c3d86b64e1a4b102002ed919db63066e Mon Sep 17 00:00:00 2001 From: David Nadlinger Date: Mon, 2 Sep 2024 18:35:41 +0100 Subject: [PATCH 006/111] gui/models: Add DictSyncModel default args to match C++ [nfc] --- artiq/gui/models.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/artiq/gui/models.py b/artiq/gui/models.py index 810fa4505..a6a316367 100644 --- a/artiq/gui/models.py +++ b/artiq/gui/models.py @@ -84,20 +84,20 @@ class DictSyncModel(QtCore.QAbstractTableModel): key=lambda k: self.sort_key(k, self.backing_store[k])) QtCore.QAbstractTableModel.__init__(self) - def rowCount(self, parent): + def rowCount(self, parent=QtCore.QModelIndex()): return len(self.backing_store) - def columnCount(self, parent): + def columnCount(self, parent=QtCore.QModelIndex()): return len(self.headers) - def data(self, index, role): + def data(self, index, role=QtCore.Qt.ItemDataRole.DisplayRole): if not index.isValid() or role != QtCore.Qt.ItemDataRole.DisplayRole: return None else: k = self.row_to_key[index.row()] return self.convert(k, self.backing_store[k], index.column()) - def headerData(self, col, orientation, role): + def headerData(self, col, orientation, role=QtCore.Qt.ItemDataRole.DisplayRole): if (orientation == QtCore.Qt.Orientation.Horizontal and role == QtCore.Qt.ItemDataRole.DisplayRole): return self.headers[col] From 6a7926ddebd62bd4b5047f48c49eb5656e782d54 Mon Sep 17 00:00:00 2001 From: David Nadlinger Date: Thu, 5 Sep 2024 16:35:17 +0100 Subject: [PATCH 007/111] dashboard: Allow moving/hiding of Schedule view columns MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This is very handy to make good use of screen real estate, as e.g. file path and Git revision are often not as important to judge the state of the system at a glance – particularly, as the schedule entries can then fit on a single line each. --- artiq/dashboard/schedule.py | 13 ++++++++-- artiq/gui/tools.py | 47 ++++++++++++++++++++++++++++++++++++- 2 files changed, 57 insertions(+), 3 deletions(-) diff --git a/artiq/dashboard/schedule.py b/artiq/dashboard/schedule.py index 8ea213239..ac77398d7 100644 --- a/artiq/dashboard/schedule.py +++ b/artiq/dashboard/schedule.py @@ -6,6 +6,7 @@ import logging from PyQt6 import QtCore, QtWidgets, QtGui from artiq.gui.models import DictSyncModel +from artiq.gui.tools import SelectableColumnTableView from artiq.tools import elide @@ -66,7 +67,7 @@ class ScheduleDock(QtWidgets.QDockWidget): self.schedule_ctl = schedule_ctl - self.table = QtWidgets.QTableView() + self.table = SelectableColumnTableView() self.table.setSelectionBehavior(QtWidgets.QAbstractItemView.SelectionBehavior.SelectRows) self.table.setSelectionMode(QtWidgets.QAbstractItemView.SelectionMode.SingleSelection) self.table.verticalHeader().setSectionResizeMode( @@ -104,6 +105,9 @@ class ScheduleDock(QtWidgets.QDockWidget): h.resizeSection(6, 20 * cw) h.resizeSection(7, 20 * cw) + # Allow user to reorder or disable columns. + h.setSectionsMovable(True) + def set_model(self, model): self.table_model = model self.table.setModel(self.table_model) @@ -154,4 +158,9 @@ class ScheduleDock(QtWidgets.QDockWidget): return bytes(self.table.horizontalHeader().saveState()) def restore_state(self, state): - self.table.horizontalHeader().restoreState(QtCore.QByteArray(state)) + h = self.table.horizontalHeader() + h.restoreState(QtCore.QByteArray(state)) + + # The state includes the sectionsMovable property, so set it again to be able to + # deal with pre-existing save files from when we used not to enable it. + h.setSectionsMovable(True) diff --git a/artiq/gui/tools.py b/artiq/gui/tools.py index 0fc9e4615..63b228632 100644 --- a/artiq/gui/tools.py +++ b/artiq/gui/tools.py @@ -1,7 +1,7 @@ import asyncio import logging -from PyQt6 import QtCore, QtWidgets +from PyQt6 import QtCore, QtGui, QtWidgets class DoubleClickLineEdit(QtWidgets.QLineEdit): @@ -88,6 +88,51 @@ class LayoutWidget(QtWidgets.QWidget): self.layout.addWidget(item, row, col, rowspan, colspan) +class SelectableColumnTableView(QtWidgets.QTableView): + """A QTableView packaged up with a header row context menu that allows users to + show/hide columns using checkable entries. + + By default, all columns are shown. If only one shown column remains, the entry is + disabled to prevent a situation where no columns are shown, which might be confusing + to the user. + + Qt considers whether columns are shown to be part of the header state, i.e. it is + included in saveState()/restoreState(). + """ + + def __init__(self): + super().__init__() + + self.horizontalHeader().setContextMenuPolicy( + QtCore.Qt.ContextMenuPolicy.CustomContextMenu) + self.horizontalHeader().customContextMenuRequested.connect( + self.show_header_context_menu) + + def show_header_context_menu(self, pos): + menu = QtWidgets.QMenu(self) + + num_columns_total = self.model().columnCount() + num_columns_shown = sum( + (not self.isColumnHidden(i)) for i in range(num_columns_total)) + for i in range(num_columns_total): + name = self.model().headerData(i, QtCore.Qt.Orientation.Horizontal) + action = QtGui.QAction(name, self) + action.setCheckable(True) + + is_currently_hidden = self.isColumnHidden(i) + action.setChecked(not is_currently_hidden) + if not is_currently_hidden: + if num_columns_shown == 1: + # Don't allow hiding of the last visible column. + action.setEnabled(False) + + action.triggered.connect( + lambda checked, i=i: self.setColumnHidden(i, not checked)) + menu.addAction(action) + + menu.exec(self.horizontalHeader().mapToGlobal(pos)) + + async def get_open_file_name(parent, caption, dir, filter): """like QtWidgets.QFileDialog.getOpenFileName(), but a coroutine""" dialog = QtWidgets.QFileDialog(parent, caption, dir, filter) From ed3b60b270ba3136adeb054d784e5fd318c13dc5 Mon Sep 17 00:00:00 2001 From: David Nadlinger Date: Fri, 6 Sep 2024 16:32:07 +0100 Subject: [PATCH 008/111] RELEASE_NOTES: group dashboard changes, mention Schedule column hiding --- RELEASE_NOTES.rst | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/RELEASE_NOTES.rst b/RELEASE_NOTES.rst index aa12c7816..d976b1510 100644 --- a/RELEASE_NOTES.rst +++ b/RELEASE_NOTES.rst @@ -6,8 +6,11 @@ Release notes ARTIQ-9 (Unreleased) -------------------- -* GUI state files are now automatically backed up upon successful loading. -* Zotino monitoring in the dashboard now displays the values in volts. +* Dashboard: + - Zotino monitoring now displays the values in volts. + - Schedule display columns can now be reordered and shown/hidden using the table + header context menu. + - State files are now automatically backed up upon successful loading. * afws_client now uses the "happy eyeballs" algorithm (RFC 6555) for a faster and more reliable connection to the server. * The Zadig driver installer was added to the MSYS2 offline installer. From 03606f4d7ee07cf0eb22a286c8246caf644dbab8 Mon Sep 17 00:00:00 2001 From: Adam Chatterley Date: Mon, 9 Sep 2024 13:56:22 +0200 Subject: [PATCH 009/111] core: Add option to detect kernel invariants There is already invariant detection in the the compiler, this just exposes it by adding a flag to Core Signed-off-by: Adam Chatterley --- artiq/coredevice/core.py | 9 +++++++-- artiq/dashboard/experiments.py | 1 + 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/artiq/coredevice/core.py b/artiq/coredevice/core.py index 1e9527ba2..a7d41cc92 100644 --- a/artiq/coredevice/core.py +++ b/artiq/coredevice/core.py @@ -85,6 +85,8 @@ class Core: (optional). :param analyze_at_run_end: automatically trigger the core device analyzer proxy after the Experiment's run stage finishes. + :param report_invariants: report variables which are not changed inside + kernels and are thus candidates for inclusion in kernel_invariants """ kernel_invariants = { @@ -95,7 +97,8 @@ class Core: host, ref_period, analyzer_proxy=None, analyze_at_run_end=False, ref_multiplier=8, - target="rv32g", satellite_cpu_targets={}): + target="rv32g", satellite_cpu_targets={}, + report_invariants=False): self.ref_period = ref_period self.ref_multiplier = ref_multiplier self.satellite_cpu_targets = satellite_cpu_targets @@ -107,6 +110,7 @@ class Core: self.comm = CommKernel(host) self.analyzer_proxy_name = analyzer_proxy self.analyze_at_run_end = analyze_at_run_end + self.report_invariants = report_invariants self.first_run = True self.dmgr = dmgr @@ -139,7 +143,8 @@ class Core: module = Module(stitcher, ref_period=self.ref_period, - attribute_writeback=attribute_writeback) + attribute_writeback=attribute_writeback, + remarks=self.report_invariants) target = target if target is not None else self.target_cls() library = target.compile_and_link([module]) diff --git a/artiq/dashboard/experiments.py b/artiq/dashboard/experiments.py index 53323b2d4..4be556d79 100644 --- a/artiq/dashboard/experiments.py +++ b/artiq/dashboard/experiments.py @@ -187,6 +187,7 @@ class _ExperimentDock(QtWidgets.QMdiSubWindow): devarg_override.lineEdit().setPlaceholderText("Override device arguments") devarg_override.lineEdit().setClearButtonEnabled(True) devarg_override.insertItem(0, "core:analyze_at_run_end=True") + devarg_override.insertItem(1, "core:report_invariants=True") self.layout.addWidget(devarg_override, 2, 3) devarg_override.setCurrentText(options["devarg_override"]) From 6f70d629cf26ab10788f737f0bfc18c79c3bc007 Mon Sep 17 00:00:00 2001 From: Adam Chatterley Date: Mon, 9 Sep 2024 14:26:28 +0200 Subject: [PATCH 010/111] doc: Add kernel invariant detection description Signed-off-by: Adam Chatterley --- doc/manual/compiler.rst | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/doc/manual/compiler.rst b/doc/manual/compiler.rst index 0b020d6bf..29f623561 100644 --- a/doc/manual/compiler.rst +++ b/doc/manual/compiler.rst @@ -253,3 +253,7 @@ In the synthetic example above, the compiler will be able to detect that the res for _ in range(100): delay_mu(precomputed_delay_mu) self.worker.work() + +Kernel invariants are defined for every object by the ``kernel_invariants`` atttribute, which is a set containing the names of every invariant attribute of this object. + +At compile time it is possible to automatically detect attributes that are never altered in a kernel, and thus may be good candidates for inclusion into ``kernel_invariants``. This is done by specifying ``report_invariants=True`` when initializing the core device driver (in the dashboard you can do this using the "Override device arguments" option). \ No newline at end of file From d8a3da449e2b5d91de945d5a929814366c82ea4a Mon Sep 17 00:00:00 2001 From: Adam Chatterley Date: Mon, 9 Sep 2024 16:33:05 +0200 Subject: [PATCH 011/111] RELEASE_NOTES: invariant detection --- RELEASE_NOTES.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/RELEASE_NOTES.rst b/RELEASE_NOTES.rst index d976b1510..f27e68040 100644 --- a/RELEASE_NOTES.rst +++ b/RELEASE_NOTES.rst @@ -17,6 +17,7 @@ ARTIQ-9 (Unreleased) * Fastino monitoring with Moninj is now supported. * Qt6 support. * Python 3.12 support. +* Compiler can now give automatic suggestions for ``kernel_invariants``. ARTIQ-8 ------- From 38e0d6b9536d1d305d29a8f4751d8a2cd5fcd667 Mon Sep 17 00:00:00 2001 From: Simon Renblad Date: Tue, 10 Sep 2024 11:37:12 +0800 Subject: [PATCH 012/111] moninj: fix pyqt6 context menu exec --- artiq/dashboard/moninj.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/artiq/dashboard/moninj.py b/artiq/dashboard/moninj.py index 201101838..e5218d8a1 100644 --- a/artiq/dashboard/moninj.py +++ b/artiq/dashboard/moninj.py @@ -847,7 +847,7 @@ class _MonInjDock(QDockWidgetCloseDetect): delete_action = QtGui.QAction("Delete widget", menu) delete_action.triggered.connect(partial(self.delete_widget, index)) menu.addAction(delete_action) - menu.exec_(self.flow.mapToGlobal(pos)) + menu.exec(self.flow.mapToGlobal(pos)) def delete_all_widgets(self): for index in reversed(range(self.flow.count())): From 41203df73547d47c1ca61c645bad8d3b1bd70cb1 Mon Sep 17 00:00:00 2001 From: Simon Renblad Date: Fri, 13 Sep 2024 13:24:11 +0800 Subject: [PATCH 013/111] almazny: fix missing delay import --- artiq/coredevice/almazny.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/artiq/coredevice/almazny.py b/artiq/coredevice/almazny.py index 2cc21697e..9f3a8ce58 100644 --- a/artiq/coredevice/almazny.py +++ b/artiq/coredevice/almazny.py @@ -1,4 +1,4 @@ -from artiq.language.core import kernel, portable +from artiq.language.core import kernel, portable, delay from artiq.language.units import us from numpy import int32 From 40227421a88a90956f0b1fa3f69c45eee97510a8 Mon Sep 17 00:00:00 2001 From: Simon Renblad Date: Fri, 9 Aug 2024 13:44:34 +0800 Subject: [PATCH 014/111] run idle kernel on flash --- artiq/firmware/runtime/main.rs | 13 +++++++++---- artiq/firmware/runtime/mgmt.rs | 31 +++++++++++++++++++++++++------ artiq/firmware/runtime/session.rs | 19 +++++++++++++------ 3 files changed, 47 insertions(+), 16 deletions(-) diff --git a/artiq/firmware/runtime/main.rs b/artiq/firmware/runtime/main.rs index ca942caa5..dd52aabaf 100644 --- a/artiq/firmware/runtime/main.rs +++ b/artiq/firmware/runtime/main.rs @@ -29,9 +29,10 @@ extern crate riscv; extern crate tar_no_std; use alloc::collections::BTreeMap; -use core::cell::RefCell; +use core::cell::{RefCell, Cell}; use core::convert::TryFrom; use smoltcp::wire::HardwareAddress; +use urc::Urc; use board_misoc::{csr, ident, clock, spiflash, config, net_settings, pmp, boot}; #[cfg(has_ethmac)] @@ -196,6 +197,7 @@ fn startup() { let ddma_mutex = sched::Mutex::new(); let subkernel_mutex = sched::Mutex::new(); + let restart_idle = Urc::new(Cell::new(false)); let mut scheduler = sched::Scheduler::new(interface); let io = scheduler.io(); @@ -205,15 +207,18 @@ fn startup() { } rtio_mgt::startup(&io, &aux_mutex, &drtio_routing_table, &up_destinations, &ddma_mutex, &subkernel_mutex); - - io.spawn(4096, mgmt::thread); + { + let restart_idle = restart_idle.clone(); + io.spawn(4096, move |io| { mgmt::thread(io, &restart_idle) }); + } { let aux_mutex = aux_mutex.clone(); let drtio_routing_table = drtio_routing_table.clone(); let up_destinations = up_destinations.clone(); let ddma_mutex = ddma_mutex.clone(); let subkernel_mutex = subkernel_mutex.clone(); - io.spawn(32768, move |io| { session::thread(io, &aux_mutex, &drtio_routing_table, &up_destinations, &ddma_mutex, &subkernel_mutex) }); + let restart_idle = restart_idle.clone(); + io.spawn(32768, move |io| { session::thread(io, &aux_mutex, &drtio_routing_table, &up_destinations, &ddma_mutex, &subkernel_mutex, &restart_idle) }); } #[cfg(any(has_rtio_moninj, has_drtio))] { diff --git a/artiq/firmware/runtime/mgmt.rs b/artiq/firmware/runtime/mgmt.rs index 449ed91be..5b6fc6d01 100644 --- a/artiq/firmware/runtime/mgmt.rs +++ b/artiq/firmware/runtime/mgmt.rs @@ -1,10 +1,12 @@ use log::{self, LevelFilter}; +use core::cell::Cell; use io::{Write, ProtoWrite, Error as IoError}; use board_misoc::{config, spiflash}; use logger_artiq::BufferLogger; use mgmt_proto::*; use sched::{Io, TcpListener, TcpStream, Error as SchedError}; +use urc::Urc; impl From for Error { fn from(value: SchedError) -> Error { @@ -12,7 +14,7 @@ impl From for Error { } } -fn worker(io: &Io, stream: &mut TcpStream) -> Result<(), Error> { +fn worker(io: &Io, stream: &mut TcpStream, restart_idle: &Urc>) -> Result<(), Error> { read_magic(stream)?; Write::write_all(stream, "e".as_bytes())?; info!("new connection from {}", stream.remote_endpoint()); @@ -84,20 +86,36 @@ fn worker(io: &Io, stream: &mut TcpStream) -> Result<(), Error> { } Request::ConfigWrite { ref key, ref value } => { match config::write(key, value) { - Ok(_) => Reply::Success.write_to(stream), + Ok(_) => { + if key == "idle_kernel" { + io.until(|| !restart_idle.get())?; + restart_idle.set(true); + } + Reply::Success.write_to(stream) + }, Err(_) => Reply::Error.write_to(stream) }?; } Request::ConfigRemove { ref key } => { match config::remove(key) { - Ok(()) => Reply::Success.write_to(stream), + Ok(()) => { + if key == "idle_kernel" { + io.until(|| !restart_idle.get())?; + restart_idle.set(true); + } + Reply::Success.write_to(stream) + }, Err(_) => Reply::Error.write_to(stream) }?; } Request::ConfigErase => { match config::erase() { - Ok(()) => Reply::Success.write_to(stream), + Ok(()) => { + io.until(|| !restart_idle.get())?; + restart_idle.set(true); + Reply::Success.write_to(stream) + }, Err(_) => Reply::Error.write_to(stream) }?; } @@ -117,16 +135,17 @@ fn worker(io: &Io, stream: &mut TcpStream) -> Result<(), Error> { } } -pub fn thread(io: Io) { +pub fn thread(io: Io, restart_idle: &Urc>) { let listener = TcpListener::new(&io, 8192); listener.listen(1380).expect("mgmt: cannot listen"); info!("management interface active"); loop { let stream = listener.accept().expect("mgmt: cannot accept").into_handle(); + let restart_idle = restart_idle.clone(); io.spawn(4096, move |io| { let mut stream = TcpStream::from_handle(&io, stream); - match worker(&io, &mut stream) { + match worker(&io, &mut stream, &restart_idle) { Ok(()) => (), Err(Error::Io(IoError::UnexpectedEnd)) => (), Err(err) => error!("aborted: {}", err) diff --git a/artiq/firmware/runtime/session.rs b/artiq/firmware/runtime/session.rs index b69e47e09..579b2094c 100644 --- a/artiq/firmware/runtime/session.rs +++ b/artiq/firmware/runtime/session.rs @@ -839,7 +839,7 @@ fn flash_kernel_worker(io: &Io, aux_mutex: &Mutex, routing_table: &drtio_routing::RoutingTable, up_destinations: &Urc>, ddma_mutex: &Mutex, subkernel_mutex: &Mutex, congress: &mut Congress, - config_key: &str) -> Result<(), Error> { + config_key: &str, restart_idle: Option<&Urc>>) -> Result<(), Error> { let mut session = Session::new(congress); config::read(config_key, |result| { @@ -859,11 +859,16 @@ fn flash_kernel_worker(io: &Io, aux_mutex: &Mutex, } })?; kern_run(&mut session)?; - loop { if !rpc_queue::empty() { unexpected!("unexpected background RPC in flash kernel") } + + if let Some(r_idle) = restart_idle { + if r_idle.get() { + return Err(Error::KernelNotFound) + } + } if mailbox::receive() != 0 { if process_kern_message(io, aux_mutex, routing_table, up_destinations, ddma_mutex, subkernel_mutex, None, &mut session)? { @@ -897,7 +902,7 @@ fn respawn(io: &Io, handle: &mut Option, f: F) pub fn thread(io: Io, aux_mutex: &Mutex, routing_table: &Urc>, up_destinations: &Urc>, - ddma_mutex: &Mutex, subkernel_mutex: &Mutex) { + ddma_mutex: &Mutex, subkernel_mutex: &Mutex, restart_idle: &Urc>) { let listener = TcpListener::new(&io, 65535); listener.listen(1381).expect("session: cannot listen"); info!("accepting network sessions"); @@ -910,7 +915,7 @@ pub fn thread(io: Io, aux_mutex: &Mutex, let mut congress = congress.borrow_mut(); info!("running startup kernel"); match flash_kernel_worker(&io, &aux_mutex, &routing_table, &up_destinations, - ddma_mutex, subkernel_mutex, &mut congress, "startup_kernel") { + ddma_mutex, subkernel_mutex, &mut congress, "startup_kernel", None) { Ok(()) => info!("startup kernel finished"), Err(Error::KernelNotFound) => @@ -994,11 +999,12 @@ pub fn thread(io: Io, aux_mutex: &Mutex, let congress = congress.clone(); let ddma_mutex = ddma_mutex.clone(); let subkernel_mutex = subkernel_mutex.clone(); + let restart_idle = restart_idle.clone(); respawn(&io, &mut kernel_thread, move |io| { let routing_table = routing_table.borrow(); let mut congress = congress.borrow_mut(); match flash_kernel_worker(&io, &aux_mutex, &routing_table, &up_destinations, - &ddma_mutex, &subkernel_mutex, &mut *congress, "idle_kernel") { + &ddma_mutex, &subkernel_mutex, &mut *congress, "idle_kernel", Some(&restart_idle)) { Ok(()) => info!("idle kernel finished, standing by"), Err(Error::Protocol(host::Error::Io( @@ -1010,7 +1016,8 @@ pub fn thread(io: Io, aux_mutex: &Mutex, } Err(Error::KernelNotFound) => { debug!("no idle kernel found"); - while io.relinquish().is_ok() {} + while !restart_idle.get() && io.relinquish().is_ok() {} + restart_idle.set(false); } Err(err) => { error!("idle kernel aborted: {}", err); From 1d093d0bce7c23eecb4aa4a489263e66ca157124 Mon Sep 17 00:00:00 2001 From: Simon Renblad Date: Thu, 5 Sep 2024 10:16:25 +0800 Subject: [PATCH 015/111] RELEASE_NOTES: restart idle kernels on flash --- RELEASE_NOTES.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/RELEASE_NOTES.rst b/RELEASE_NOTES.rst index f27e68040..9037d071b 100644 --- a/RELEASE_NOTES.rst +++ b/RELEASE_NOTES.rst @@ -18,6 +18,7 @@ ARTIQ-9 (Unreleased) * Qt6 support. * Python 3.12 support. * Compiler can now give automatic suggestions for ``kernel_invariants``. +* Idle kernels now restart when written with ``artiq_coremgmt`` and stop when erased/removed from config. ARTIQ-8 ------- From 5b72a1faa5194144e1fb3e1fea3fb8fd983f41d8 Mon Sep 17 00:00:00 2001 From: newell Date: Sun, 29 Sep 2024 22:50:58 -0700 Subject: [PATCH 016/111] Fix typo --- doc/manual/getting_started_mgmt.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/manual/getting_started_mgmt.rst b/doc/manual/getting_started_mgmt.rst index 1a243365f..d5d3cb722 100644 --- a/doc/manual/getting_started_mgmt.rst +++ b/doc/manual/getting_started_mgmt.rst @@ -188,7 +188,7 @@ We will use our current ``repository`` folder as the working directory for makin $ mv repository ~/artiq-work $ mkdir repository $ cd repository - $ git init bare + $ git init --bare Now initialize a regular (non-bare) Git repository in our working directory: :: From 5b63cbe9862e954f4cb5b3abe7da0bc4d261489f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Bourdeauducq?= Date: Mon, 30 Sep 2024 14:58:02 +0800 Subject: [PATCH 017/111] flake: update qasync, sync with nixpkgs --- flake.nix | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/flake.nix b/flake.nix index 528663cd2..4ee507f74 100644 --- a/flake.nix +++ b/flake.nix @@ -69,15 +69,21 @@ qasync = pkgs.python3Packages.buildPythonPackage rec { pname = "qasync"; - version = "0.25.0"; + version = "0.27.1"; + format = "pyproject"; src = pkgs.fetchFromGitHub { owner = "CabbageDevelopment"; repo = "qasync"; - rev = "v${version}"; - sha256 = "sha256-lfH8FNA8cP7dmxR+ihoe2Gr8uOxXHdqn1AhNLIkX5ko="; + rev = "refs/tags/v${version}"; + sha256 = "sha256-oXzwilhJ1PhodQpOZjnV9gFuoDy/zXWva9LhhK3T00g="; }; + postPatch = '' + rm qasync/_windows.py # Ignoring it is not taking effect and it will not be used on Linux + ''; + buildInputs = [ pkgs.python3Packages.poetry-core ]; propagatedBuildInputs = [ pkgs.python3Packages.pyqt6 ]; - nativeCheckInputs = [ pkgs.python3Packages.pytest-runner pkgs.python3Packages.pytestCheckHook ]; + checkInputs = [ pkgs.python3Packages.pytestCheckHook ]; + pythonImportsCheck = [ "qasync" ]; disabledTestPaths = [ "tests/test_qeventloop.py" ]; }; From 21cc0f7273aeec2e831eca04ace95a2681acc03c Mon Sep 17 00:00:00 2001 From: Sebastien Bourdeauducq Date: Mon, 30 Sep 2024 14:58:58 +0800 Subject: [PATCH 018/111] flake: update dependencies --- flake.lock | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/flake.lock b/flake.lock index 6409cf8bb..05e74feea 100644 --- a/flake.lock +++ b/flake.lock @@ -44,11 +44,11 @@ }, "nixpkgs": { "locked": { - "lastModified": 1724224976, - "narHash": "sha256-Z/ELQhrSd7bMzTO8r7NZgi9g5emh+aRKoCdaAv5fiO0=", + "lastModified": 1727348695, + "narHash": "sha256-J+PeFKSDV+pHL7ukkfpVzCOO7mBSrrpJ3svwBFABbhI=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "c374d94f1536013ca8e92341b540eba4c22f9c62", + "rev": "1925c603f17fc89f4c8f6bf6f631a802ad85d784", "type": "github" }, "original": { @@ -97,11 +97,11 @@ ] }, "locked": { - "lastModified": 1717637367, - "narHash": "sha256-4mSm9wl5EMgzzrW6w86IDUevkEOT99FESHGcxcyQbD0=", + "lastModified": 1724921939, + "narHash": "sha256-/S5iip1LHLiCP2VY7PwClDteP9ZMRZvzzKR1LZuV3fs=", "owner": "m-labs", "repo": "sipyco", - "rev": "02b96ec2473a3c3d3c980899de2564ddce949dab", + "rev": "32ddd78ff3641b75054793ea0d5681c951766754", "type": "github" }, "original": { @@ -113,11 +113,11 @@ "src-migen": { "flake": false, "locked": { - "lastModified": 1724304798, - "narHash": "sha256-tQ02N0eXY5W/Z7CrOy3Cu4WjDZDQWb8hYlzsFzr3Mus=", + "lastModified": 1727677091, + "narHash": "sha256-Zg3SQnTwMM/VkOGKogbPyuCC2NhLy8HB2SPEUWWNgCU=", "owner": "m-labs", "repo": "migen", - "rev": "832a7240ba32af9cbd4fdd519ddcb4f912534726", + "rev": "c19ae9f8ae162ffe2d310a92bfce53ac2a821bc8", "type": "github" }, "original": { From 333623e24bdec00783bc89c1e8b6b49a74bc9e1c Mon Sep 17 00:00:00 2001 From: Florian Agbuya Date: Tue, 1 Oct 2024 13:04:42 +0800 Subject: [PATCH 019/111] flake: update asyncserial Signed-off-by: Florian Agbuya --- flake.nix | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/flake.nix b/flake.nix index 4ee507f74..e8b9dbe54 100644 --- a/flake.nix +++ b/flake.nix @@ -195,8 +195,8 @@ src = pkgs.fetchFromGitHub { owner = "m-labs"; repo = "asyncserial"; - rev = "d95bc1d6c791b0e9785935d2f62f628eb5cdf98d"; - sha256 = "0yzkka9jk3612v8gx748x6ziwykq5lr7zmr9wzkcls0v2yilqx9k"; + rev = "446559fec892a556876b17d17f182ae9647d5952"; + sha256 = "sha256-WExmgh55sTH2w7wV3i96J1F1FN7L5rX3L/Ayvt2Kw/g="; }; propagatedBuildInputs = [ pkgs.python3Packages.pyserial ]; }; From b751e5455b59a6b14d72003d149cb744f2d8fe02 Mon Sep 17 00:00:00 2001 From: newell Date: Sat, 5 Oct 2024 20:58:17 -0700 Subject: [PATCH 020/111] ebaz4205 docs --- doc/manual/core_device.rst | 5 +++++ doc/manual/flashing.rst | 6 +++--- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/doc/manual/core_device.rst b/doc/manual/core_device.rst index bdd311782..ef4859141 100644 --- a/doc/manual/core_device.rst +++ b/doc/manual/core_device.rst @@ -135,6 +135,11 @@ Common KC705 problems * When connected, the CLOCK adapter breaks the JTAG chain due to TDI not being connected to TDO on the FMC mezzanine. * On some boards, the JTAG USB connector is not correctly soldered. +EBAZ4205 +^^^^^^^^ + +The `EBAZ4205 `_ Zynq-SoC control card, originally used in the Ebit E9+ BTC miner, is a low-cost development board (around $20-$30 USD), making it an ideal option for experimenting with ARTIQ. To use the EBAZ4205, it's important to carefully follow the board documentation to configure it to boot from the SD card, as network booting via ``artiq_netboot`` is currently unsupported. This is because the Ethernet PHY is routed through the EMIO, requiring the FPGA to be programmed before the board can establish a network connection. + VADJ """" diff --git a/doc/manual/flashing.rst b/doc/manual/flashing.rst index 776169b29..715cd63b0 100644 --- a/doc/manual/flashing.rst +++ b/doc/manual/flashing.rst @@ -25,7 +25,7 @@ Installing and configuring OpenOCD ---------------------------------- .. warning:: - These instructions are not applicable to Zynq devices (Kasli-SoC or ZC706), which do not use the utility :mod:`~artiq.frontend.artiq_flash`. If your core device is a Zynq device, skip straight to :ref:`writing-flash`. + These instructions are not applicable to Zynq devices (Kasli-SoC, ZC706 or EBAZ4205), which do not use the utility :mod:`~artiq.frontend.artiq_flash`. If your core device is a Zynq device, skip straight to :ref:`writing-flash`. ARTIQ supplies the utility :mod:`~artiq.frontend.artiq_flash`, which uses OpenOCD to write the binary images into an FPGA board's flash memory. For both Nix and MSYS2, OpenOCD are included with the installation by default. Note that in the case of Nix this is the package ``artiq.openocd-bscanspi`` and not ``pkgs.openocd``; the second is OpenOCD from the Nix package collection, which does not support ARTIQ/Sinara boards. @@ -78,9 +78,9 @@ On Windows Writing the flash ----------------- -First ensure the board is connected to your computer. In the case of Kasli, the JTAG adapter is integrated into the Kasli board; for flashing (and debugging) you can simply connect your computer to the micro-USB connector on the Kasli front panel. For Kasli-SoC, which uses :mod:`~artiq.frontend.artiq_coremgmt` to flash over network, an Ethernet connection and an IP address, supplied either with the ``-D`` option or in your :ref:`device database `, are sufficient. +First ensure the board is connected to your computer. In the case of Kasli, the JTAG adapter is integrated into the Kasli board; for flashing (and debugging) you can simply connect your computer to the micro-USB connector on the Kasli front panel. For Kasli-SoC, ZC706, or EBAZ4205, which use :mod:`~artiq.frontend.artiq_coremgmt` to flash over network, an Ethernet connection and an IP address, supplied either with the ``-D`` option or in your :ref:`device database `, are sufficient. -For Kasli-SoC or ZC706: +For Kasli-SoC, ZC706 or EBAZ4205: :: $ artiq_coremgmt [-D IP_address] config write -f boot /boot.bin From 049ef9022016f05119454335972d328b9e3ca539 Mon Sep 17 00:00:00 2001 From: newell Date: Sat, 5 Oct 2024 21:36:38 -0700 Subject: [PATCH 021/111] Update RELEASE_NOTES.rst --- RELEASE_NOTES.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/RELEASE_NOTES.rst b/RELEASE_NOTES.rst index 9037d071b..1b374d1f5 100644 --- a/RELEASE_NOTES.rst +++ b/RELEASE_NOTES.rst @@ -19,6 +19,7 @@ ARTIQ-9 (Unreleased) * Python 3.12 support. * Compiler can now give automatic suggestions for ``kernel_invariants``. * Idle kernels now restart when written with ``artiq_coremgmt`` and stop when erased/removed from config. +* New support for the EBAZ4205 Zynq-SoC control card. ARTIQ-8 ------- From 6afbd90c597a6a08ed853ba6a761779b48c1da08 Mon Sep 17 00:00:00 2001 From: Sebastien Bourdeauducq Date: Tue, 8 Oct 2024 15:11:34 +0800 Subject: [PATCH 022/111] update dependencies --- flake.lock | 12 ++++++------ flake.nix | 6 +++--- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/flake.lock b/flake.lock index 05e74feea..517aaecde 100644 --- a/flake.lock +++ b/flake.lock @@ -44,11 +44,11 @@ }, "nixpkgs": { "locked": { - "lastModified": 1727348695, - "narHash": "sha256-J+PeFKSDV+pHL7ukkfpVzCOO7mBSrrpJ3svwBFABbhI=", + "lastModified": 1728241625, + "narHash": "sha256-yumd4fBc/hi8a9QgA9IT8vlQuLZ2oqhkJXHPKxH/tRw=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "1925c603f17fc89f4c8f6bf6f631a802ad85d784", + "rev": "c31898adf5a8ed202ce5bea9f347b1c6871f32d1", "type": "github" }, "original": { @@ -97,11 +97,11 @@ ] }, "locked": { - "lastModified": 1724921939, - "narHash": "sha256-/S5iip1LHLiCP2VY7PwClDteP9ZMRZvzzKR1LZuV3fs=", + "lastModified": 1728371104, + "narHash": "sha256-PPnAyDedUQ7Og/Cby9x5OT9wMkNGTP8GS53V6N/dk4w=", "owner": "m-labs", "repo": "sipyco", - "rev": "32ddd78ff3641b75054793ea0d5681c951766754", + "rev": "094a6cd63ffa980ef63698920170e50dc9ba77fd", "type": "github" }, "original": { diff --git a/flake.nix b/flake.nix index e8b9dbe54..8d05492c9 100644 --- a/flake.nix +++ b/flake.nix @@ -191,12 +191,12 @@ asyncserial = pkgs.python3Packages.buildPythonPackage rec { pname = "asyncserial"; - version = "0.1"; + version = "1.0"; src = pkgs.fetchFromGitHub { owner = "m-labs"; repo = "asyncserial"; - rev = "446559fec892a556876b17d17f182ae9647d5952"; - sha256 = "sha256-WExmgh55sTH2w7wV3i96J1F1FN7L5rX3L/Ayvt2Kw/g="; + rev = version; + sha256 = "sha256-ZHzgJnbsDVxVcp09LXq9JZp46+dorgdP8bAiTB59K28="; }; propagatedBuildInputs = [ pkgs.python3Packages.pyserial ]; }; From cdcaee80d1cefd66388f860064084e6341dc0642 Mon Sep 17 00:00:00 2001 From: newell Date: Mon, 7 Oct 2024 21:34:18 -0700 Subject: [PATCH 023/111] Point to Hydra server for EBAZ4205 --- doc/manual/flashing.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/manual/flashing.rst b/doc/manual/flashing.rst index 715cd63b0..00301e5c2 100644 --- a/doc/manual/flashing.rst +++ b/doc/manual/flashing.rst @@ -17,7 +17,7 @@ Run the command:: Replace ```` with the login name that was given to you with the subscription, ```` with the name of your system variant, and ```` with the name of an empty directory, which will be created by the command if it does not exist. Enter your password when prompted and wait for the build (if applicable) and download to finish. If you experience issues with the AFWS client, write to the helpdesk@ email. For more information about :mod:`~artiq.frontend.afws_client` see also the corresponding entry on the :ref:`Utilities ` page. -For certain configurations (KC705 or ZC706 only) it is also possible to source firmware from `the M-Labs Hydra server `_ (in ``main`` and ``zynq`` respectively). +For certain configurations (KC705, ZC706, or EBAZ4205 only) it is also possible to source firmware from `the M-Labs Hydra server `_ (in ``main``, ``zynq`` and ``zynq`` respectively). Without a subscription, you may build the firmware yourself from the open source code. See the section :doc:`building_developing`. From 2fa60cc084e0b8f049a03550f07fe4163c90ac63 Mon Sep 17 00:00:00 2001 From: newell Date: Tue, 8 Oct 2024 18:27:10 -0700 Subject: [PATCH 024/111] doc: couple typos in extending rtio --- doc/manual/extending_rtio.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/manual/extending_rtio.rst b/doc/manual/extending_rtio.rst index 9c53c6dbe..707a13e3a 100644 --- a/doc/manual/extending_rtio.rst +++ b/doc/manual/extending_rtio.rst @@ -45,7 +45,7 @@ The 'configuration' of an FPGA, the circuit design it is programmed with, is its The low-level software that runs directly on the core device's CPU, softcore or hardcore, is its *firmware.* This is the 'operating system' of the core device. The firmware is tasked, among other things, with handling the low-level communication between the core device and the host machine, as well as between the core devices in a DRTIO setting. It is written in bare-metal `Rust `__. There are currently two active versions of the ARTIQ firmware (the version used for ARTIQ-Zynq, NAR3, is more modern than that used on Kasli and KC705, and will likely eventually replace it) but they are functionally equivalent except for internal details. -Experiment kernels themselves -- ARTIQ Python, processed by the ARTIQ compiler and loaded from the host machine -- rest on top of and are framed and supported by the firmware, in the same sense way that application software on your PC rests on top of an operating system. All together, software kernels communicate with the firmware to set parameters for the gateware, which passes signals directly to the hardware. +Experiment kernels themselves -- ARTIQ Python, processed by the ARTIQ compiler and loaded from the host machine -- rest on top of and are framed and supported by the firmware, in the same way that application software on your PC rests on top of an operating system. All together, software kernels communicate with the firmware to set parameters for the gateware, which passes signals directly to the hardware. These frameworks are built to be self-contained and extensible. To make additions to the gateware and software, for example, we do not need to make changes to the firmware; we can interact purely with the interfaces provided on either side. @@ -150,7 +150,7 @@ Add the combinatorial block as follows: :: self.comb += [ pad0.eq(pad0_o), If(reg, - pad1.eq(pad0_k) + pad1.eq(pad0_o) ) ] From a761b9cf9c12f0cdb1bce25dd01c5e301faef951 Mon Sep 17 00:00:00 2001 From: architeuthidae <93191635+architeuthidae@users.noreply.github.com> Date: Wed, 9 Oct 2024 07:25:52 +0200 Subject: [PATCH 025/111] doc: Additional Nix config details (#2584) Co-authored-by: architeuthidae --- doc/manual/building_developing.rst | 27 ++++++++++++++++++++++----- doc/manual/installing.rst | 2 +- 2 files changed, 23 insertions(+), 6 deletions(-) diff --git a/doc/manual/building_developing.rst b/doc/manual/building_developing.rst index 9bfec655a..e196c689d 100644 --- a/doc/manual/building_developing.rst +++ b/doc/manual/building_developing.rst @@ -82,11 +82,31 @@ Nix development environment --------------------------- * Install `Nix `_ if you haven't already. Prefer a single-user installation for simplicity. -* Enable flakes in Nix, for example by adding ``experimental-features = nix-command flakes`` to ``nix.conf``; see the `NixOS Wiki on flakes `_ for details and more options. +* Configure Nix to support building ARTIQ: + + - Enable flakes, for example by adding ``experimental-features = nix-command flakes`` to ``nix.conf``. See also the `NixOS Wiki on flakes `_. + - Add ``/opt`` (or your Vivado location) as an Nix sandbox, for example by adding ``extra-sandbox-paths = /opt`` to ``nix.conf``. + - Create a file called ``trusted-settings.json`` in ``~/.local/share/nix/``, if it doesn't exist already. Make sure it contains the following: + + :: + + { + "extra-sandbox-paths":{ + "/opt":true + }, + "extra-substituters":{ + "https://nixbld.m-labs.hk":true + }, + "extra-trusted-public-keys":{ + "nixbld.m-labs.hk-1:5aSRVA5b320xbNvu30tqxVPXpld73bhtOeH6uAjRyHc=":true + } + } + + - If using NixOS, instead make the equivalent changes to your ``configuration.nix``. + * Clone `the ARTIQ Git repository `_, or `the ARTIQ-Zynq repository `__ for Zynq devices (Kasli-SoC or ZC706). By default, you are working with the ``master`` branch, which represents the beta version and is not stable (see :doc:`releases`). Checkout the most recent release (``git checkout release-[number]``) for a stable version. * If your Vivado installation is not in its default location ``/opt``, open ``flake.nix`` and edit it accordingly (note that the edits must be made in the main ARTIQ flake, even if you are working with Zynq, see also tip below). * Run ``nix develop`` at the root of the repository, where ``flake.nix`` is. -* Answer ``y``/'yes' to any Nix configuration questions if necessary, as in :ref:`installing-troubleshooting`. .. note:: You can also target legacy versions of ARTIQ; use Git to checkout older release branches. Note however that older releases of ARTIQ required different processes for developing and building, which you are broadly more likely to figure out by (also) consulting the corresponding older versions of the manual. @@ -228,9 +248,6 @@ For ZC706: where ``fw-type`` is ``runtime`` for standalone or DRTIO master builds and ``satman`` for DRTIO satellites. Both the gateware and the firmware will generate into the ``../build`` destination directory. At this stage you can :ref:`boot from JTAG `; either of the ``*_run.sh`` scripts will expect the gateware and firmware files at their default locations, and the ``szl.elf`` bootloader is retrieved automatically. -.. warning:: - Note that in between runs of ``make`` it is necessary to manually clear ``build``, even for different targets, or ``make`` will do nothing. - If you prefer to boot from SD card, you will need to construct your own ``boot.bin``. Build ``szl.elf`` from source by running a command of the form: :: $ nix build git+https://git.m-labs.hk/m-labs/zynq-rs#-szl diff --git a/doc/manual/installing.rst b/doc/manual/installing.rst index e3655138b..e558ce3e5 100644 --- a/doc/manual/installing.rst +++ b/doc/manual/installing.rst @@ -97,7 +97,7 @@ To find more packages you can browse the `Nix package search Date: Tue, 17 Sep 2024 11:38:46 +0200 Subject: [PATCH 026/111] doc: Attribute writeback / data transfer FAQ --- doc/manual/compiler.rst | 4 +++- doc/manual/faq.rst | 20 +++++++++++++++++++- 2 files changed, 22 insertions(+), 2 deletions(-) diff --git a/doc/manual/compiler.rst b/doc/manual/compiler.rst index 29f623561..b91d3d7ae 100644 --- a/doc/manual/compiler.rst +++ b/doc/manual/compiler.rst @@ -89,6 +89,8 @@ Multidimensional arrays are allowed (using NumPy syntax). Element-wise operation User-defined classes are supported, provided their attributes are of other supported types (attributes that are not used in the kernel are ignored and thus unrestricted). When several instances of a user-defined class are referenced from the same kernel, every attribute must have the same type in every instance of the class. +.. _basic-artiq-python: + Basic ARTIQ Python ^^^^^^^^^^^^^^^^^^ @@ -117,7 +119,7 @@ Kernels can freely modify attributes of objects shared with the host. However, b # results in memory corruption return func([1, 2, 3]) - will compile, **but corrupts at runtime.** On the other hand, lists, arrays, or strings can and should be used as inputs for RPCs, and this is the preferred method of returning data to the host. In this way the data is inherently read and sent before the kernel completes and there are no allocation issues. + will compile, **but corrupts at runtime.** On the other hand, lists, arrays, or strings can and should be used as inputs for RPCs, and this is the preferred method of returning data to the host. In this way the data is sent before the kernel completes and there are no allocation issues. Available built-in functions ^^^^^^^^^^^^^^^^^^^^^^^^^^^^ diff --git a/doc/manual/faq.rst b/doc/manual/faq.rst index 547329b30..e5ef530fa 100644 --- a/doc/manual/faq.rst +++ b/doc/manual/faq.rst @@ -106,7 +106,7 @@ fix ``failed to connect to moninj`` in the dashboard? This and other similar messages almost always indicate that your device database lists controllers (for example, ``aqctl_moninj_proxy``) that either haven't been started or aren't reachable at the given host and port. See :ref:`mgmt-ctlmgr`, or simply run: :: - $ artiq_ctlgmr + $ artiq_ctlmgr to let the controller manager start the necessary controllers automatically. @@ -196,6 +196,24 @@ create and use variable-length arrays in kernels? You can't, in general; see the corresponding notes under :ref:`compiler-types`. ARTIQ kernels do not support heap allocation, meaning in particular that lists, arrays, and strings must be of constant size. One option is to preallocate everything, as mentioned on the Compiler page; another option is to chunk it and e.g. read 100 events per function call, push them upstream and retry until the gate time closes. +understand how best to send data between kernel and host? +--------------------------------------------------------- + +See also :ref:`basic-artiq-python`. Let's run down the options for kernel-host data transfer: + + - Kernels can return single values directly. They *cannot* return lists, arrays or strings, because of the way these values are allocated, which prevents values of these types from outliving the kernel they are created in. This is still true when the values in question are wrapped in functions or objects, in which case they may be missed by lifetime tracking and accepted by the compiler, but will cause memory corruption when run. + + - Kernels can freely make changes to attributes of objects shared with the host, including ``self``. However, these changes will be made to a kernel-owned copy of the object, which is only synchronized with the host copy when the kernel completes. This means that host-side operations executed during the runtime of the kernel, including RPCs, will be handling an unmodified version of the object, and modifications made by those operations will simply be overwritten when the kernel returns. + + .. note:: + Attribute writeback happens *once per kernel*, that is, if your experiment contains many separate kernels called from the host, modifications will be written back when each separate kernel completes. This is generally not suitable for data transfer, however, as new kernels are costly to create, and experiments often try to avoid doing so. It is also important to specify that kernels called *from* a kernel will not write back to the host upon completion. Attribute writeback is only executed upon return to the host. + + - Kernels can interact with datasets, either as attributes (if :meth:`~artiq.language.environment.HasEnvironment.setattr_dataset` is used) or by RPC of the get and set methods (:meth:`~artiq.language.environment.HasEnvironment.get_dataset`, :meth:`~artiq.language.environment.HasEnvironment.set_dataset`, etc.). In this case note that, like certain other host-side methods, :meth:`~artiq.language.environment.HasEnvironment.get_dataset` will not actually be accepted by the compiler, because its return type is not specified. To call it as an RPC, simply wrap it in another function which *does* specify a return type. :meth:`~artiq.language.environment.HasEnvironment.set_dataset` can be similarly wrapped to make it asynchronous. + + - Kernels can of course also call arbitrary RPCs. When sending data to the host, these can be asynchronous, and this is normally the recommended way of transferring data back to the host, resulting in a relatively minor amount of delay in the kernel. Keep in mind however that asynchronous RPCs may still block execution for some time if the arguments are very large or if many RPCs are submitted in close succession. When receiving data from the host, RPCs must be synchronous, which is still considerably faster than starting a new kernel. Note that if data is being both (asynchronously) sent and received, there is a small possibility of minor race conditions (i.e. retrieved data may not yet show updates sent in an earlier RPC). + +Kernel attributes and data transfer remain somewhat of an open area of development. Many such developments are or will be implemented in `NAC3 `_, the next-generation ARTIQ compiler. The overhead for starting new kernels, which is largely dominated by compile time, should be significantly reduced (NAC3 can be expected to complete compilations 6x - 30x faster than currently). + write part of my experiment as a coroutine/asyncio task/generator? ------------------------------------------------------------------ From e06534913cb0053a11c2b2e91a93825875336c3f Mon Sep 17 00:00:00 2001 From: occheung Date: Thu, 10 Oct 2024 17:36:42 +0800 Subject: [PATCH 027/111] satman: gate gt_drtio against having drtio-eem --- artiq/firmware/satman/main.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/artiq/firmware/satman/main.rs b/artiq/firmware/satman/main.rs index d402a440a..8f5470e3e 100644 --- a/artiq/firmware/satman/main.rs +++ b/artiq/firmware/satman/main.rs @@ -752,7 +752,7 @@ pub extern fn main() -> i32 { io_expander.service().unwrap(); } - #[cfg(not(soc_platform = "efc"))] + #[cfg(not(has_drtio_eem))] unsafe { csr::gt_drtio::txenable_write(0xffffffffu32 as _); } From a570e6fd8788d6c1262d725cfa655553749a9635 Mon Sep 17 00:00:00 2001 From: thatschatt Date: Mon, 14 Oct 2024 12:47:02 +0200 Subject: [PATCH 028/111] doc: Add TTuple to compiler section (#2591) --- doc/manual/compiler.rst | 70 ++++++++++++++++++++++++----------------- 1 file changed, 41 insertions(+), 29 deletions(-) diff --git a/doc/manual/compiler.rst b/doc/manual/compiler.rst index b91d3d7ae..10fdc0270 100644 --- a/doc/manual/compiler.rst +++ b/doc/manual/compiler.rst @@ -48,35 +48,37 @@ ARTIQ types Python/NumPy types correspond to ARTIQ types as follows: -+---------------+-------------------------+ -| Python | ARTIQ | -+===============+=========================+ -| NoneType | TNone | -+---------------+-------------------------+ -| bool | TBool | -+---------------+-------------------------+ -| int | TInt32 or TInt64 | -+---------------+-------------------------+ -| float | TFloat | -+---------------+-------------------------+ -| str | TStr | -+---------------+-------------------------+ -| bytes | TBytes | -+---------------+-------------------------+ -| bytearray | TByteArray | -+---------------+-------------------------+ -| list of T | TList(T) | -+---------------+-------------------------+ -| NumPy array | TArray(T, num_dims) | -+---------------+-------------------------+ -| range | TRange32, TRange64 | -+---------------+-------------------------+ -| numpy.int32 | TInt32 | -+---------------+-------------------------+ -| numpy.int64 | TInt64 | -+---------------+-------------------------+ -| numpy.float64 | TFloat | -+---------------+-------------------------+ ++------------------------+-------------------------+ +| Python | ARTIQ | ++========================+=========================+ +| NoneType | TNone | ++------------------------+-------------------------+ +| bool | TBool | ++------------------------+-------------------------+ +| int | TInt32 or TInt64 | ++------------------------+-------------------------+ +| float | TFloat | ++------------------------+-------------------------+ +| str | TStr | ++------------------------+-------------------------+ +| bytes | TBytes | ++------------------------+-------------------------+ +| bytearray | TByteArray | ++------------------------+-------------------------+ +| list of T | TList(T) | ++------------------------+-------------------------+ +| NumPy array | TArray(T, num_dims) | ++------------------------+-------------------------+ +| tuple of (T1, T2, ...) | TTuple([T1, T2, ...]) | ++------------------------+-------------------------+ +| range | TRange32, TRange64 | ++------------------------+-------------------------+ +| numpy.int32 | TInt32 | ++------------------------+-------------------------+ +| numpy.int64 | TInt64 | ++------------------------+-------------------------+ +| numpy.float64 | TFloat | ++------------------------+-------------------------+ Integers are 32-bit by default but may be converted to 64-bit with ``numpy.int64``. @@ -87,6 +89,8 @@ The ARTIQ compiler can be thought of as overriding all built-in Python types, an Multidimensional arrays are allowed (using NumPy syntax). Element-wise operations (e.g. ``+``, ``/``), matrix multiplication (``@``) and multidimensional indexing are supported; slices and views (currently) are not. +Tuples may contain a mixture of different types. They cannot be iterated over or dynamically indexed, although they may be indexed by constants and multiple assignment is supported. + User-defined classes are supported, provided their attributes are of other supported types (attributes that are not used in the kernel are ignored and thus unrestricted). When several instances of a user-defined class are referenced from the same kernel, every attribute must have the same type in every instance of the class. .. _basic-artiq-python: @@ -101,6 +105,14 @@ Kernel code can call host functions without any additional ceremony. However, su def return_four() -> TInt32: return 4 +.. tip:: + Multiple variables of different types can be sent in one RPC call by returning a tuple, e.g. :: + + def return_many() -> TTuple([TInt32, TFloat, TStr]): + return (4, 12.34, "hello",) + + Which can be retrieved from a kernel with ``(a, b, c) = return_many()`` + Kernels can freely modify attributes of objects shared with the host. However, by necessity, these modifications are actually applied to local copies of the objects, as the latency of immediate writeback would be unsupportable in a real-time environment. Instead, modifications are written back *when the kernel completes;* notably, this means RPCs called by a kernel itself will only have access to the unmodified host version of the object, as the kernel hasn't finished execution yet. In some cases, accessing data on the host is better handled by calling RPCs specifically to make the desired modifications. .. warning:: From 1fc6ab8c57c7f766b5aa11641585fb1acde809bc Mon Sep 17 00:00:00 2001 From: newell Date: Mon, 14 Oct 2024 09:54:47 -0700 Subject: [PATCH 029/111] doc: Fix Python indentation --- doc/manual/getting_started_mgmt.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/manual/getting_started_mgmt.rst b/doc/manual/getting_started_mgmt.rst index d5d3cb722..40a38b219 100644 --- a/doc/manual/getting_started_mgmt.rst +++ b/doc/manual/getting_started_mgmt.rst @@ -139,8 +139,8 @@ Experiments may have arguments, values which can be set in the dashboard on subm self.setattr_argument("count", NumberValue(precision=0, step=1)) def run(self): - for i in range(self.count): - print("Hello World", i) + for i in range(self.count): + print("Hello World", i) The method :meth:`~artiq.language.environment.HasEnvironment.setattr_argument` acts to set the argument and make its value accessible, similar to the effect of :meth:`~artiq.language.environment.HasEnvironment.setattr_device`. The second input sets the type of the argument; here, :class:`~artiq.language.environment.NumberValue` represents a floating point numerical value. To learn what other types are supported, see :class:`artiq.language.environment` and :class:`artiq.language.scan`. From fcac2ea20e165196c9093cdd9173d25445a561c3 Mon Sep 17 00:00:00 2001 From: Sebastien Bourdeauducq Date: Tue, 15 Oct 2024 15:57:09 +0800 Subject: [PATCH 030/111] flake: update dependencies --- flake.lock | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/flake.lock b/flake.lock index 517aaecde..cfa15f0db 100644 --- a/flake.lock +++ b/flake.lock @@ -44,11 +44,11 @@ }, "nixpkgs": { "locked": { - "lastModified": 1728241625, - "narHash": "sha256-yumd4fBc/hi8a9QgA9IT8vlQuLZ2oqhkJXHPKxH/tRw=", + "lastModified": 1728492678, + "narHash": "sha256-9UTxR8eukdg+XZeHgxW5hQA9fIKHsKCdOIUycTryeVw=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "c31898adf5a8ed202ce5bea9f347b1c6871f32d1", + "rev": "5633bcff0c6162b9e4b5f1264264611e950c8ec7", "type": "github" }, "original": { @@ -129,11 +129,11 @@ "src-misoc": { "flake": false, "locked": { - "lastModified": 1715647536, - "narHash": "sha256-q+USDcaKHABwW56Jzq8u94iGPWlyLXMyVt0j/Gyg+IE=", + "lastModified": 1728978817, + "narHash": "sha256-b4633jrhh4i+KunZq4kNlyhdm9BCsEJwKs+6KINKV2o=", "ref": "refs/heads/master", - "rev": "fea9de558c730bc394a5936094ae95bb9d6fa726", - "revCount": 2455, + "rev": "386b544776b66cea148da67d06a4b3a4151179f9", + "revCount": 2459, "submodules": true, "type": "git", "url": "https://github.com/m-labs/misoc.git" From 700812471c6754938bbcde88b9bf5aa243c1481e Mon Sep 17 00:00:00 2001 From: newell Date: Tue, 15 Oct 2024 23:12:40 -0700 Subject: [PATCH 031/111] doc: Additonal EBAZ4205 documentation, fix JSON description terminology (#2588) --- doc/manual/building_developing.rst | 40 +++++++++++++++++++----------- doc/manual/configuring.rst | 3 +++ doc/manual/core_device.rst | 23 ++++++++++++++++- doc/manual/extending_rtio.rst | 8 +++--- doc/manual/flashing.rst | 6 ++--- 5 files changed, 57 insertions(+), 23 deletions(-) diff --git a/doc/manual/building_developing.rst b/doc/manual/building_developing.rst index e196c689d..db9ed1a0c 100644 --- a/doc/manual/building_developing.rst +++ b/doc/manual/building_developing.rst @@ -33,13 +33,13 @@ Download and run the official installer. If using NixOS, note that this will req System description file ----------------------- -ARTIQ gateware and firmware binaries are dependent on the system configuration. In other words, a specific set of ARTIQ binaries is bound to the exact arrangement of real-time hardware it was generated for: the core device itself, its role in a DRTIO context (master, satellite, or standalone), the (real-time) peripherals in use, the physical EEM ports they will be connected to, and various other basic specifications. This information is normally provided to the software in the form of a JSON file called the system description or system configuration file. +ARTIQ gateware and firmware binaries are dependent on the system configuration. In other words, a specific set of ARTIQ binaries is bound to the exact arrangement of real-time hardware it was generated for: the core device itself, its role in a DRTIO context (master, satellite, or standalone), the (real-time) peripherals in use, the physical EEM ports they will be connected to, and various other basic specifications. This information is normally provided to the software in the form of a JSON file called the system description file. .. warning:: - System configuration files are only used with Kasli and Kasli-SoC boards. KC705 and ZC706 ARTIQ configurations, due to their relative rarity and specialization, are handled on a case-by-case basis and selected through a variant name such as ``nist_clock``, with no system description file necessary. See below in :ref:`building` for where to find the list of supported variants. Writing new KC705 or ZC706 variants is not a trivial task, and not particularly recommended, unless you are an FPGA developer and know what you're doing. + Not all core devices use system description files. Devices that use system description files for configuration are referred to as JSON variants (see :ref:`JSON variant devices `). Some rare or specialized boards use hardcoded variants, selected by a variant name such as ``nist_clock``, without needing a system description file (see :ref:`Hardcoded variant devices `). For the list of supported variants, see the :ref:`building` section. Writing new hardcoded variants is not a trivial task and is generally not recommended unless you are an experienced FPGA developer. -If you already have your system configuration file on hand, you can edit it to reflect any changes in configuration. If you purchased your original system from M-Labs, or recently purchased new hardware to add to it, you can obtain your up-to-date system configuration file through AFWS at any time using the command ``$ afws_client get_json`` (see :ref:`AFWS client`). If you are starting from scratch, a close reading of ``coredevice_generic.schema.json`` in ``artiq/coredevice`` will be helpful. +If you already have your system description file on hand, you can edit it to reflect any changes in configuration. If you purchased your original system from M-Labs, or recently purchased new hardware to add to it, you can obtain your up-to-date system description file through AFWS at any time using the command ``$ afws_client get_json`` (see :ref:`AFWS client`). If you are starting from scratch, a close reading of ``coredevice_generic.schema.json`` in ``artiq/coredevice`` will be helpful. System descriptions do not need to be very complex. At its most basic, a system description looks something like: :: @@ -104,7 +104,7 @@ Nix development environment - If using NixOS, instead make the equivalent changes to your ``configuration.nix``. -* Clone `the ARTIQ Git repository `_, or `the ARTIQ-Zynq repository `__ for Zynq devices (Kasli-SoC or ZC706). By default, you are working with the ``master`` branch, which represents the beta version and is not stable (see :doc:`releases`). Checkout the most recent release (``git checkout release-[number]``) for a stable version. +* Clone `the ARTIQ Git repository `_, or `the ARTIQ-Zynq repository `__ for :ref:`Zynq devices ` (Kasli-SoC, ZC706, or EBAZ4205). By default, you are working with the ``master`` branch, which represents the beta version and is not stable (see :doc:`releases`). Checkout the most recent release (``git checkout release-[number]``) for a stable version. * If your Vivado installation is not in its default location ``/opt``, open ``flake.nix`` and edit it accordingly (note that the edits must be made in the main ARTIQ flake, even if you are working with Zynq, see also tip below). * Run ``nix develop`` at the root of the repository, where ``flake.nix`` is. @@ -171,10 +171,10 @@ This will create a directory ``artiq_kasli`` or ``artiq_kc705`` containing the b Look for the option ``-V VARIANT, --variant VARIANT``. -Kasli-SoC or ZC706 (ARTIQ on Zynq) -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +Kasli-SoC, ZC706 or EBAZ4205 (ARTIQ on Zynq) +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -The building process for Zynq devices is a little more complex. The easiest method is to leverage ``nix build`` and the ``makeArtiqZynqPackage`` utility provided by the official flake. The ensuing command is rather long, because it uses a multi-clause expression in the Nix language to describe the desired result; it can be executed piece-by-piece using the `Nix REPL `_, but ``nix build`` provides a lot of useful conveniences. +The building process for :ref:`Zynq devices ` is a little more complex. The easiest method is to leverage ``nix build`` and the ``makeArtiqZynqPackage`` utility provided by the official flake. The ensuing command is rather long, because it uses a multi-clause expression in the Nix language to describe the desired result; it can be executed piece-by-piece using the `Nix REPL `_, but ``nix build`` provides a lot of useful conveniences. For Kasli-SoC, run: :: @@ -182,18 +182,18 @@ For Kasli-SoC, run: :: Replace ```` with ``master``, ``satellite``, or ``standalone``, depending on your targeted DRTIO role. Remove ``?ref=release-[number]`` to use the current beta version rather than a numbered release. If you have cloned the repository and prefer to use your local copy of the flake, replace the corresponding clause with ``builtins.getFlake "/absolute/path/to/your/artiq-zynq"``. -For ZC706, you can use a command of the same form: :: +For ZC706 or EBAZ4205, you can use a command of the same form (replace ```` with ``zc706`` or ``ebaz4205``): :: - $ nix build --print-build-logs --impure --expr 'let fl = builtins.getFlake "git+https://git.m-labs.hk/m-labs/artiq-zynq?ref=release-[number]"; in (fl.makeArtiqZynqPackage {target="zc706"; variant="";}).zc706--sd' + $ nix build --print-build-logs --impure --expr 'let fl = builtins.getFlake "git+https://git.m-labs.hk/m-labs/artiq-zynq?ref=release-[number]"; in (fl.makeArtiqZynqPackage {target=""; variant="";}).--sd' or you can use the more direct version: :: - $ nix build --print-build-logs git+https://git.m-labs.hk/m-labs/artiq-zynq\?ref=release-[number]#zc706--sd + $ nix build --print-build-logs git+https://git.m-labs.hk/m-labs/artiq-zynq\?ref=release-[number]#--sd -(which is possible for ZC706 because there is no need to be able to specify a system description file in the arguments.) +(which is possible for ZC706 or EBAZ4205 because there is no need to be able to specify a system description file in the arguments.) .. note:: - To see supported ZC706 variants, you can run the following at the root of the repository: :: + To see supported ZC706 variants (for EBAZ4205 variants, replace ``zc706`` with ``ebaz4205``), you can run the following at the root of the repository: :: $ src/gateware/zc706.py --help @@ -207,7 +207,11 @@ Any of these commands should produce a directory ``result`` which contains a fil 1. Power off the board, extract the SD card and load ``boot.bin`` onto it manually. 2. Insert the SD card back into the board. -3. Ensure that the DIP switches (labeled BOOT MODE) are set correctly, to SD. +3. Set to boot from SD card: + + - For Kasli-SoC or ZC706, ensure that the DIP switches (labeled BOOT MODE) are set correctly, to SD. + - For EBAZ4205, set up the `boot select resistor `_ to boot from SD card. + 4. Power the board back on. Optionally, the SD card may also be loaded at the same time with an additional file ``config.txt``, which can contain preset configuration values in the format ``key=value``, one per line. The keys are those used with :mod:`~artiq.frontend.artiq_coremgmt`. This allows e.g. presetting an IP address and any other configuration information. @@ -219,7 +223,7 @@ After a successful boot, the "FPGA DONE" light should be illuminated and the boa Booting over JTAG/Ethernet """""""""""""""""""""""""" -It is also possible to boot Zynq devices over USB and Ethernet. Flip the DIP switches to JTAG. The scripts ``remote_run.sh`` and ``local_run.sh`` in the ARTIQ-Zynq repository, intended for use with a remote JTAG server or a local connection to the core device respectively, are used at M-Labs to accomplish this. Both make use of the netboot tool ``artiq_netboot``, see also its source `here `__, which is included in the ARTIQ-Zynq development environment. Adapt the relevant script to your system or read it closely to understand the options and the commands being run; note for example that ``remote_run.sh`` as written only supports ZC706. +It is also possible to boot :ref:`Zynq devices ` over USB and Ethernet (EBAZ4205 not currently supported). Flip the DIP switches to JTAG. The scripts ``remote_run.sh`` and ``local_run.sh`` in the ARTIQ-Zynq repository, intended for use with a remote JTAG server or a local connection to the core device respectively, are used at M-Labs to accomplish this. Both make use of the netboot tool ``artiq_netboot``, see also its source `here `__, which is included in the ARTIQ-Zynq development environment. Adapt the relevant script to your system or read it closely to understand the options and the commands being run; note for example that ``remote_run.sh`` as written only supports ZC706. You will need to generate the gateware, firmware and bootloader first, either through ``nix build`` or incrementally as below. After an incremental build add the option ``-i`` when running either of the scripts. If using ``nix build``, note that target names of the form ``--jtag`` (run ``nix flake show`` to see all targets) will output the three necessary files without combining them into ``boot.bin``. @@ -246,7 +250,13 @@ For ZC706: $ gateware/zc706.py -g ../build/gateware -V $ make TARGET=zc706 GWARGS="-V " -where ``fw-type`` is ``runtime`` for standalone or DRTIO master builds and ``satman`` for DRTIO satellites. Both the gateware and the firmware will generate into the ``../build`` destination directory. At this stage you can :ref:`boot from JTAG `; either of the ``*_run.sh`` scripts will expect the gateware and firmware files at their default locations, and the ``szl.elf`` bootloader is retrieved automatically. +For EBAZ4205: + :: + + $ gateware/ebaz4205.py -g ../build/gateware -V + $ make TARGET=ebaz4205 GWARGS="-V " + +where ``fw-type`` is ``runtime`` for standalone or DRTIO master builds and ``satman`` for DRTIO satellites. Both the gateware and the firmware will generate into the ``../build`` destination directory. At this stage, if supported, you can :ref:`boot from JTAG `; either of the ``*_run.sh`` scripts will expect the gateware and firmware files at their default locations, and the ``szl.elf`` bootloader is retrieved automatically. If you prefer to boot from SD card, you will need to construct your own ``boot.bin``. Build ``szl.elf`` from source by running a command of the form: :: diff --git a/doc/manual/configuring.rst b/doc/manual/configuring.rst index 219e37ce4..998344f24 100644 --- a/doc/manual/configuring.rst +++ b/doc/manual/configuring.rst @@ -55,6 +55,9 @@ For Kasli-SoC: For ZC706: If the ``ip`` config is not set, ZC706 firmware defaults to using the IP address ``192.168.1.52``. +For EBAZ4205: + If the ``ip`` config is not set, EBAZ4205 firmware defaults to using the IP address ``192.168.1.57``. + For Kasli or KC705: If the ``ip`` config field is not set or set to ``use_dhcp``, the device will attempt to obtain an IP address and default gateway using DHCP. The chosen IP address will be in log output, which can be accessed via the :ref:`UART log `. diff --git a/doc/manual/core_device.rst b/doc/manual/core_device.rst index ef4859141..b55c638be 100644 --- a/doc/manual/core_device.rst +++ b/doc/manual/core_device.rst @@ -39,7 +39,7 @@ The core device reserves some storage space (either flash or directly on SD card ``device_map`` If set, allows the core log to connect RTIO channels to device names and use device names as well as channel numbers in log output. A correctly formatted table can be automatically generated with :mod:`~artiq.frontend.artiq_rtiomap`, see :ref:`Utilities`. ``net_trace`` - If set to ``1``, will activate net trace (print all packets sent and received to UART and core log). This will considerably slow down all network response from the core. Not applicable for ARTIQ-Zynq (Kasli-SoC, ZC706). + If set to ``1``, will activate net trace (print all packets sent and received to UART and core log). This will considerably slow down all network response from the core. Not applicable for ARTIQ-Zynq (see :ref:`Zynq devices `). ``panic_reset`` If set to ``1``, core device will restart automatically. Not applicable for ARTIQ-Zynq. ``no_flash_boot`` @@ -106,6 +106,27 @@ If not using WRPLL, PLL can also be bypassed entirely with the options Bypassing the PLL ensures the skews between input clock, downstream clock outputs, and RTIO clock are deterministic across reboots of the system. This is useful when phase determinism is required in situations where the reference clock fans out to other devices before reaching the master. +.. _types-of-boards: + +Types of boards +--------------- + +To clarify the terminology used in ARTIQ, we can distinguish the boards into a few key groups. There are two primary ways to categorize them. The first is based on the ARTIQ platform itself: either ARTIQ or ARTIQ-Zynq. ARTIQ-Zynq boards specifically refer to those that feature a Xilinx Zynq FPGA. The second distinction is based on how the boards are configured: some use a :ref:`JSON system description file `, while others do not. + +Below are the current groups of boards: + +.. _devices-table: + ++------------------------------+------------------------------+ +| **Device Type** | **Devices** | ++==============================+==============================+ +| Zynq devices | Kasli-SoC, ZC706, EBAZ4205 | ++------------------------------+------------------------------+ +| JSON variant devices | Kasli, Kasli-SoC | ++------------------------------+------------------------------+ +| Hardcoded variant devices | KC705, ZC706, EBAZ4205 | ++------------------------------+------------------------------+ + Board details ------------- diff --git a/doc/manual/extending_rtio.rst b/doc/manual/extending_rtio.rst index 707a13e3a..b3347f9e2 100644 --- a/doc/manual/extending_rtio.rst +++ b/doc/manual/extending_rtio.rst @@ -37,7 +37,7 @@ Introduction to the ARTIQ internal stack \draw[primary, -Stealth] (gateware) to (firmware); \draw[primary, -Stealth] (hardware) to (gateware); -Like any other modern piece of software, kernel code running on an ARTIQ core device rests upon a layered infrastructure, starting with the hardware: the physical carrier board and its peripherals. Generally, though not exclusively, this is the `Sinara device family `_, which is designed to work with ARTIQ. Other carrier boards, such as the Xilinx KC705 and ZC706, are also supported. +Like any other modern piece of software, kernel code running on an ARTIQ core device rests upon a layered infrastructure, starting with the hardware: the physical carrier board and its peripherals. Generally, though not exclusively, this is the `Sinara device family `_, which is designed to work with ARTIQ. Other carrier boards, such as the :ref:`Hardcoded variant devices `, are also supported. All of the ARTIQ core device carrier boards necessarily center around a physical field-programmable gate array, or FPGA. If you have never worked with FPGAs before, it is easiest to understand them as 'rearrangeable' circuits. Ideally, they are capable of approaching the tremendous speed and timing precision advantages of custom-designed, application-specific hardware, while still being reprogrammable, allowing development and revision to continue after manufacturing. @@ -59,7 +59,7 @@ As briefly explained in :doc:`rtio`, when we talk about RTIO infrastructure, we .. warning:: Note that FPGA resources are finite, and buffer sizes, lane counts, etc., are generally chosen to maximize available resources already, with different values depending on the core device in use. Depending on the peripherals you include (some are more resource-intensive than others) blanket increases will likely quickly outstrip the capacity of your FPGA and fail to build. Increasing the depth of a particular channel you know to be heavily used is more likely to succeed; the easiest way to find out is to attempt the build and observe what results. -Gateware in ARTIQ is housed in ``artiq/gateware`` on the main ARTIQ repository and (for Zynq-specific additions) in ``artiq-zynq/src/gateware`` on ARTIQ-Zynq. The starting point for figuring out your changes will often be the *target file*, which is core device-specific and which you may recognize as the primary module called when building gateware. Depending on your core device, simply track down the file named after it, as in ``kasli.py``, ``kasli_soc.py``, and so on. Note that the Kasli and Kasli-SoC targets are designed to take JSON description files as input, whereas their KC705 and ZC706 equivalents work with hardcoded variants instead. +Gateware in ARTIQ is housed in ``artiq/gateware`` on the main ARTIQ repository and (for Zynq-specific additions) in ``artiq-zynq/src/gateware`` on ARTIQ-Zynq. The starting point for figuring out your changes will often be the *target file*, which is core device-specific and which you may recognize as the primary module called when building gateware. Depending on your core device, simply track down the file named after it, as in ``kasli.py``, ``kasli_soc.py``, and so on. Note that the Kasli and Kasli-SoC targets are designed to take JSON description files as input (see :ref:`JSON variant devices `), whereas their KC705, ZC706 and EBAZ4205 (see :ref:`Hardcoded variant devices `) equivalents work with hardcoded variants instead. To change parameters related to particular peripherals, see also the files ``eem.py`` and ``eem_7series.py``, which describe the core device's interface with other EEM cards in Migen terms, and contain ``add_std`` methods that in turn reference specific gateware modules and assign RTIO channels. @@ -279,7 +279,7 @@ Adding a custom EEM ------------------- .. note:: - Adding a custom EEM to a Kasli or Kasli-SoC system is not much more difficult than adding new gateware logic for existing hardware, and may in some cases be simpler, if no custom PHY is required. On the other hand, modifying hardware in KC705 or ZC706-based systems is a different process, and gateware generation for these boards does not use the files and modules described below. Creating new KC705 or ZC706 variants is not directly addressed in this tutorial. That said, it would begin and end largely in the respective target file, where the variants are defined. + Adding a custom EEM to a Kasli or Kasli-SoC system is not much more difficult than adding new gateware logic for existing hardware, and may in some cases be simpler, if no custom PHY is required. On the other hand, modifying :ref:`Hardcoded variant devices ` is a different process, and gateware generation for these boards does not use the files and modules described below. Creating new hardcoded variants is not directly addressed in this tutorial. That said, it would begin and end largely in the respective target file, where the variants are defined. Non-realtime hardware which does not need to connect directly to the core device or require gateware support should instead be handled through an NDSP, see :doc:`developing_a_ndsp`. This is a more accessible process in general and does not vary based on core device. @@ -330,7 +330,7 @@ Now your EEM is fully supported by the ARTIQ gateware infrastructure. All that r Target file and system description ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -In the :ref:`extending-gateware-logic` tutorial above, we made modifications directly to the target file, to hardcode a certain PHY for a certain set of pads. This is reasonable to do in the case of the core device LEDs, which are always present and cannot be rearranged. It is theoretically possible to hardcode the addition of your new EEM in the same way. In this case it would not be necessary to make modifications to ``eem.py`` and ``eem_7series.py``; the pad assignments, requisite PHYs, and RTIO channels could all be defined directly in the target file. This is essentially how things are done for KC705 and ZC706 variants. +In the :ref:`extending-gateware-logic` tutorial above, we made modifications directly to the target file, to hardcode a certain PHY for a certain set of pads. This is reasonable to do in the case of the core device LEDs, which are always present and cannot be rearranged. It is theoretically possible to hardcode the addition of your new EEM in the same way. In this case it would not be necessary to make modifications to ``eem.py`` and ``eem_7series.py``; the pad assignments, requisite PHYs, and RTIO channels could all be defined directly in the target file. This is essentially how things are done for :ref:`Hardcoded variant devices `. However, with EEM cards, which can be present in different numbers and rearranged at will, it is preferable to be more flexible. This is the reason system description files are used. Assuming you have added your EEM to ``eem.py`` and the ``peripheral_processors`` dictionary, no modifications to the target file are actually necessarily. All Kasli and Kasli-SoC targets already contain the line: :: diff --git a/doc/manual/flashing.rst b/doc/manual/flashing.rst index 00301e5c2..abfc5ec13 100644 --- a/doc/manual/flashing.rst +++ b/doc/manual/flashing.rst @@ -17,7 +17,7 @@ Run the command:: Replace ```` with the login name that was given to you with the subscription, ```` with the name of your system variant, and ```` with the name of an empty directory, which will be created by the command if it does not exist. Enter your password when prompted and wait for the build (if applicable) and download to finish. If you experience issues with the AFWS client, write to the helpdesk@ email. For more information about :mod:`~artiq.frontend.afws_client` see also the corresponding entry on the :ref:`Utilities ` page. -For certain configurations (KC705, ZC706, or EBAZ4205 only) it is also possible to source firmware from `the M-Labs Hydra server `_ (in ``main``, ``zynq`` and ``zynq`` respectively). +For :ref:`hardcoded variant devices ` it is also possible to source firmware from `the M-Labs Hydra server `_ (in ``main`` and ``zynq``). Without a subscription, you may build the firmware yourself from the open source code. See the section :doc:`building_developing`. @@ -25,7 +25,7 @@ Installing and configuring OpenOCD ---------------------------------- .. warning:: - These instructions are not applicable to Zynq devices (Kasli-SoC, ZC706 or EBAZ4205), which do not use the utility :mod:`~artiq.frontend.artiq_flash`. If your core device is a Zynq device, skip straight to :ref:`writing-flash`. + These instructions are not applicable to :ref:`Zynq devices `, which do not use the utility :mod:`~artiq.frontend.artiq_flash`. If your core device is a Zynq device, skip straight to :ref:`writing-flash`. ARTIQ supplies the utility :mod:`~artiq.frontend.artiq_flash`, which uses OpenOCD to write the binary images into an FPGA board's flash memory. For both Nix and MSYS2, OpenOCD are included with the installation by default. Note that in the case of Nix this is the package ``artiq.openocd-bscanspi`` and not ``pkgs.openocd``; the second is OpenOCD from the Nix package collection, which does not support ARTIQ/Sinara boards. @@ -78,7 +78,7 @@ On Windows Writing the flash ----------------- -First ensure the board is connected to your computer. In the case of Kasli, the JTAG adapter is integrated into the Kasli board; for flashing (and debugging) you can simply connect your computer to the micro-USB connector on the Kasli front panel. For Kasli-SoC, ZC706, or EBAZ4205, which use :mod:`~artiq.frontend.artiq_coremgmt` to flash over network, an Ethernet connection and an IP address, supplied either with the ``-D`` option or in your :ref:`device database `, are sufficient. +First ensure the board is connected to your computer. In the case of Kasli, the JTAG adapter is integrated into the Kasli board; for flashing (and debugging) you can simply connect your computer to the micro-USB connector on the Kasli front panel. For :ref:`Zynq devices `, which use :mod:`~artiq.frontend.artiq_coremgmt` to flash over network, an Ethernet connection and an IP address, supplied either with the ``-D`` option or in your :ref:`device database `, are sufficient. For Kasli-SoC, ZC706 or EBAZ4205: :: From 30b94794379fefcadecf8aaffbd4e6225eb431d8 Mon Sep 17 00:00:00 2001 From: newell Date: Wed, 16 Oct 2024 23:14:45 -0700 Subject: [PATCH 032/111] add AD9834 core device driver (#2596) --- RELEASE_NOTES.rst | 1 + artiq/coredevice/ad9834.py | 397 ++++++++++++++++++++++++++ artiq/test/coredevice/test_ad9834.py | 321 +++++++++++++++++++++ doc/manual/core_device.rst | 30 +- doc/manual/core_drivers_reference.rst | 6 + 5 files changed, 752 insertions(+), 3 deletions(-) create mode 100644 artiq/coredevice/ad9834.py create mode 100644 artiq/test/coredevice/test_ad9834.py diff --git a/RELEASE_NOTES.rst b/RELEASE_NOTES.rst index 1b374d1f5..eb97387e8 100644 --- a/RELEASE_NOTES.rst +++ b/RELEASE_NOTES.rst @@ -20,6 +20,7 @@ ARTIQ-9 (Unreleased) * Compiler can now give automatic suggestions for ``kernel_invariants``. * Idle kernels now restart when written with ``artiq_coremgmt`` and stop when erased/removed from config. * New support for the EBAZ4205 Zynq-SoC control card. +* New core device driver for the AD9834 DDS, tested with the ZonRi Technology Co., Ltd. AD9834-Module. ARTIQ-8 ------- diff --git a/artiq/coredevice/ad9834.py b/artiq/coredevice/ad9834.py new file mode 100644 index 000000000..306aa712d --- /dev/null +++ b/artiq/coredevice/ad9834.py @@ -0,0 +1,397 @@ +""" +RTIO Driver for the Analog Devices AD9834 DDS via 3-wire SPI interface. +""" + +# https://www.analog.com/media/en/technical-documentation/data-sheets/AD9834.pdf +# https://www.analog.com/media/en/technical-documentation/app-notes/an-1070.pdf + +from artiq.coredevice import spi2 as spi +from artiq.experiment import * +from artiq.language.core import * +from artiq.language.types import * +from artiq.language.units import * + +AD9834_B28 = 1 << 13 +AD9834_HLB = 1 << 12 +AD9834_FSEL = 1 << 11 +AD9834_PSEL = 1 << 10 +AD9834_PIN_SW = 1 << 9 +AD9834_RESET = 1 << 8 +AD9834_SLEEP1 = 1 << 7 +AD9834_SLEEP12 = 1 << 6 +AD9834_OPBITEN = 1 << 5 +AD9834_SIGN_PIB = 1 << 4 +AD9834_DIV2 = 1 << 3 +AD9834_MODE = 1 << 1 + +AD9834_FREQ_REG_0 = 0b01 << 14 +AD9834_FREQ_REG_1 = 0b10 << 14 +FREQ_REGS = [AD9834_FREQ_REG_0, AD9834_FREQ_REG_1] + +AD9834_PHASE_REG = 0b11 << 14 +AD9834_PHASE_REG_0 = AD9834_PHASE_REG | (0 << 13) +AD9834_PHASE_REG_1 = AD9834_PHASE_REG | (1 << 13) +PHASE_REGS = [AD9834_PHASE_REG_0, AD9834_PHASE_REG_1] + + +class AD9834: + """ + AD9834 DDS driver. + + This class provides control for the DDS AD9834. + + The driver utilizes bit-controlled :const:`AD9834_FSEL`, :const:`AD9834_PSEL`, and + :const:`AD9834_RESET`. To pin control ``FSELECT``, ``PSELECT``, and ``RESET`` set + :const:`AD9834_PIN_SW`. The ``ctrl_reg`` attribute is used to maintain the state of + the control register, enabling persistent management of various configurations. + + :param spi_device: SPI bus device name. + :param spi_freq: SPI bus clock frequency (default: 10 MHz, max: 40 MHz). + :param clk_freq: DDS clock frequency (default: 75 MHz). + :param core_device: Core device name (default: "core"). + """ + + kernel_invariants = {"core", "bus", "spi_freq", "clk_freq"} + + def __init__( + self, dmgr, spi_device, spi_freq=10 * MHz, clk_freq=75 * MHz, core_device="core" + ): + self.core = dmgr.get(core_device) + self.bus = dmgr.get(spi_device) + assert spi_freq <= 40 * MHz, "SPI frequency exceeds maximum value of 40 MHz" + self.spi_freq = spi_freq + self.clk_freq = clk_freq + self.ctrl_reg = 0x0000 # Reset control register + + @kernel + def init(self): + """ + Initialize the AD9834: configure the SPI bus and reset the DDS. + + This method performs the necessary setup for the AD9834 device, including: + - Configuring the SPI bus parameters (clock polarity, data width, and frequency). + - Putting the AD9834 into a reset state to ensure proper initialization. + + The SPI bus is configured to use 16 bits of data width with the clock frequency + provided as a parameter when creating the AD9834 instance. After configuring + the SPI bus, the method invokes :meth:`enable_reset()` to reset the AD9834. + This is an essential step to prepare the device for subsequent configuration + of frequency and phase. + + This method should be called before any other operations are performed + on the AD9834 to ensure that the device is in a known state. + """ + self.bus.set_config(spi.SPI_CLK_POLARITY | spi.SPI_END, 16, self.spi_freq, 1) + self.enable_reset() + + @kernel + def set_frequency_reg(self, freq_reg, frequency: TFloat): + """ + Set the frequency for the specified frequency register. + + This method calculates the frequency word based on the provided frequency in Hz + and writes it to the specified frequency register. + + :param freq_reg: The frequency register to write to, must be one of + :const:`AD9834_FREQ_REG_0` or :const:`AD9834_FREQ_REG_1`. + :param frequency: The desired frequency in Hz, which will be converted to a + frequency word suitable for the AD9834. + + The frequency word is calculated using the formula: + + ``freq_word = (frequency * (1 << 28)) / clk_freq`` + + The result is limited to the lower 28 bits for compatibility with the AD9834. + + The method first sets the control register to enable the appropriate settings, + then sends the fourteen least significant bits LSBs and fourteen most significant + bits MSBs in two consecutive writes to the specified frequency register. + """ + if freq_reg not in FREQ_REGS: + raise ValueError("Invalid frequency register") + assert frequency <= 37.5 * MHz, "Frequency exceeds maximum value of 37.5 MHz" + freq_word = int((frequency * (1 << 28)) / self.clk_freq) & 0x0FFFFFFF + self.ctrl_reg |= AD9834_B28 + self.write(self.ctrl_reg) + lsb = freq_word & 0x3FFF + msb = (freq_word >> 14) & 0x3FFF + self.write(freq_reg | lsb) + self.write(freq_reg | msb) + + @kernel + def set_frequency_reg_msb(self, freq_reg, word: TInt32): + """ + Set the fourteen most significant bits MSBs of the specified frequency register. + + This method updates the specified frequency register with the provided MSB value. + It configures the control register to indicate that the MSB is being set. + + :param freq_reg: The frequency register to update, must be one of + :const:`AD9834_FREQ_REG_0` or :const:`AD9834_FREQ_REG_1`. + :param word: The value to be written to the fourteen MSBs of the frequency register. + + The method first clears the appropriate control bits, sets :const:`AD9834_HLB` to + indicate that the MSB is being sent, and then writes the updated control register + followed by the MSB value to the specified frequency register. + """ + if freq_reg not in FREQ_REGS: + raise ValueError("Invalid frequency register") + self.ctrl_reg &= ~AD9834_B28 + self.ctrl_reg |= AD9834_HLB + self.write(self.ctrl_reg) + self.write(freq_reg | (word & 0x3FFF)) + + @kernel + def set_frequency_reg_lsb(self, freq_reg, word: TInt32): + """ + Set the fourteen least significant bits LSBs of the specified frequency register. + + This method updates the specified frequency register with the provided LSB value. + It configures the control register to indicate that the LSB is being set. + + :param freq_reg: The frequency register to update, must be one of + :const:`AD9834_FREQ_REG_0` or :const:`AD9834_FREQ_REG_1`. + :param word: The value to be written to the fourteen LSBs of the frequency register. + + The method first clears the appropriate control bits and writes the updated control + register followed by the LSB value to the specified frequency register. + """ + if freq_reg not in FREQ_REGS: + raise ValueError("Invalid frequency register") + self.ctrl_reg &= ~AD9834_B28 + self.ctrl_reg &= ~AD9834_HLB + self.write(self.ctrl_reg) + self.write(freq_reg | (word & 0x3FFF)) + + @kernel + def select_frequency_reg(self, freq_reg): + """ + Select the active frequency register for the phase accumulator. + + This method chooses between the two available frequency registers in the AD9834 to + control the frequency of the output waveform. The control register is updated + to reflect the selected frequency register. + + :param freq_reg: The frequency register to select. Must be one of + :const:`AD9834_FREQ_REG_0` or :const:`AD9834_FREQ_REG_1`. + """ + if freq_reg not in FREQ_REGS: + raise ValueError("Invalid frequency register") + if freq_reg == FREQ_REGS[0]: + self.ctrl_reg &= ~AD9834_FSEL + else: + self.ctrl_reg |= AD9834_FSEL + + self.ctrl_reg &= ~AD9834_PIN_SW + self.write(self.ctrl_reg) + + @kernel + def set_phase_reg(self, phase_reg, phase: TInt32): + """ + Set the phase for the specified phase register. + + This method updates the specified phase register with the provided phase value. + + :param phase_reg: The phase register to update, must be one of + :const:`AD9834_PHASE_REG_0` or :const:`AD9834_PHASE_REG_1`. + :param phase: The value to be written to the phase register. + + The method masks the phase value to ensure it fits within the 12-bit limit + and writes it to the specified phase register. + """ + if phase_reg not in PHASE_REGS: + raise ValueError("Invalid phase register") + phase_word = phase & 0x0FFF + self.write(phase_reg | phase_word) + + @kernel + def select_phase_reg(self, phase_reg): + """ + Select the active phase register for the phase accumulator. + + This method chooses between the two available phase registers in the AD9834 to + control the phase of the output waveform. The control register is updated + to reflect the selected phase register. + + :param phase_reg: The phase register to select. Must be one of + :const:`AD9834_PHASE_REG_0` or :const:`AD9834_PHASE_REG_1`. + """ + if phase_reg not in PHASE_REGS: + raise ValueError("Invalid phase register") + if phase_reg == PHASE_REGS[0]: + self.ctrl_reg &= ~AD9834_PSEL + else: + self.ctrl_reg |= AD9834_PSEL + + self.ctrl_reg &= ~AD9834_PIN_SW + self.write(self.ctrl_reg) + + @kernel + def enable_reset(self): + """ + Enable the DDS reset. + + This method sets :const:`AD9834_RESET`, putting the AD9834 into a reset state. + While in this state, the digital-to-analog converter (DAC) is not operational. + + This method should be called during initialization or when a reset is required + to reinitialize the device and ensure proper operation. + """ + self.ctrl_reg |= AD9834_RESET + self.write(self.ctrl_reg) + + @kernel + def output_enable(self): + """ + Disable the DDS reset and start signal generation. + + This method clears :const:`AD9834_RESET`, allowing the AD9834 to begin generating + signals. Once this method is called, the device will resume normal operation and + output the generated waveform. + + This method should be called after configuration of the frequency and phase + settings to activate the output. + """ + self.ctrl_reg &= ~AD9834_RESET + self.write(self.ctrl_reg) + + @kernel + def sleep(self, dac_pd: bool = False, clk_dis: bool = False): + """ + Put the AD9834 into sleep mode by selectively powering down the DAC and/or disabling + the internal clock. + + This method controls the sleep mode behavior of the AD9834 by setting or clearing the + corresponding bits in the control register. Two independent options can be specified: + + :param dac_pd: Set to ``True`` to power down the DAC (:const:`AD9834_SLEEP12` is set). + ``False`` will leave the DAC active. + :param clk_dis: Set to ``True`` to disable the internal clock (:const:`AD9834_SLEEP1` is set). + ``False`` will keep the clock running. + + Both options can be enabled independently, allowing the DAC and/or clock to be powered down as needed. + + The method updates the control register and writes the changes to the AD9834 device. + """ + if dac_pd: + self.ctrl_reg |= AD9834_SLEEP12 + else: + self.ctrl_reg &= ~AD9834_SLEEP12 + + if clk_dis: + self.ctrl_reg |= AD9834_SLEEP1 + else: + self.ctrl_reg &= ~AD9834_SLEEP1 + + self.write(self.ctrl_reg) + + @kernel + def awake(self): + """ + Exit sleep mode and restore normal operation. + + This method brings the AD9834 out of sleep mode by clearing any DAC power-down or + internal clock disable settings. It calls :meth:`sleep()` with no arguments, + effectively setting both ``dac_powerdown`` and ``internal_clk_disable`` to ``False``. + + The device will resume generating output based on the current frequency and phase + settings. + """ + self.sleep() + + @kernel + def config_sign_bit_out( + self, + high_z: bool = False, + msb_2: bool = False, + msb: bool = False, + comp_out: bool = False, + ): + """ + Configure the ``SIGN BIT OUT`` pin for various output modes. + + This method sets the output mode for the ``SIGN BIT OUT`` pin of the AD9834 based on the provided flags. + The user can enable one of several modes, including high impedance, MSB/2 output, MSB output, + or comparator output. These modes are mutually exclusive, and passing ``True`` to one flag will + configure the corresponding mode, while other flags should be left as ``False``. + + :param high_z: Set to ``True`` to place the ``SIGN BIT OUT`` pin in high impedance (disabled) mode. + :param msb_2: Set to ``True`` to output DAC Data MSB divided by 2 on the ``SIGN BIT OUT`` pin. + :param msb: Set to ``True`` to output DAC Data MSB on the ``SIGN BIT OUT`` pin. + :param comp_out: Set to ``True`` to output the comparator signal on the ``SIGN BIT OUT`` pin. + + Only one flag should be set to ``True`` at a time. If no valid mode is selected, the ``SIGN BIT OUT`` + pin will default to high impedance mode. + + The method updates the control register with the appropriate configuration and writes it to the AD9834. + """ + if high_z: + self.ctrl_reg &= ~AD9834_OPBITEN + elif msb_2: + self.ctrl_reg |= AD9834_OPBITEN + self.ctrl_reg &= ~AD9834_MODE + self.ctrl_reg &= ~AD9834_SIGN_PIB + self.ctrl_reg &= ~AD9834_DIV2 + elif msb: + self.ctrl_reg |= AD9834_OPBITEN + self.ctrl_reg &= ~AD9834_MODE + self.ctrl_reg &= ~AD9834_SIGN_PIB + self.ctrl_reg |= AD9834_DIV2 + elif comp_out: + self.ctrl_reg |= AD9834_OPBITEN + self.ctrl_reg &= ~AD9834_MODE + self.ctrl_reg |= AD9834_SIGN_PIB + self.ctrl_reg |= AD9834_DIV2 + else: + self.ctrl_reg &= ~AD9834_OPBITEN + + self.write(self.ctrl_reg) + + @kernel + def enable_triangular_waveform(self): + """ + Enable triangular waveform generation. + + This method configures the AD9834 to output a triangular waveform. It does so + by clearing :const:`AD9834_OPBITEN` in the control register and setting :const:`AD9834_MODE`. + Once this method is called, the AD9834 will begin generating a triangular waveform + at the frequency set for the selected frequency register. + + This method should be called when a triangular waveform is desired for signal + generation. Ensure that the frequency is set appropriately before invoking this method. + """ + self.ctrl_reg &= ~AD9834_OPBITEN + self.ctrl_reg |= AD9834_MODE + self.write(self.ctrl_reg) + + @kernel + def disable_triangular_waveform(self): + """ + Disable triangular waveform generation. + + This method disables the triangular waveform output by clearing :const:`AD9834_MODE`. + After invoking this method, the AD9834 will cease generating a triangular waveform. + The device can then be configured to output other waveform types if needed. + + This method should be called when switching to a different waveform type or + when the triangular waveform is no longer required. + """ + self.ctrl_reg &= ~AD9834_MODE + self.write(self.ctrl_reg) + + @kernel + def write(self, data: TInt32): + """ + Write a 16-bit word to the AD9834. + + This method sends a 16-bit data word to the AD9834 via the SPI bus. The input + data is left-shifted by 16 bits to ensure proper alignment for the SPI controller, + allowing for accurate processing of the command by the AD9834. + + This method is used internally by other methods to update the control registers + and frequency settings of the AD9834. It should not be called directly unless + low-level register manipulation is required. + + :param data: The 16-bit word to be sent to the AD9834. + """ + self.bus.write(data << 16) diff --git a/artiq/test/coredevice/test_ad9834.py b/artiq/test/coredevice/test_ad9834.py new file mode 100644 index 000000000..0a1eadb36 --- /dev/null +++ b/artiq/test/coredevice/test_ad9834.py @@ -0,0 +1,321 @@ +from artiq.coredevice.ad9834 import ( + AD9834_B28, + AD9834_DIV2, + AD9834_FSEL, + AD9834_HLB, + AD9834_MODE, + AD9834_OPBITEN, + AD9834_PIN_SW, + AD9834_PSEL, + AD9834_RESET, + AD9834_SIGN_PIB, + AD9834_SLEEP1, + AD9834_SLEEP12, + FREQ_REGS, + PHASE_REGS, +) +from artiq.experiment import * +from artiq.language.units import MHz +from artiq.test.hardware_testbench import ExperimentCase + + +class AD9834Exp(EnvExperiment): + def build(self, runner): + self.setattr_device("core") + self.dev = self.get_device("dds0") + self.runner = runner + + def run(self): + getattr(self, self.runner)() + + @kernel + def instantiate(self): + pass + + @kernel + def init(self): + self.core.break_realtime() + self.dev.init() + self.set_dataset("spi_freq", self.dev.spi_freq) + self.set_dataset("clk_freq", self.dev.clk_freq) + self.set_dataset("ctrl_reg", self.dev.ctrl_reg) + + @kernel + def set_frequency_reg_fail1(self): + self.core.break_realtime() + frequency = 10 * MHz + self.dev.set_frequency_reg(19, frequency) + + @kernel + def set_frequency_reg_fail2(self): + self.core.break_realtime() + self.dev.set_frequency_reg(FREQ_REGS[0], 37.6 * MHz) + + @kernel + def set_frequency_reg(self): + self.core.break_realtime() + self.dev.init() + self.dev.set_frequency_reg(FREQ_REGS[1], 19 * MHz) + self.set_dataset("ctrl_reg", self.dev.ctrl_reg) + + @kernel + def set_frequency_reg_msb(self): + self.core.break_realtime() + self.dev.init() + self.dev.ctrl_reg |= AD9834_B28 + self.dev.set_frequency_reg_msb(FREQ_REGS[0], 0x1111) + self.set_dataset("ctrl_reg", self.dev.ctrl_reg) + + @kernel + def set_frequency_reg_lsb(self): + self.core.break_realtime() + self.dev.init() + self.dev.ctrl_reg |= AD9834_B28 | AD9834_HLB + self.dev.set_frequency_reg_lsb(FREQ_REGS[1], 0xFFFF) + self.set_dataset("ctrl_reg", self.dev.ctrl_reg) + + @kernel + def select_frequency_reg_0(self): + self.core.break_realtime() + self.dev.ctrl_reg |= AD9834_FSEL | AD9834_PIN_SW + self.dev.select_frequency_reg(FREQ_REGS[0]) + self.set_dataset("ctrl_reg", self.dev.ctrl_reg) + + @kernel + def select_frequency_reg_1(self): + self.core.break_realtime() + self.dev.ctrl_reg |= AD9834_PIN_SW + self.dev.select_frequency_reg(FREQ_REGS[1]) + self.set_dataset("ctrl_reg", self.dev.ctrl_reg) + + @kernel + def set_phase_reg_fail(self): + self.core.break_realtime() + self.dev.set_phase_reg(19, 0x123) + + @kernel + def set_phase_reg(self): + self.core.break_realtime() + self.dev.init() + self.dev.set_phase_reg(PHASE_REGS[0], 0x123) + self.set_dataset("ctrl_reg", self.dev.ctrl_reg) + + @kernel + def select_phase_reg_0(self): + self.core.break_realtime() + self.dev.ctrl_reg |= AD9834_PSEL | AD9834_PIN_SW + self.dev.select_phase_reg(PHASE_REGS[0]) + self.set_dataset("ctrl_reg", self.dev.ctrl_reg) + + @kernel + def select_phase_reg_1(self): + self.core.break_realtime() + self.dev.ctrl_reg |= AD9834_PIN_SW + self.dev.select_phase_reg(PHASE_REGS[1]) + self.set_dataset("ctrl_reg", self.dev.ctrl_reg) + + @kernel + def enable_reset(self): + self.core.break_realtime() + self.dev.enable_reset() + self.set_dataset("ctrl_reg", self.dev.ctrl_reg) + + @kernel + def output_enable(self): + self.core.break_realtime() + self.dev.ctrl_reg |= AD9834_RESET + self.dev.output_enable() + self.set_dataset("ctrl_reg", self.dev.ctrl_reg) + + @kernel + def sleep_dac_powerdown(self): + self.core.break_realtime() + self.dev.sleep(dac_pd=True) + self.set_dataset("ctrl_reg", self.dev.ctrl_reg) + + @kernel + def sleep_internal_clk_disable(self): + self.core.break_realtime() + self.dev.sleep(clk_dis=True) + self.set_dataset("ctrl_reg", self.dev.ctrl_reg) + + @kernel + def sleep(self): + self.core.break_realtime() + self.dev.sleep(dac_pd=True, clk_dis=True) + self.set_dataset("ctrl_reg", self.dev.ctrl_reg) + + @kernel + def awake(self): + self.core.break_realtime() + self.dev.ctrl_reg |= AD9834_SLEEP1 | AD9834_SLEEP12 + self.dev.sleep() + self.set_dataset("ctrl_reg", self.dev.ctrl_reg) + + @kernel + def sign_bit_high_z(self): + self.core.break_realtime() + self.dev.ctrl_reg |= AD9834_OPBITEN + self.dev.config_sign_bit_out() + self.set_dataset("ctrl_reg", self.dev.ctrl_reg) + + @kernel + def sign_bit_msb_2(self): + self.core.break_realtime() + self.dev.ctrl_reg |= AD9834_MODE | AD9834_SIGN_PIB | AD9834_DIV2 + self.dev.config_sign_bit_out(msb_2=True) + self.set_dataset("ctrl_reg", self.dev.ctrl_reg) + + @kernel + def sign_bit_msb(self): + self.core.break_realtime() + self.dev.ctrl_reg |= AD9834_MODE | AD9834_SIGN_PIB + self.dev.config_sign_bit_out(msb=True) + self.set_dataset("ctrl_reg", self.dev.ctrl_reg) + + @kernel + def sign_bit_comp_out(self): + self.core.break_realtime() + self.dev.ctrl_reg |= AD9834_MODE + self.dev.config_sign_bit_out(comp_out=True) + self.set_dataset("ctrl_reg", self.dev.ctrl_reg) + + @kernel + def enable_triangular_waveform(self): + self.core.break_realtime() + self.dev.ctrl_reg |= AD9834_OPBITEN + self.dev.enable_triangular_waveform() + self.set_dataset("ctrl_reg", self.dev.ctrl_reg) + + @kernel + def disable_triangular_waveform(self): + self.core.break_realtime() + self.dev.ctrl_reg |= AD9834_MODE + self.dev.disable_triangular_waveform() + self.set_dataset("ctrl_reg", self.dev.ctrl_reg) + + +class AD9834Test(ExperimentCase): + def test_instantiate(self): + self.execute(AD9834Exp, "instantiate") + + def test_init(self): + self.execute(AD9834Exp, "init") + spi_freq = self.dataset_mgr.get("spi_freq") + clk_freq = self.dataset_mgr.get("clk_freq") + ctrl_reg = self.dataset_mgr.get("ctrl_reg") + self.assertEqual(spi_freq, 10 * MHz) + self.assertEqual(clk_freq, 75 * MHz) + self.assertEqual(ctrl_reg, 0x0000 | AD9834_RESET) + + def test_set_frequency_reg_fail(self): + with self.assertRaises(ValueError): + self.execute(AD9834Exp, "set_frequency_reg_fail1") + with self.assertRaises(AssertionError): + self.execute(AD9834Exp, "set_frequency_reg_fail2") + + def test_set_frequency_reg(self): + self.execute(AD9834Exp, "set_frequency_reg") + ctrl_reg = self.dataset_mgr.get("ctrl_reg") + self.assertEqual(ctrl_reg, 0x0000 | AD9834_RESET | AD9834_B28) + + def test_set_frequency_reg_msb(self): + self.execute(AD9834Exp, "set_frequency_reg_msb") + ctrl_reg = self.dataset_mgr.get("ctrl_reg") + self.assertEqual(ctrl_reg, 0x0000 | AD9834_RESET | AD9834_HLB) + + def test_set_frequency_reg_lsb(self): + self.execute(AD9834Exp, "set_frequency_reg_lsb") + ctrl_reg = self.dataset_mgr.get("ctrl_reg") + self.assertEqual(ctrl_reg, 0x0000 | AD9834_RESET) + + def test_select_frequency_reg_0(self): + self.execute(AD9834Exp, "select_frequency_reg_0") + ctrl_reg = self.dataset_mgr.get("ctrl_reg") + self.assertEqual(ctrl_reg, 0x0000) + + def test_select_frequency_reg_1(self): + self.execute(AD9834Exp, "select_frequency_reg_1") + ctrl_reg = self.dataset_mgr.get("ctrl_reg") + self.assertEqual(ctrl_reg, 0x0000 | AD9834_FSEL) + + def test_set_phase_reg_fail(self): + with self.assertRaises(ValueError): + self.execute(AD9834Exp, "set_phase_reg_fail") + + def test_set_phase_reg(self): + self.execute(AD9834Exp, "set_phase_reg") + ctrl_reg = self.dataset_mgr.get("ctrl_reg") + self.assertEqual(ctrl_reg, 0x0000 | AD9834_RESET) + + def test_select_phase_reg_0(self): + self.execute(AD9834Exp, "select_phase_reg_0") + ctrl_reg = self.dataset_mgr.get("ctrl_reg") + self.assertEqual(ctrl_reg, 0x0000) + + def test_select_phase_reg_1(self): + self.execute(AD9834Exp, "select_phase_reg_1") + ctrl_reg = self.dataset_mgr.get("ctrl_reg") + self.assertEqual(ctrl_reg, 0x0000 | AD9834_PSEL) + + def test_enable_reset(self): + self.execute(AD9834Exp, "enable_reset") + ctrl_reg = self.dataset_mgr.get("ctrl_reg") + self.assertEqual(ctrl_reg, 0x0000 | AD9834_RESET) + + def test_output_enable(self): + self.execute(AD9834Exp, "output_enable") + ctrl_reg = self.dataset_mgr.get("ctrl_reg") + self.assertEqual(ctrl_reg, 0x0000) + + def test_sleep_dac_powerdown(self): + self.execute(AD9834Exp, "sleep_dac_powerdown") + ctrl_reg = self.dataset_mgr.get("ctrl_reg") + self.assertEqual(ctrl_reg, 0x0000 | AD9834_SLEEP12) + + def test_sleep_internal_clk_disable(self): + self.execute(AD9834Exp, "sleep_internal_clk_disable") + ctrl_reg = self.dataset_mgr.get("ctrl_reg") + self.assertEqual(ctrl_reg, 0x0000 | AD9834_SLEEP1) + + def test_sleep(self): + self.execute(AD9834Exp, "sleep") + ctrl_reg = self.dataset_mgr.get("ctrl_reg") + self.assertEqual(ctrl_reg, 0x0000 | AD9834_SLEEP1 | AD9834_SLEEP12) + + def test_awake(self): + self.execute(AD9834Exp, "awake") + ctrl_reg = self.dataset_mgr.get("ctrl_reg") + self.assertEqual(ctrl_reg, 0x0000) + + def test_sign_bit_high_z(self): + self.execute(AD9834Exp, "sign_bit_high_z") + ctrl_reg = self.dataset_mgr.get("ctrl_reg") + self.assertEqual(ctrl_reg, 0x0000) + + def test_sign_bit_msb_2(self): + self.execute(AD9834Exp, "sign_bit_msb_2") + ctrl_reg = self.dataset_mgr.get("ctrl_reg") + self.assertEqual(ctrl_reg, 0x0000 | AD9834_OPBITEN) + + def test_sign_bit_msb(self): + self.execute(AD9834Exp, "sign_bit_msb") + ctrl_reg = self.dataset_mgr.get("ctrl_reg") + self.assertEqual(ctrl_reg, 0x0000 | AD9834_OPBITEN | AD9834_DIV2) + + def test_sign_bit_comp_out(self): + self.execute(AD9834Exp, "sign_bit_comp_out") + ctrl_reg = self.dataset_mgr.get("ctrl_reg") + self.assertEqual( + ctrl_reg, 0x0000 | AD9834_OPBITEN | AD9834_SIGN_PIB | AD9834_DIV2 + ) + + def test_enble_triangular_waveform(self): + self.execute(AD9834Exp, "enable_triangular_waveform") + ctrl_reg = self.dataset_mgr.get("ctrl_reg") + self.assertEqual(ctrl_reg, 0x0000 | AD9834_MODE) + + def test_disble_triangular_waveform(self): + self.execute(AD9834Exp, "disable_triangular_waveform") + ctrl_reg = self.dataset_mgr.get("ctrl_reg") + self.assertEqual(ctrl_reg, 0x0000) diff --git a/doc/manual/core_device.rst b/doc/manual/core_device.rst index b55c638be..4177e6414 100644 --- a/doc/manual/core_device.rst +++ b/doc/manual/core_device.rst @@ -156,15 +156,39 @@ Common KC705 problems * When connected, the CLOCK adapter breaks the JTAG chain due to TDI not being connected to TDO on the FMC mezzanine. * On some boards, the JTAG USB connector is not correctly soldered. +VADJ +"""" + +With the NIST CLOCK and QC2 adapters, for safe operation of the DDS buses (to prevent damage to the IO banks of the FPGA), the FMC VADJ rail of the KC705 should be changed to 3.3V. Plug the Texas Instruments USB-TO-GPIO PMBus adapter into the PMBus connector in the corner of the KC705 and use the Fusion Digital Power Designer software to configure (requires Windows). Write to chip number U55 (address 52), channel 4, which is the VADJ rail, to make it 3.3V instead of 2.5V. Power cycle the KC705 board to check that the startup voltage on the VADJ rail is now 3.3V. + EBAZ4205 ^^^^^^^^ The `EBAZ4205 `_ Zynq-SoC control card, originally used in the Ebit E9+ BTC miner, is a low-cost development board (around $20-$30 USD), making it an ideal option for experimenting with ARTIQ. To use the EBAZ4205, it's important to carefully follow the board documentation to configure it to boot from the SD card, as network booting via ``artiq_netboot`` is currently unsupported. This is because the Ethernet PHY is routed through the EMIO, requiring the FPGA to be programmed before the board can establish a network connection. -VADJ -"""" +One useful application of the EBAZ4205 is controlling external devices like the AD9834 DDS Module from ZonRi Technology Co., Ltd. To establish communication between the EBAZ4205 and the AD9834 module, proper configuration of the SPI interface pins is essential. The board's flexibility allows for straightforward control of the DDS once the correct pinout is known. The table below details the necessary connections between the EBAZ4205 and the AD9834 module, including power, ground, and SPI signals. -With the NIST CLOCK and QC2 adapters, for safe operation of the DDS buses (to prevent damage to the IO banks of the FPGA), the FMC VADJ rail of the KC705 should be changed to 3.3V. Plug the Texas Instruments USB-TO-GPIO PMBus adapter into the PMBus connector in the corner of the KC705 and use the Fusion Digital Power Designer software to configure (requires Windows). Write to chip number U55 (address 52), channel 4, which is the VADJ rail, to make it 3.3V instead of 2.5V. Power cycle the KC705 board to check that the startup voltage on the VADJ rail is now 3.3V. ++--------------------------+---------------------+----------------------------+ +| Pin on AD9834 Module | Chip Function | Connection on EBAZ4205 | ++==========================+=====================+============================+ +| SCLK | SCLK | CLK: DATA3-19 (Pin V20) | ++--------------------------+---------------------+----------------------------+ +| DATA | SDATA | MOSI: DATA3-17 (Pin U20) | ++--------------------------+---------------------+----------------------------+ +| SYNC | FSYNC | CS_N: DATA3-15 (Pin P19) | ++--------------------------+---------------------+----------------------------+ +| FSE (Tied to GND) | FSELECT | N/A: Bit Controlled | ++--------------------------+---------------------+----------------------------+ +| PSE (Tied to GND) | PSELECT | N/A: Bit Controlled | ++--------------------------+---------------------+----------------------------+ +| GND | Ground | GND: J8-1, J8-3 | ++--------------------------+---------------------+----------------------------+ +| VIN | AVDD/DVDD | 3.3V: J8-2 | ++--------------------------+---------------------+----------------------------+ +| RESET (Unused) | RESET | N/A: Bit Controlled | ++--------------------------+---------------------+----------------------------+ + +For a step-by-step guide, see the `EBAZ4205 and AD9834 setup guide `_. Variant details --------------- diff --git a/doc/manual/core_drivers_reference.rst b/doc/manual/core_drivers_reference.rst index c7f8de3d6..56d6cd46d 100644 --- a/doc/manual/core_drivers_reference.rst +++ b/doc/manual/core_drivers_reference.rst @@ -85,6 +85,12 @@ RF generation drivers .. automodule:: artiq.coredevice.ad9914 :members: +:mod:`artiq.coredevice.ad9834` module +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +.. automodule:: artiq.coredevice.ad9834 + :members: + :mod:`artiq.coredevice.mirny` module ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ From d602cdbc1f07864569a51c3efd2b7efca8db4287 Mon Sep 17 00:00:00 2001 From: Phillip Klein Date: Thu, 17 Oct 2024 15:38:07 +0200 Subject: [PATCH 033/111] fix typo --- doc/manual/flashing.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/manual/flashing.rst b/doc/manual/flashing.rst index abfc5ec13..bf14d3296 100644 --- a/doc/manual/flashing.rst +++ b/doc/manual/flashing.rst @@ -13,7 +13,7 @@ If you have an active firmware subscription with M-Labs or QUARTIQ, you can obta Run the command:: - $ afws_client build + $ afws_client build Replace ```` with the login name that was given to you with the subscription, ```` with the name of your system variant, and ```` with the name of an empty directory, which will be created by the command if it does not exist. Enter your password when prompted and wait for the build (if applicable) and download to finish. If you experience issues with the AFWS client, write to the helpdesk@ email. For more information about :mod:`~artiq.frontend.afws_client` see also the corresponding entry on the :ref:`Utilities ` page. From 4178fed3f76144f5501a8f3a1cb62c3b55a9dc53 Mon Sep 17 00:00:00 2001 From: mwojcik Date: Fri, 18 Oct 2024 12:36:09 +0800 Subject: [PATCH 034/111] subkernels: send now_mu when starting a subkernel --- artiq/firmware/ksupport/lib.rs | 27 ++++++++++++++++++- .../firmware/libproto_artiq/drtioaux_proto.rs | 8 +++--- artiq/firmware/libproto_artiq/kernel_proto.rs | 4 ++- artiq/firmware/runtime/kernel.rs | 5 ++-- artiq/firmware/runtime/rtio_mgt.rs | 7 +++-- artiq/firmware/runtime/session.rs | 4 +-- artiq/firmware/satman/kernel.rs | 16 +++++------ artiq/firmware/satman/main.rs | 4 +-- 8 files changed, 54 insertions(+), 21 deletions(-) diff --git a/artiq/firmware/ksupport/lib.rs b/artiq/firmware/ksupport/lib.rs index 042f3b429..4f8893a87 100644 --- a/artiq/firmware/ksupport/lib.rs +++ b/artiq/firmware/ksupport/lib.rs @@ -37,6 +37,14 @@ fn recv R>(f: F) -> R { result } +fn try_recv(f: F) { + let msg_ptr = mailbox::receive(); + if msg_ptr != 0 { + f(unsafe { &*(msg_ptr as *const Message) }); + mailbox::acknowledge(); + } +} + macro_rules! recv { ($p:pat => $e:expr) => { recv(move |request| { @@ -473,7 +481,15 @@ extern "C-unwind" fn dma_playback(timestamp: i64, ptr: i32, _uses_ddma: bool) { extern "C-unwind" fn subkernel_load_run(id: u32, destination: u8, run: bool) { - send(&SubkernelLoadRunRequest { id: id, destination: destination, run: run }); + let timestamp = unsafe { + ((csr::rtio::now_hi_read() as u64) << 32) | (csr::rtio::now_lo_read() as u64) + }; + send(&SubkernelLoadRunRequest { + id: id, + destination: destination, + run: run, + timestamp: timestamp, + }); recv!(&SubkernelLoadRunReply { succeeded } => { if !succeeded { raise!("SubkernelError", @@ -601,6 +617,15 @@ pub unsafe fn main() { }, Ok(library) => { send(&LoadReply(Ok(()))); + // Master kernel would just acknowledge kernel load + // Satellites may send UpdateNow + try_recv(move |msg| match msg { + UpdateNow(timestamp) => unsafe { + csr::rtio::now_hi_write((*timestamp >> 32) as u32); + csr::rtio::now_lo_write(*timestamp as u32); + } + _ => unreachable!() + }); library } } diff --git a/artiq/firmware/libproto_artiq/drtioaux_proto.rs b/artiq/firmware/libproto_artiq/drtioaux_proto.rs index ec1c36686..a1b17e888 100644 --- a/artiq/firmware/libproto_artiq/drtioaux_proto.rs +++ b/artiq/firmware/libproto_artiq/drtioaux_proto.rs @@ -120,7 +120,7 @@ pub enum Packet { SubkernelAddDataRequest { destination: u8, id: u32, status: PayloadStatus, length: u16, data: [u8; MASTER_PAYLOAD_MAX_SIZE] }, SubkernelAddDataReply { succeeded: bool }, - SubkernelLoadRunRequest { source: u8, destination: u8, id: u32, run: bool }, + SubkernelLoadRunRequest { source: u8, destination: u8, id: u32, run: bool, timestamp: u64 }, SubkernelLoadRunReply { destination: u8, succeeded: bool }, SubkernelFinished { destination: u8, id: u32, with_exception: bool, exception_src: u8 }, SubkernelExceptionRequest { source: u8, destination: u8 }, @@ -354,7 +354,8 @@ impl Packet { source: reader.read_u8()?, destination: reader.read_u8()?, id: reader.read_u32()?, - run: reader.read_bool()? + run: reader.read_bool()?, + timestamp: reader.read_u64()? }, 0xc5 => Packet::SubkernelLoadRunReply { destination: reader.read_u8()?, @@ -647,12 +648,13 @@ impl Packet { writer.write_u8(0xc1)?; writer.write_bool(succeeded)?; }, - Packet::SubkernelLoadRunRequest { source, destination, id, run } => { + Packet::SubkernelLoadRunRequest { source, destination, id, run, timestamp } => { writer.write_u8(0xc4)?; writer.write_u8(source)?; writer.write_u8(destination)?; writer.write_u32(id)?; writer.write_bool(run)?; + writer.write_u64(timestamp)?; }, Packet::SubkernelLoadRunReply { destination, succeeded } => { writer.write_u8(0xc5)?; diff --git a/artiq/firmware/libproto_artiq/kernel_proto.rs b/artiq/firmware/libproto_artiq/kernel_proto.rs index 31aece5c4..31df20abf 100644 --- a/artiq/firmware/libproto_artiq/kernel_proto.rs +++ b/artiq/firmware/libproto_artiq/kernel_proto.rs @@ -103,7 +103,7 @@ pub enum Message<'a> { SpiReadReply { succeeded: bool, data: u32 }, SpiBasicReply { succeeded: bool }, - SubkernelLoadRunRequest { id: u32, destination: u8, run: bool }, + SubkernelLoadRunRequest { id: u32, destination: u8, run: bool, timestamp: u64 }, SubkernelLoadRunReply { succeeded: bool }, SubkernelAwaitFinishRequest { id: u32, timeout: i64 }, SubkernelAwaitFinishReply, @@ -112,6 +112,8 @@ pub enum Message<'a> { SubkernelMsgRecvReply { count: u8 }, SubkernelError(SubkernelStatus<'a>), + UpdateNow(u64), + Log(fmt::Arguments<'a>), LogSlice(&'a str) } diff --git a/artiq/firmware/runtime/kernel.rs b/artiq/firmware/runtime/kernel.rs index 545f75981..ab0ac31ab 100644 --- a/artiq/firmware/runtime/kernel.rs +++ b/artiq/firmware/runtime/kernel.rs @@ -194,14 +194,15 @@ pub mod subkernel { } pub fn load(io: &Io, aux_mutex: &Mutex, ddma_mutex: &Mutex, subkernel_mutex: &Mutex, routing_table: &RoutingTable, - id: u32, run: bool) -> Result<(), Error> { + id: u32, run: bool, timestamp: u64) -> Result<(), Error> { let _lock = subkernel_mutex.lock(io)?; let subkernel = unsafe { SUBKERNELS.get_mut(&id).unwrap() }; if subkernel.state != SubkernelState::Uploaded { error!("for id: {} expected Uploaded, got: {:?}", id, subkernel.state); return Err(Error::IncorrectState); } - drtio::subkernel_load(io, aux_mutex, ddma_mutex, subkernel_mutex, routing_table, id, subkernel.destination, run)?; + drtio::subkernel_load(io, aux_mutex, ddma_mutex, subkernel_mutex, + routing_table, id, subkernel.destination, run, timestamp)?; if run { subkernel.state = SubkernelState::Running; } diff --git a/artiq/firmware/runtime/rtio_mgt.rs b/artiq/firmware/runtime/rtio_mgt.rs index 3ed892d2a..d065214a7 100644 --- a/artiq/firmware/runtime/rtio_mgt.rs +++ b/artiq/firmware/runtime/rtio_mgt.rs @@ -594,11 +594,14 @@ pub mod drtio { } pub fn subkernel_load(io: &Io, aux_mutex: &Mutex, ddma_mutex: &Mutex, subkernel_mutex: &Mutex, - routing_table: &drtio_routing::RoutingTable, id: u32, destination: u8, run: bool + routing_table: &drtio_routing::RoutingTable, id: u32, destination: u8, run: bool, timestamp: u64 ) -> Result<(), Error> { let linkno = routing_table.0[destination as usize][0] - 1; let reply = aux_transact(io, aux_mutex, ddma_mutex, subkernel_mutex, routing_table, linkno, - &drtioaux::Packet::SubkernelLoadRunRequest{ id: id, source: 0, destination: destination, run: run })?; + &drtioaux::Packet::SubkernelLoadRunRequest{ + id: id, source: 0, destination: destination, + run: run, timestamp: timestamp + })?; match reply { drtioaux::Packet::SubkernelLoadRunReply { destination: 0, succeeded: true } => Ok(()), drtioaux::Packet::SubkernelLoadRunReply { destination: 0, succeeded: false } => diff --git a/artiq/firmware/runtime/session.rs b/artiq/firmware/runtime/session.rs index 579b2094c..6e849e87a 100644 --- a/artiq/firmware/runtime/session.rs +++ b/artiq/firmware/runtime/session.rs @@ -661,9 +661,9 @@ fn process_kern_message(io: &Io, aux_mutex: &Mutex, } } #[cfg(has_drtio)] - &kern::SubkernelLoadRunRequest { id, destination: _, run } => { + &kern::SubkernelLoadRunRequest { id, destination: _, run, timestamp } => { let succeeded = match subkernel::load( - io, aux_mutex, ddma_mutex, subkernel_mutex, routing_table, id, run) { + io, aux_mutex, ddma_mutex, subkernel_mutex, routing_table, id, run, timestamp) { Ok(()) => true, Err(e) => { error!("Error loading subkernel: {}", e); false } }; diff --git a/artiq/firmware/satman/kernel.rs b/artiq/firmware/satman/kernel.rs index 6f8f5b7c7..c47f3515e 100644 --- a/artiq/firmware/satman/kernel.rs +++ b/artiq/firmware/satman/kernel.rs @@ -363,17 +363,17 @@ impl Manager { unsafe { self.cache.unborrow() } } - pub fn run(&mut self, source: u8, id: u32) -> Result<(), Error> { + pub fn run(&mut self, source: u8, id: u32, timestamp: u64) -> Result<(), Error> { info!("starting subkernel #{}", id); if self.session.kernel_state != KernelState::Loaded || self.current_id != id { self.load(id)?; - } + } self.session.source = source; self.session.kernel_state = KernelState::Running; cricon_select(RtioMaster::Kernel); - kern_acknowledge() + kern_send(&kern::UpdateNow(timestamp)) } pub fn message_handle_incoming(&mut self, status: PayloadStatus, length: usize, id: u32, slice: &[u8; MASTER_PAYLOAD_MAX_SIZE]) { @@ -825,21 +825,21 @@ impl Manager { // ID equal to -1 indicates wildcard for receiving arguments let id = if id == -1 { self.current_id } else { id as u32 }; self.session.kernel_state = KernelState::MsgAwait { - id: id, max_time: max_time, tags: tags.to_vec() }; + id, max_time, tags: tags.to_vec() }; Ok(()) }, - &kern::SubkernelLoadRunRequest { id, destination: sk_destination, run } => { + &kern::SubkernelLoadRunRequest { id, destination: sk_destination, run, timestamp } => { self.session.kernel_state = KernelState::SubkernelAwaitLoad; router.route(drtioaux::Packet::SubkernelLoadRunRequest { - source: destination, destination: sk_destination, id: id, run: run + source: destination, destination: sk_destination, id, run, timestamp }, routing_table, rank, destination); Ok(()) } - &kern::SubkernelAwaitFinishRequest{ id, timeout } => { + &kern::SubkernelAwaitFinishRequest { id, timeout } => { let max_time = if timeout > 0 { clock::get_ms() as i64 + timeout } else { timeout }; - self.session.kernel_state = KernelState::SubkernelAwaitFinish { max_time: max_time, id: id }; + self.session.kernel_state = KernelState::SubkernelAwaitFinish { max_time, id }; Ok(()) } diff --git a/artiq/firmware/satman/main.rs b/artiq/firmware/satman/main.rs index 8f5470e3e..11da3e1a1 100644 --- a/artiq/firmware/satman/main.rs +++ b/artiq/firmware/satman/main.rs @@ -427,7 +427,7 @@ fn process_aux_packet(dmamgr: &mut DmaManager, analyzer: &mut Analyzer, kernelmg drtioaux::send(0, &drtioaux::Packet::SubkernelAddDataReply { succeeded: succeeded }) } - drtioaux::Packet::SubkernelLoadRunRequest { source, destination: _destination, id, run } => { + drtioaux::Packet::SubkernelLoadRunRequest { source, destination: _destination, id, run, timestamp } => { forward!(router, _routing_table, _destination, *rank, *self_destination, _repeaters, &packet); let mut succeeded = kernelmgr.load(id).is_ok(); // allow preloading a kernel with delayed run @@ -436,7 +436,7 @@ fn process_aux_packet(dmamgr: &mut DmaManager, analyzer: &mut Analyzer, kernelmg // cannot run kernel while DDMA is running succeeded = false; } else { - succeeded |= kernelmgr.run(source, id).is_ok(); + succeeded |= kernelmgr.run(source, id, timestamp).is_ok(); } } router.send(drtioaux::Packet::SubkernelLoadRunReply { From fd2df7ce68bbcd38732db93f2d055907b42641fa Mon Sep 17 00:00:00 2001 From: mwojcik Date: Fri, 18 Oct 2024 13:25:20 +0800 Subject: [PATCH 035/111] docs: mention that subkernels pass now_mu --- doc/manual/using_drtio_subkernels.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/doc/manual/using_drtio_subkernels.rst b/doc/manual/using_drtio_subkernels.rst index 74a257bfe..780990c8b 100644 --- a/doc/manual/using_drtio_subkernels.rst +++ b/doc/manual/using_drtio_subkernels.rst @@ -132,6 +132,8 @@ If a subkernel is called on a satellite where a kernel is already running, the n If a subkernel is complex and its binary relatively large, the delay between the call and actually running the subkernel may be substantial. If it's necessary to minimize this delay, ``subkernel_preload(function)`` should be used before the call. +Subkernels receive the value of the timeline cursor ``now_mu`` from the caller at the moment of the call. As there is a delay between calling the subkernel and its actual start, there will be a difference in ``now_mu`` that can be compensated with a delay in the subkernel. Additionally, preloading the subkernel would decrease the difference, as the subkernel does not have to be loaded before running. + While a subkernel is running, the satellite is disconnected from the RTIO interface of the master. As a result, regardless of what devices the subkernel itself uses, none of the RTIO devices on that satellite will be available to the master, nor will messages be passed on to any further satellites downstream. This applies both to regular RTIO operations and DDMA. While a subkernel is running, a satellite may use its own local DMA, but an attempt by any other device to run DDMA through the satellite will fail. Control is returned to the master when no subkernel is running -- to be sure that a device will be accessible, await before performing any RTIO operations on the affected satellite. .. note:: From 58ea3b5bcce87cb02d815c622f7a1d85cf71300f Mon Sep 17 00:00:00 2001 From: newell Date: Thu, 17 Oct 2024 09:37:53 -0700 Subject: [PATCH 036/111] Fix some typos --- artiq/coredevice/ad9910.py | 2 +- artiq/coredevice/suservo.py | 8 ++++---- artiq/gui/applets.py | 2 +- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/artiq/coredevice/ad9910.py b/artiq/coredevice/ad9910.py index f6d7928ab..821411b8e 100644 --- a/artiq/coredevice/ad9910.py +++ b/artiq/coredevice/ad9910.py @@ -195,7 +195,7 @@ class AD9910: when changing frequency or phase. The DDS phase is the sum of the phase accumulator and the phase offset. The only discontinuous changes in the DDS output phase come from changes to the phase - offset. This mode is also knows as "relative phase mode". + offset. This mode is also known as "relative phase mode". :math:`\phi(t) = q(t^\prime) + p + (t - t^\prime) f` * :const:`PHASE_MODE_ABSOLUTE`: the phase accumulator is reset when diff --git a/artiq/coredevice/suservo.py b/artiq/coredevice/suservo.py index 2e82a8539..7bb000218 100644 --- a/artiq/coredevice/suservo.py +++ b/artiq/coredevice/suservo.py @@ -489,7 +489,7 @@ class Channel: def get_y_mu(self, profile): """Get a profile's IIR state (filter output, Y0) in machine units. - The IIR state is also know as the "integrator", or the DDS amplitude + The IIR state is also known as the "integrator", or the DDS amplitude scale factor. It is 17 bits wide and unsigned. This method does not advance the timeline but consumes all slack. @@ -507,7 +507,7 @@ class Channel: def get_y(self, profile): """Get a profile's IIR state (filter output, Y0). - The IIR state is also know as the "integrator", or the DDS amplitude + The IIR state is also known as the "integrator", or the DDS amplitude scale factor. It is 17 bits wide and unsigned. This method does not advance the timeline but consumes all slack. @@ -525,7 +525,7 @@ class Channel: def set_y_mu(self, profile, y): """Set a profile's IIR state (filter output, Y0) in machine units. - The IIR state is also know as the "integrator", or the DDS amplitude + The IIR state is also known as the "integrator", or the DDS amplitude scale factor. It is 17 bits wide and unsigned. This method must not be used when the servo could be writing to the @@ -545,7 +545,7 @@ class Channel: def set_y(self, profile, y): """Set a profile's IIR state (filter output, Y0). - The IIR state is also know as the "integrator", or the DDS amplitude + The IIR state is also known as the "integrator", or the DDS amplitude scale factor. It is 17 bits wide and unsigned. This method must not be used when the servo could be writing to the diff --git a/artiq/gui/applets.py b/artiq/gui/applets.py index 0cb25142d..99347084a 100644 --- a/artiq/gui/applets.py +++ b/artiq/gui/applets.py @@ -398,7 +398,7 @@ class _CompleterDelegate(QtWidgets.QStyledItemDelegate): # case, but causes unnecessary flickering and trashing of the user # selection when datasets are modified due to Qt's naive handler. # Doing this is of course convoluted due to Qt's arrogance - # about private fields and not letting users knows what + # about private fields and not letting users know what # slots are connected to signals, but thanks to the complicated # model system there is a short dirty hack in this particular case. nodatachanged_model = QtCore.QIdentityProxyModel() From 02235b2d80780de2ce0bb3cea09fe05be191fa5b Mon Sep 17 00:00:00 2001 From: architeuthidae <93191635+architeuthidae@users.noreply.github.com> Date: Fri, 18 Oct 2024 12:59:22 +0200 Subject: [PATCH 037/111] doc: Add section on management system communications (#2564) Co-authored-by: architeuthidae --- doc/manual/conf.py | 4 +++ doc/manual/management_system.rst | 61 ++++++++++++++++++++++---------- 2 files changed, 47 insertions(+), 18 deletions(-) diff --git a/doc/manual/conf.py b/doc/manual/conf.py index 4f27b77f0..348df5bb2 100644 --- a/doc/manual/conf.py +++ b/doc/manual/conf.py @@ -152,6 +152,10 @@ nitpick_ignore_regex = [ (r'py:.*', r'artiq.gateware.*'), ('py:mod', r'artiq.test.*'), ('py:mod', r'artiq.applets.*'), + # we can't use artiq.master.* because we shouldn't ignore the scheduler + ('py:class', r'artiq.master.experiments.*'), + ('py:class', r'artiq.master.databases.*'), + ('py:.*', r'artiq.master.worker.*'), ('py:class', 'dac34H84'), ('py:class', 'trf372017'), ('py:class', r'list(.*)'), diff --git a/doc/manual/management_system.rst b/doc/manual/management_system.rst index 849837be1..dd0d20aca 100644 --- a/doc/manual/management_system.rst +++ b/doc/manual/management_system.rst @@ -66,30 +66,30 @@ Experiment scheduling Basics ^^^^^^ -To make more efficient use of hardware resources, experiments are generally split into three phases and pipelined, such that potentially compute-intensive pre-computation or analysis phases may be executed in parallel with the bodies of other experiments, which access hardware. +To make more efficient use of resources, experiments are generally split into three phases and pipelined. While one experiment has control of the specialized hardware, others may carry out pre-computation or post-analysis in parallel. There are three stages of a standard experiment users may write code for: + +1. The **preparation** stage, which pre-fetches and pre-computes any data that is necessary to run the experiment. Users may implement this stage by overloading the :meth:`~artiq.language.environment.Experiment.prepare` method. It is not permitted to access hardware in this stage. + +2. The **run** stage, which corresponds to the body of the experiment. Users *must* implement this stage and overload the :meth:`~artiq.language.environment.Experiment.run` method. In this stage, the experiment has the right to run kernels and access hardware. + +3. The **analysis** stage, where raw results collected in the running stage can be post-processed and/or saved. This stage may be implemented by overloading the :meth:`~artiq.language.environment.Experiment.analyze` method. It is not permitted to access hardware in this stage. .. seealso:: - These steps are implemented in :class:`~artiq.language.environment.Experiment`. However, user-written experiments should usually derive from (sub-class) :class:`artiq.language.environment.EnvExperiment`, which additionally provides access to the methods of :class:`artiq.language.environment.HasEnvironment`. + These steps are implemented in :class:`artiq.language.environment.Experiment`. User-written experiments should usually derive from (sub-class) :class:`artiq.language.environment.EnvExperiment`, which additionally provides access to the methods of :class:`~artiq.language.environment.HasEnvironment`. -There are three stages of a standard experiment users may write code in: +Only the :meth:`~artiq.language.environment.Experiment.run` method implementation is mandatory; if the experiment does not fit into the pipelined scheduling model, it can leave one or both of the other methods empty (which is the default). Preparation and analysis stages are forbidden from accessing hardware so as not to interfere with a potential concurrent run stage. Note that they are not *prevented* from doing so, and it is up to the programmer to respect these guidelines. -1. The **preparation** stage, which pre-fetches and pre-computes any data that necessary to run the experiment. Users may implement this stage by overloading the :meth:`~artiq.language.environment.Experiment.prepare` method. It is not permitted to access hardware in this stage, as doing so may conflict with other experiments using the same devices. -2. The **run** stage, which corresponds to the body of the experiment and generally accesses hardware. Users must implement this stage and overload the :meth:`~artiq.language.environment.Experiment.run` method. -3. The **analysis** stage, where raw results collected in the running stage are post-processed and may lead to updates of the parameter database. This stage may be implemented by overloading the :meth:`~artiq.language.environment.Experiment.analyze` method. - -Only the :meth:`~artiq.language.environment.Experiment.run` method implementation is mandatory; if the experiment does not fit into the pipelined scheduling model, it can leave one or both of the other methods empty (which is the default). - -Consecutive experiments are then executed in a pipelined manner by the ARTIQ master's scheduler: first experiment A runs its preparation stage, than experiment A executes its running stage while experiment B executes its preparation stage, and so on. +Consecutive experiments are automatically pipelined by the ARTIQ master's scheduler: first experiment A executes its preparation stage, then experiment A executes its running stage while experiment B executes its preparation stage, and so on. .. note:: - The next experiment (B) may start its :meth:`~artiq.language.environment.Experiment.run` before all events placed into (core device) RTIO buffers by the previous experiment (A) have been executed. These events may then execute while experiment B's :meth:`~artiq.language.environment.Experiment.run` is already in progress. Using :meth:`~artiq.coredevice.core.Core.reset` in experiment B will clear the RTIO buffers, discarding pending events, including those left over from A. + An experiment A can exit its :meth:`~artiq.language.environment.Experiment.run` method before all its RTIO events have been executed, i.e., while those events are still 'waiting' in the RTIO core buffers. If the next experiment entering the running stage uses :meth:`~artiq.coredevice.core.Core.reset`, those buffers will be cleared, and any remaining events discarded, potentially including those scheduled by A. - Interactions between events of different experiments can be avoided by preventing the :meth:`~artiq.language.environment.Experiment.run` method of experiment A from returning until all events have been executed. This is discussed in the section on RTIO :ref:`rtio-handover-synchronization`. + This is a deliberate feature of seamless handover, but can cause problems if the events scheduled by A were important and should not have been skipped. In those cases, it is recommended to ensure the :meth:`~artiq.language.environment.Experiment.run` method of experiment A does not return until *all* its scheduled events have been executed, or that it is followed only by experiments which do not perform a core reset. See also :ref:`RTIO Synchronization`. Priorities and timed runs ^^^^^^^^^^^^^^^^^^^^^^^^^ -When determining what experiment should begin executing next (i.e. enter the preparation stage), the scheduling looks at the following factors, by decreasing order of precedence: +When determining what experiment should begin executing next (i.e. enter its preparation stage), the scheduling looks at the following factors, by decreasing order of precedence: 1. Experiments may be scheduled with a due date. This is considered the *earliest possible* time of their execution (rather than a deadline, or latest possible -- ARTIQ makes no guarantees about experiments being started or completed before any specified time). If a due date is set and it has not yet been reached, the experiment is not eligible for preparation. 2. The integer priority value specified by the user. @@ -106,11 +106,36 @@ When using multiple pipelines it is the responsibility of the user to ensure tha Pauses ^^^^^^ -In the run stage, an experiment may yield to the scheduler by calling the :meth:`pause` method of the scheduler. -If there are other experiments with higher priority (e.g. a high-priority experiment has been newly submitted, or reached its due date and become eligible for execution), the higher-priority experiments are executed first, and then :meth:`pause` returns. If there are no such experiments, :meth:`pause` returns immediately. To check whether :meth:`pause` would in fact *not* return immediately, use :meth:`artiq.master.scheduler.Scheduler.check_pause`. +In the run stage, an experiment may yield to the scheduler by calling the :meth:`pause` method of the scheduler. If there are other experiments with higher priority (e.g. a high-priority experiment has been newly submitted, or reached its due date and become eligible for execution), the higher-priority experiments are executed first, and then :meth:`pause` returns. If there are no such experiments, :meth:`pause` returns immediately. To check whether :meth:`pause` would in fact *not* return immediately, use :meth:`~artiq.master.scheduler.Scheduler.check_pause`. -The experiment must place the hardware in a safe state and disconnect from the core device (typically, by calling ``self.core.comm.close()`` from the kernel, which is equivalent to :meth:`artiq.coredevice.core.Core.close`) before calling :meth:`pause`. +The experiment must place the hardware in a safe state and disconnect from the core device before calling :meth:`pause` - typically by calling ``self.core.comm.close()``, which is equivalent to :meth:`~artiq.coredevice.core.Core.close`, from the host after completion of the kernel. -Accessing the :meth:`pause` and :meth:`~artiq.master.scheduler.Scheduler.check_pause` methods is done through a virtual device called ``scheduler`` that is accessible to all experiments. The scheduler virtual device is requested like regular devices using :meth:`~artiq.language.environment.HasEnvironment.get_device` (``self.get_device()``) or :meth:`~artiq.language.environment.HasEnvironment.setattr_device` (``self.setattr_device()``). +Accessing the :meth:`pause` and :meth:`~artiq.master.scheduler.Scheduler.check_pause` methods is done through a virtual device called ``scheduler`` that is accessible to all experiments. The scheduler virtual device is requested like any other device, with :meth:`~artiq.language.environment.HasEnvironment.get_device` or :meth:`~artiq.language.environment.HasEnvironment.setattr_device`. See also the detailed reference on the :doc:`mgmt_system_reference` page. -:meth:`~artiq.master.scheduler.Scheduler.check_pause` can be called (via RPC) from a kernel, but :meth:`pause` must not be. \ No newline at end of file +.. note:: + For maximum compatibility, the ``scheduler`` virtual device can also be accessed when running experiments with :mod:`~artiq.frontend.artiq_run`. However, since there is no :mod:`~artiq.master.scheduler.Scheduler` backend, the methods are replaced by simple dummies, e.g. :meth:`~artiq.master.scheduler.Scheduler.check_pause` simply returns false, and requests are printed into the console. Much the same is true of client control broadcasts (see again :doc:`mgmt_system_reference`). + +:meth:`~artiq.master.scheduler.Scheduler.check_pause` can be called (via RPC) from a kernel, but :meth:`pause` cannot be. + +Internal details +---------------- + +Internally, the ARTIQ management system uses Simple Python Communications, or `SiPyCo `_, which was originally written as part of ARTIQ and later split away as a generic communications library. The SiPyCo manual is hosted `here `_. The core of the management system is largely contained within ``artiq.master``, which contains the :class:`~artiq.master.scheduler.Scheduler`, the various environment and filesystem databases, and the worker processes that execute the experiments themselves. + +By default, the master communicates with other processes over four network ports, see :doc:`default_network_ports`, for logging, broadcasts, notifications, and control. All four of these can be customized by using the ``--port`` flags, see :ref:`the front-end reference`. + +- The logging port is occupied by a :class:`sipyco.logging_tools.Server`, and used only by the worker processes to transmit exceptions and other information to the master. +- The broadcast port is occupied by a :class:`sipyco.broadcast.Broadcaster`, which inherits from :class:`sipyco.pc_rpc.AsyncioServer`. Both the dashboard and the client automatically connect to this port, using :class:`sipyco.broadcast.Receiver` to receive logs and CCB messages. +- The notification port is occupied by a :class:`sipyco.sync_struct.Publisher`. The dashboard and client automatically connect to this port, using :class:`sipyco.sync_struct.Subscriber`. Several objects are given to the :class:`~sipyco.sync_struct.Publisher` to monitor, among them the experiment schedule, the device database, the dataset database, and the experiment list. It notifies the subscribers whenever these objects are modified. +- The control port is occupied by a :class:`sipyco.pc_rpc.Server`, which when running can be queried with :mod:`sipyco.sipyco_rpctool` like any other source of RPC targets. Multiple concurrent calls to target methods are supported. Through this server, the clients are provided with access to control methods to access the various databases and repositories the master handles, through classes like :class:`artiq.master.databases.DeviceDB`, :class:`artiq.master.databases.DatasetDB`, and :class:`artiq.master.experiments.ExperimentDB`. + +The experiment database is supported by :class:`artiq.master.experiments.GitBackend` when Git integration is active, and :class:`artiq.master.experiments.FilesystemBackend` if not. + +Experiment workers +^^^^^^^^^^^^^^^^^^ + +The :mod:`~artiq.frontend.artiq_run` tool makes use of many of the same databases and handlers as the master (whereas the scheduler and CCB manager are replaced by dummies, as mentioned above), but also directly runs the build, run, and analyze stages of the experiments. On the other hand, within the management system, the master's :class:`~artiq.master.scheduler.Scheduler` spawns a new worker process for each experiment. This allows for the parallelization of stages and pipelines described above in :ref:`experiment-scheduling`. + +The master and the worker processes communicate through IPC, Inter Process Communcation, implemented with :mod:`sipyco.pipe_ipc`. Specifically, it is :mod:`artiq.master.worker_impl` which is spawned as a new process for each experiment, and the class :class:`artiq.master.worker.Worker` which manages the IPC requests of the workers, including access to :class:`~artiq.master.scheduler.Scheduler` but also to devices, datasets, arguments, and CCBs. This allows the worker to support experiment :meth:`~artiq.language.environment.HasEnvironment.build` methods and the :doc:`management system interfaces `. + +The worker process also executes the experiment code itself. Within the experiment, kernel decorators -- :class:`~artiq.language.core.kernel`, :class:`~artiq.language.core.subkernel`, etc. -- call the ARTIQ compiler as necessary and trigger core device execution. \ No newline at end of file From 4bf2331d6a03c5e84702fe22e666c41a043f603e Mon Sep 17 00:00:00 2001 From: architeuthidae Date: Tue, 22 Oct 2024 19:57:25 +0200 Subject: [PATCH 038/111] doc: Elaboration on Git workflows --- doc/manual/getting_started_mgmt.rst | 11 +++--- doc/manual/management_system.rst | 61 +++++++++++++++++++++++------ 2 files changed, 55 insertions(+), 17 deletions(-) diff --git a/doc/manual/getting_started_mgmt.rst b/doc/manual/getting_started_mgmt.rst index 40a38b219..8ce5c7dca 100644 --- a/doc/manual/getting_started_mgmt.rst +++ b/doc/manual/getting_started_mgmt.rst @@ -171,7 +171,6 @@ Now, in the same dock as 'Explorer', navigate to the tab 'Interactive Args'. You In order to request and supply multiple interactive arguments at once, simply place them in the same ``with`` block; see also the example ``interactive.py`` in ``examples/no_hardware``. - .. _master-setting-up-git: Setting up Git integration @@ -180,7 +179,7 @@ Setting up Git integration So far, we have used the bare filesystem for the experiment repository, without any version control. Using Git to host the experiment repository helps with tracking modifications to experiments and with the traceability to a particular version of an experiment. .. note:: - The workflow we will describe in this tutorial corresponds to a situation where the computer running the ARTIQ master is also used as a Git server to which multiple users may contribute code. The Git setup can be customized according to your needs; the main point to remember is that when scanning or submitting, the ARTIQ master uses the internal Git data (*not* any working directory that may be present) to fetch the latest *fully completed commit* at the repository's head. See the :ref:`Management system ` page for notes on alternate workflows. + The workflow we will describe in this tutorial is designed for a situation where the computer running the ARTIQ master is also used as a Git server to which multiple users may contribute code. This is not the only way Git integration can be useful, and the setup can be customized according to your needs. The main point to remember is that when scanning or submitting, the ARTIQ master uses the internal Git data, *not* any checked-out files that may be present, to fetch the latest *fully completed commit* at the repository's head. See also :ref:`mgmt-git-integration`, especially if you are unfamiliar with Git. We will use our current ``repository`` folder as the working directory for making local modifications to the experiments, move it away from the master's data directory, and replace it with a new ``repository`` folder, which will hold only the Git data used by the master. Stop the master with Ctrl+C and enter the following commands: :: @@ -207,7 +206,7 @@ and finally, connect the two repositories and push the commit upstream to the ma $ git push -u origin master .. tip:: - If you are not familiar with command-line Git and would like to understand these commands in more detail, search for some tutorials in basic use of Git; there are many available online. + If you are not familiar with command-line Git and would like to understand these commands in more detail, look for tutorials on the basic use of Git; there are many available online. Start the master again with the ``-g`` flag, which tells it to treat its ``repository`` folder as a bare Git repository: :: @@ -233,10 +232,10 @@ Then set its execution permissions: :: Let's now make a modification to the experiments. In the working directory ``artiq-work``, open ``mgmt_tutorial.py`` again and add an exclamation mark to the end of "Hello World". Before committing it, check that the experiment can still be executed correctly by submitting it directly from the working directory, using the command-line client: :: - $ artiq_client submit ~/artiq-work/mgmt_tutorial.py + $ artiq_client submit --content ~/artiq-work/mgmt_tutorial.py .. note:: - Alternatively, right-click in the Explorer dock and select the 'Open file outside repository' option for the same effect. + The ``--content`` flag submits by content, that is, by sending a raw file rather than selecting an experiment from the master's local environment. Since you are likely running the client and the master on the same machine, it is probably not strictly necessary here. In a distributed setup across multiple machines, the master will not have access to the client's filesystem, and the ``--content`` flag is the only way to run experiments directly from a client file. See also :ref:`submission-details`. Verify the log in the GUI. If you are happy with the result, commit the new version and push it into the master's repository: :: @@ -257,4 +256,4 @@ Arguments to the individual tools (including ``-s`` and ``--bind``) can still be $ artiq_session -m=-g -to start the session with the master in Git mode. See also :mod:`~artiq.frontend.artiq_session`. \ No newline at end of file +to start the session with Git integration. See also :mod:`~artiq.frontend.artiq_session`. \ No newline at end of file diff --git a/doc/manual/management_system.rst b/doc/manual/management_system.rst index dd0d20aca..69a575cbf 100644 --- a/doc/manual/management_system.rst +++ b/doc/manual/management_system.rst @@ -7,19 +7,23 @@ Management system Components ---------- +See also :doc:`overview` for a visual idea of the management system. + Master ^^^^^^ The :ref:`ARTIQ master ` is responsible for managing the dataset and device databases, the experiment repository, scheduling and running experiments, archiving results, and distributing real-time results. It is a headless component, and one or several clients (command-line or GUI) use the network to interact with it. -The master expects to be given a directory on startup, the experiment repository, containing these experiments which are automatically tracked and communicated to clients. By default, it simply looks for a directory called ``repository``. The ``-r`` flag can be used to substitute an alternate location. +The master expects to be given a directory on startup, the experiment repository, containing these experiments which are automatically tracked and communicated to clients. By default, it simply looks for a directory called ``repository``. The ``-r`` flag can be used to substitute an alternate location. Subdirectories in ``repository`` are also read, and experiments stored in them are known to the master. They will be displayed as folders in the dashboard's explorer. -It also expects access to a ``device_db.py``, with a corresponding flag ``--device-db`` to substitute a different file name. Additionally, it will reference or create certain files in the directory it is run in, among them ``dataset_db.mdb``, the LMDB database containing persistent datasets, ``last_rid.pyon``, which simply contains the last used RID, and the ``results`` directory. +It also expects access to a ``device_db.py``, with a corresponding flag ``--device-db`` to substitute a different file name. Additionally, it will reference or create certain files in the directory it is run in, among them ``dataset_db.mdb``, the LMDB database containing persistent datasets, ``last_rid.pyon``, which simply holds the last used RID, and the ``results`` directory. For more on the device and dataset databases, see also :doc:`environment`. .. note:: - Because the other parts of the management system all seem to be able to access the information stored in these files, confusion can sometimes result about where it is really stored and how it is distributed. Device databases, datasets, results, and experiments are all solely kept and administered by the master, which communicates information to dashboards, browsers, and clients over the network whenever necessary. + Because the other parts of the management system often display knowledge of the information stored in these files, confusion can sometimes result about where it is really stored and how it is distributed. Device databases, datasets, results, and experiments are all solely kept and administered by the master, which communicates with dashboards, clients, and controller managers over the network whenever necessary. - Notably, clients and dashboards do not send in experiments to the master; they request them from the array of experiments the master knows about, primarily those in ``repository``, but also in the master's local file system, if 'Open file outside repository' is selected. This is true even if ``repository`` is configured as a Git repository and cloned on other machines. + Notably, clients and dashboards normally do not *send in* experiments to the master. Rather, they make requests from the list of experiments the master already knows about, primarily those in ``repository``, but also in the master's local file system. This is true even if ``repository`` is configured as a Git repository and cloned onto other machines. + + The only exception is the command line client's ``--content`` flag, which allows submission by content, i.e. sending in experiment files which may be otherwise unknown to the master. This feature however has some important limitations; see below in :ref:`submission-details`. The ARTIQ master should not be confused with the 'master' device in a DRTIO system, which is only a designation for the particular core device acting as central node in a distributed configuration of ARTIQ. The two concepts are otherwise unrelated. @@ -32,31 +36,66 @@ The :ref:`dashboard ` connects to the master and is th The dashboard remembers and restores GUI state (window/dock positions, last values entered by the user, etc.) in between instances. This information is stored in a file called ``artiq_dashboard_{server}_{port}.pyon`` in the configuration directory (e.g. generally ``~/.config/artiq`` for Unix, same as data directory for Windows), distinguished in subfolders by ARTIQ version. +.. note:: + + To find where the configuration files are stored on your machine, try the command: :: + + python -c "from artiq.tools import get_user_config_dir; print(get_user_config_dir())" + Browser ^^^^^^^ -The :ref:`browser ` is used to read ARTIQ ``results`` HDF5 files and run experiment :meth:`~artiq.language.environment.Experiment.analyze` functions, in particular to retrieve previous result databases, process them, and display them in ARTIQ applets. The browser also remembers and restores its GUI state; this is stored in a file called simply ``artiq_browser``, kept in the same configuration directory as the dashboard. +The :ref:`browser ` is used to read ARTIQ ``results`` HDF5 files and run experiment :meth:`~artiq.language.environment.Experiment.analyze` functions, in particular to retrieve previous result databases, process them, and display them in ARTIQ applets. The browser also remembers and restores its GUI state; this is stored in a file called simply ``artiq_browser.pyon``, kept in the same configuration directory as the dashboard. + +The browser *can* connect to the master, specifically in order to be able to access the master's store of datasets and to upload new datasets to it, but it does not require such a connection and can also be run completely standalone. However, it requires filesystem access to the ``results`` files to be of much use. Controller manager ^^^^^^^^^^^^^^^^^^ The controller manager is provided in the ``artiq-comtools`` package (which is also made available separately from mainline ARTIQ, to allow independent use with minimal dependencies) and started with the :mod:`~artiq_comtools.artiq_ctlmgr` command. It is responsible for running and stopping controllers on a machine. One controller manager must be run by each network node that runs controllers. -A controller manager connects to the master and accesses the device database through it to determine what controllers need to be run. The local network address of the connection is used to filter for only those controllers allocated to the current node. Hostname resolution is supported. Changes to the device database are tracked and controllers will be stopped and started accordingly. +A controller manager connects to the master and accesses the device database through it to determine what controllers need to be run. The local network address of the connection is used to filter for only those controllers allocated to the current node. Hostname resolution is supported. Changes to the device database are tracked upon rescan and controllers will be stopped and started accordingly. .. _mgmt-git-integration: Git integration --------------- -The master may use a Git repository to store experiment source code. Using Git has many advantages. For example, each result file (HDF5) contains the commit ID corresponding to the exact source code it was produced by, which helps reproducibility. Although the master also supports non-bare repositories, it is recommended to use a bare repository (e.g. ``git init --bare``) to easily support push transactions from clients. +The master may use a Git repository to store experiment source code. Using Git rather than the bare filesystem has many advantages. For example, each HDF5 result file contains the commit ID corresponding to the exact source code it was produced by, making results more reproduceable. See also :ref:`master-setting-up-git`. Generally, it is recommended to use a bare repository (i.e. ``git init --bare``), to easily support push transactions from clients, but both bare and non-bare repositories are supported. -You will want Git to notify the master every time the repository is pushed to (e.g. updated), so that the master knows to rescan the repository for new or changed experiments. This is easiest done with the ``post-receive`` hook, as described in :ref:`master-setting-up-git`. +.. tip:: + If you are not familiar with Git, you may find the idea of the master reading experiment files from a bare repository confusing. A bare repository does not normally contain copies of the objects it stores; that is to say, you won't be able to find your experiment files listed in it. What it *does* contain is Git's internal data structures, i.e., ``hooks``, ``objects``, ``config``, and so forth. Among other things, this structure also stores, in compressed form, the full contents of every commit made to the repository. It is this compressed data which the master has access to and can read the experiments from. It is not meant to be directly edited, but it is updated every time new commits are received. -.. note:: - If you plan to run the ARTIQ system entirely on a single machine, you may also consider using a non-bare repository and the ``post-commit`` hook to trigger repository scans every time you commit changes (locally). In this case, note that the ARTIQ master never uses the repository's working directory, but only what is committed. More precisely, when scanning the repository, it fetches the last (atomically) completed commit at that time of repository scan and checks it out in a temporary folder. This commit ID is used by default when subsequently submitting experiments. There is one temporary folder by commit ID currently referenced in the system, so concurrently running experiments from different repository revisions is fully supported by the master. + It may be useful to note that a normal Git repository, created with ``git init``, contains all the same internal data, kept in a hidden directory called ``.git`` to protect it from accidental modifications. Unlike a bare repository, it *also* normally contains working copies of all the files tracked by Git. When working with a non-bare repository, it is important to understand that the master still takes its image of the available experiments from the internal data, and *not* from the working copies. This is why, even in a non-bare repository, changes are only reflected once they are committed. The working copies are simply ignored. -By default, the dashboard runs experiments from the repository, whereas the command-line client (``artiq_client submit``) runs experiments from the raw filesystem (which is useful for iterating rapidly without creating many disorganized commits). In order to run from the raw filesystem when using the dashboard, right-click in the Explorer window and select the option 'Open file outside repository'. In order to run from the repository when using the command-line client, simply pass the ``-R`` flag. + Other important files -- the device database, the dataset database, the ``results`` directory, and so on -- are normally kept outside of the experiment repository, and in this case, they are not stored or handled by Git at all. The master accesses them through the regular filesystem, not through Git, and other ARTIQ components access them through the master. This can be seen visualized in the :doc:`overview`. + +With a bare repository, a Git ``post-receive`` hook can be used to trigger a repository scan every time the repository is pushed to (i.e. updated), as described in the tutorial. This removes the need to trigger repository rescans manually. If you plan to run your ARTIQ system from a single PC, without distributed clients, you may also consider using a non-bare repository and the ``post-commit`` hook instead. In this workflow, changes can be drafted directly in the master's repository, but the master continues to submit from the last completed commit until a new commit is made (and the repository is rescanned). + +Behind the scenes, when scanning the repository, the master fetches the last (atomically) completed commit at that time of repository scan and checks it out in a temporary folder. This commit ID is used by default when subsequently submitting experiments. There is one temporary folder by commit ID currently referenced in the system, so concurrently running experiments from different repository revisions is fully supported by the master. + +The use of the Git backend is triggered when the master is started with the ``-g`` flag. Otherwise the raw filesystem is read and Git-based features will not be available. + +.. _submission-details: + +Submission from the raw filesystem +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +By default, the dashboard runs experiments from the repository, that is, the master's temporary checkout folder, whereas the command-line client (``artiq_client submit``) runs experiments from the raw filesystem. This is convenient in order to be able to run working drafts without first committing them. + +Be careful with this behavior, however, as it is rather particular. *The raw filesystem* means the immediate local filesystem of the running master. If the client is being run remotely, and you want to submit an experiment from the *client's* local filesystem, e.g. an uncommitted draft in a clone of the experiment repository, use the ``--content`` flag. If you would like to submit an experiment from the repository, in the same way the dashboard does, use the flag ``--repository`` / ``-R``. + +To be precise: + +- ``artiq_client submit`` should be given a file path that is relative to the location of the master, that is, if the master is run in the directory above its ``repository``, an experiment can be submitted as ``repository/experiment_file.py``. Keep in mind that when working with a bare repository, there may be no copies of experiment files in the raw local filesystem. In this case, files can still be made accessible to the master by network filesystem share or some other method for testing. + +- ``artiq_client submit --repository`` should be given a file path relative to the root of the repository, that is, if the experiment is directly within ``repository``, it should be submitted as ``experiment_file.py``. Just as in the dashboard, this file is taken from the last completed commit. + +- ``artiq_client submit --content`` should be given a file path that is relative to the location of the client, whether that is local or remote to the master; the contents of the file will be submitted directly to the master to be run. This essentially transfers a raw string, and will not work if the experiment imports or otherwise accesses other files. + +Other flags can also be used, such as ``--class-name`` / ``-c`` to select a class name in an experiment which contains several, or ``--revision`` / ``-r`` to use a particular revision. See the reference of :mod:`~artiq.frontend.artiq_client` in :doc:`main_frontend_tools`. + +In order to run from the raw filesystem when using the dashboard, right-click in the Explorer window and select the option 'Open file outside repository'. This will open a file explorer window displaying the master's local filesystem, which can be used to select and submit experiments outside of the chosen repository directory. There is no GUI support for submission by content. It is recommended to simply use the command-line client for this purpose. .. _experiment-scheduling: From 56bd975a347155181cf1fdd5fc383742d22daf57 Mon Sep 17 00:00:00 2001 From: architeuthidae Date: Thu, 24 Oct 2024 22:22:14 +0200 Subject: [PATCH 039/111] doc: Typo --- doc/manual/core_device.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/manual/core_device.rst b/doc/manual/core_device.rst index 4177e6414..5a405ac46 100644 --- a/doc/manual/core_device.rst +++ b/doc/manual/core_device.rst @@ -140,7 +140,7 @@ Kasli and Kasli-SoC `Kasli `_ and `Kasli-SoC `_ are versatile core devices designed for ARTIQ as part of the open-source `Sinara `_ family of boards. All support interfacing to various EEM daughterboards (TTL, DDS, ADC, DAC...) through twelve onboard EEM ports. Kasli is based on a Xilinx Artix-7 FPGA, and Kasli-SoC, which runs on a separate `Zynq port `_ of the ARTIQ firmware, is based on a Zynq-7000 SoC, notably including an ARM CPU allowing for much heavier software computations at high speeds. They are architecturally very different but supply similar feature sets. Kasli itself exists in two versions, of which the improved Kasli v2.0 is now in more common use, but the original v1.0 remains supported by ARTIQ. -Kasli can be connected to the network using a 10000Base-X SFP module, installed into the SFP0 cage. Kasli-SoC features a built-in Ethernet port to use instead. If configured as a DRTIO satellite, both boards instead reserve SFP0 for the upstream DRTIO connection; remaining SFP cages are available for downstream connections. Equally, if used as a DRTIO master, all free SFP cages are available for downstream connections (i.e. all but SFP0 on Kasli, all four on Kasli-SoC). +Kasli can be connected to the network using a 1000Base-X SFP module, installed into the SFP0 cage. Kasli-SoC features a built-in Ethernet port to use instead. If configured as a DRTIO satellite, both boards instead reserve SFP0 for the upstream DRTIO connection; remaining SFP cages are available for downstream connections. Equally, if used as a DRTIO master, all free SFP cages are available for downstream connections (i.e. all but SFP0 on Kasli, all four on Kasli-SoC). The DRTIO line rate depends upon the RTIO clock frequency running, e.g., at 125MHz the line rate is 2.5Gbps, at 150MHz 3.0Gbps, etc. See below for information on RTIO clocks. From 65d20b7857c4f5af37ebc2386184f064a7a7bba4 Mon Sep 17 00:00:00 2001 From: Sebastien Bourdeauducq Date: Wed, 30 Oct 2024 13:31:16 +0800 Subject: [PATCH 040/111] flake: update dependencies --- flake.lock | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/flake.lock b/flake.lock index cfa15f0db..aec76b50c 100644 --- a/flake.lock +++ b/flake.lock @@ -44,11 +44,11 @@ }, "nixpkgs": { "locked": { - "lastModified": 1728492678, - "narHash": "sha256-9UTxR8eukdg+XZeHgxW5hQA9fIKHsKCdOIUycTryeVw=", + "lastModified": 1729880355, + "narHash": "sha256-RP+OQ6koQQLX5nw0NmcDrzvGL8HDLnyXt/jHhL1jwjM=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "5633bcff0c6162b9e4b5f1264264611e950c8ec7", + "rev": "18536bf04cd71abd345f9579158841376fdd0c5a", "type": "github" }, "original": { @@ -129,11 +129,11 @@ "src-misoc": { "flake": false, "locked": { - "lastModified": 1728978817, - "narHash": "sha256-b4633jrhh4i+KunZq4kNlyhdm9BCsEJwKs+6KINKV2o=", + "lastModified": 1729234629, + "narHash": "sha256-TLsTCXV5AC2xh+bS7EhBVBKqdqIU3eKrnlWcFF9LtAM=", "ref": "refs/heads/master", - "rev": "386b544776b66cea148da67d06a4b3a4151179f9", - "revCount": 2459, + "rev": "6085a312bca26adeca6584e37d08c8ba2e1d6e38", + "revCount": 2460, "submodules": true, "type": "git", "url": "https://github.com/m-labs/misoc.git" From b4085312b0dbdc156eed135d063ec6d074c8689e Mon Sep 17 00:00:00 2001 From: Florian Agbuya Date: Thu, 31 Oct 2024 10:03:12 +0800 Subject: [PATCH 041/111] flake: add paramiko ed25519 dependencies --- flake.nix | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/flake.nix b/flake.nix index 8d05492c9..7d7a947b7 100644 --- a/flake.nix +++ b/flake.nix @@ -470,7 +470,11 @@ #__impure = true; # Nix 2.8+ buildInputs = [ - (pkgs.python3.withPackages(ps: with packages.x86_64-linux; [ artiq ps.paramiko ])) + (pkgs.python3.withPackages(ps: with packages.x86_64-linux; [ + artiq + ps.paramiko + ] ++ ps.paramiko.optional-dependencies.ed25519 + )) pkgs.llvm_15 pkgs.lld_15 pkgs.openssh From e336e33a46604185a00bfa339298ceb961767e52 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Robert=20J=C3=B6rdens?= Date: Sun, 3 Nov 2024 15:12:40 +0100 Subject: [PATCH 042/111] artiq.svg: clean up * Remove several broken open paths that do not add visible geometry (AI/Inkscape residue) * Ungroup a spurious group * Convert two remaining objects from polygon to paths such that the file only contains paths. Increases compatibility with other tools (CAD, PCB layout, conversion tools) --- doc/logo/artiq.svg | 37 ++++++++++--------------------------- 1 file changed, 10 insertions(+), 27 deletions(-) diff --git a/doc/logo/artiq.svg b/doc/logo/artiq.svg index 2cab7ec30..375c47ab8 100644 --- a/doc/logo/artiq.svg +++ b/doc/logo/artiq.svg @@ -61,18 +61,6 @@ id="path319" d="m 69.975,234.695 c -3.226,0.31 -6.272,0.602 -9.352,0.754 -1.868,0.093 -3.594,0.139 -5.278,0.139 -7.127,0 -13.339,-0.867 -18.903,-2.646 -0.794,-0.254 -1.576,-0.526 -2.345,-0.817 -11.329,-4.29 -16.078,-12.875 -13.733,-24.827 2.135,-10.872 7.632,-19.988 13.253,-28.221 1.117,-1.634 2.315,-3.259 3.474,-4.83 0.454,-0.616 0.909,-1.233 1.364,-1.857 L 28.098,162.386 c -7.526,9.307 -16.644,21.933 -20.824,37.338 -3.192,11.767 -2.23,22.453 2.783,30.906 5.009,8.446 13.909,14.409 25.738,17.245 6.106,1.465 12.57,2.177 19.76,2.177 3.754,-0.001 7.687,-0.192 12.023,-0.588 2.495,-0.227 4.928,-0.557 7.504,-0.906 0.973,-0.132 1.95,-0.265 2.934,-0.392 l -3.897,-13.857 c -1.413,0.124 -2.791,0.256 -4.144,0.386 z" /> \ No newline at end of file + d="m 279.656,208.102 c -0.147,-0.262 -0.314,-0.56 -0.358,-0.905 -0.992,-8.005 -3.834,-16.142 -8.689,-24.875 -7.945,-14.297 -18.829,-27.683 -34.25,-42.126 -3.812,-3.572 -7.724,-6.949 -11.864,-10.523 -1.677,-1.448 -3.376,-2.915 -5.096,-4.419 -0.006,0.032 -0.011,0.062 -0.017,0.092 -0.062,0.355 -0.097,0.551 -0.09,0.713 l 0.149,3.794 c 0.176,4.559 0.358,9.272 0.669,13.896 0.046,0.706 0.615,1.672 1.521,2.583 2.133,2.144 4.345,4.286 6.484,6.358 3.806,3.687 7.742,7.5 11.388,11.467 11.612,12.634 19.076,24.245 23.489,36.543 2.048,5.705 2.706,10.802 2.011,15.581 -1.146,7.896 -6.144,13.235 -15.281,16.322 -2.455,0.829 -5.003,1.474 -7.658,1.956 l 9.738,12.6 c 1.551,-0.468 3.08,-0.975 4.576,-1.562 12.387,-4.858 19.753,-12.956 22.521,-24.758 l 0.87,-3.686 0,-8.847 c -0.036,-0.067 -0.075,-0.135 -0.113,-0.204 z" /> From 83254d13dafcc18653d94d0b3c9f45c3c553b2fc Mon Sep 17 00:00:00 2001 From: architeuthidae Date: Mon, 4 Nov 2024 23:34:58 +0100 Subject: [PATCH 043/111] doc: Mock platformdirs --- doc/manual/conf.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/manual/conf.py b/doc/manual/conf.py index 348df5bb2..70ae703dc 100644 --- a/doc/manual/conf.py +++ b/doc/manual/conf.py @@ -35,7 +35,7 @@ mock_modules = ["artiq.gui.waitingspinnerwidget", "artiq.dashboard.waveform", "artiq.coredevice.jsondesc", "qasync", "lmdb", "dateutil.parser", "prettytable", "PyQt6", - "h5py", "llvmlite", "pythonparser", "tqdm", "jsonschema"] + "h5py", "llvmlite", "pythonparser", "tqdm", "jsonschema", "platformdirs"] for module in mock_modules: sys.modules[module] = Mock() From bca30becbfefb5e98b2047dda53360a44c5bfdaf Mon Sep 17 00:00:00 2001 From: Charlie Root Date: Thu, 7 Nov 2024 18:20:07 +0100 Subject: [PATCH 044/111] flake.nix: clean up inputs - Put all inputs into a single attrSet to improve readabiliy - Fix deprecated links since links without quotes surrounding them have been deprecated for quite some time. --- flake.nix | 44 +++++++++++++++++++++++++++++++++----------- 1 file changed, 33 insertions(+), 11 deletions(-) diff --git a/flake.nix b/flake.nix index 7d7a947b7..a7e461e66 100644 --- a/flake.nix +++ b/flake.nix @@ -1,20 +1,42 @@ { description = "A leading-edge control system for quantum information experiments"; - inputs.nixpkgs.url = github:NixOS/nixpkgs/nixos-unstable; - inputs.rust-overlay = { + inputs = { + nixpkgs.url = "github:NixOS/nixpkgs/nixos-unstable"; + + rust-overlay = { url = "github:oxalica/rust-overlay?ref=snapshot/2024-08-01"; inputs.nixpkgs.follows = "nixpkgs"; - }; - inputs.sipyco.url = github:m-labs/sipyco; - inputs.sipyco.inputs.nixpkgs.follows = "nixpkgs"; - inputs.src-pythonparser = { url = github:m-labs/pythonparser; flake = false; }; - inputs.artiq-comtools.url = github:m-labs/artiq-comtools; - inputs.artiq-comtools.inputs.nixpkgs.follows = "nixpkgs"; - inputs.artiq-comtools.inputs.sipyco.follows = "sipyco"; + }; - inputs.src-migen = { url = github:m-labs/migen; flake = false; }; - inputs.src-misoc = { type = "git"; url = "https://github.com/m-labs/misoc.git"; submodules = true; flake = false; }; + artiq-comtools = { + url = "github:m-labs/artiq-comtools"; + inputs.nixpkgs.follows = "nixpkgs"; + inputs.sipyco.follows = "sipyco"; + }; + + sipyco = { + url = "github:m-labs/sipyco"; + inputs.nixpkgs.follows = "nixpkgs"; + }; + + src-migen = { + url = "github:m-labs/migen"; + flake = false; + }; + + src-misoc = { + url = "https://github.com/m-labs/misoc.git"; + type = "git"; + submodules = true; + flake = false; + }; + + src-pythonparser = { + url = "github:m-labs/pythonparser"; + flake = false; + }; + }; outputs = { self, nixpkgs, rust-overlay, sipyco, src-pythonparser, artiq-comtools, src-migen, src-misoc }: let From 82505a22030bf650f47469e8ff75125baa46c9aa Mon Sep 17 00:00:00 2001 From: Mike Birtwell Date: Wed, 13 Nov 2024 18:43:52 +0000 Subject: [PATCH 045/111] firmware: Disable the Nagle algorithm on sockets Signed-off-by: Michael Birtwell --- artiq/firmware/runtime/sched.rs | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/artiq/firmware/runtime/sched.rs b/artiq/firmware/runtime/sched.rs index 0d751da8e..f0a2fdddd 100644 --- a/artiq/firmware/runtime/sched.rs +++ b/artiq/firmware/runtime/sched.rs @@ -402,16 +402,18 @@ impl<'a> TcpListener<'a> { socket.may_send() || socket.may_recv() })?; - let accepted = self.handle.get(); + let accepted = TcpStream { + io: self.io, + handle: self.handle.get(), + }; + accepted.with_lower(|s| s.set_nagle_enabled(false)); + self.handle.set(Self::new_lower(self.io, self.buffer_size.get())); match self.listen(self.endpoint.get()) { Ok(()) => (), _ => unreachable!() } - Ok(TcpStream { - io: self.io, - handle: accepted - }) + Ok(accepted) } pub fn close(&self) { From 9aae89be69fa3ab1dd0a75f399ffc6cf208818d2 Mon Sep 17 00:00:00 2001 From: Sebastien Bourdeauducq Date: Fri, 15 Nov 2024 10:12:48 +0800 Subject: [PATCH 046/111] flake: update dependencies --- flake.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/flake.lock b/flake.lock index aec76b50c..2cd1b97ab 100644 --- a/flake.lock +++ b/flake.lock @@ -44,11 +44,11 @@ }, "nixpkgs": { "locked": { - "lastModified": 1729880355, - "narHash": "sha256-RP+OQ6koQQLX5nw0NmcDrzvGL8HDLnyXt/jHhL1jwjM=", + "lastModified": 1731319897, + "narHash": "sha256-PbABj4tnbWFMfBp6OcUK5iGy1QY+/Z96ZcLpooIbuEI=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "18536bf04cd71abd345f9579158841376fdd0c5a", + "rev": "dc460ec76cbff0e66e269457d7b728432263166c", "type": "github" }, "original": { From 27d54cb8f3a394b4f4adcbb3c2c9160c5bf3df47 Mon Sep 17 00:00:00 2001 From: architeuthidae <93191635+architeuthidae@users.noreply.github.com> Date: Sat, 16 Nov 2024 06:19:04 +0100 Subject: [PATCH 047/111] doc: Add FAQ on rollbacks/releases (#2609) --- doc/manual/faq.rst | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/doc/manual/faq.rst b/doc/manual/faq.rst index e5ef530fa..5dd925678 100644 --- a/doc/manual/faq.rst +++ b/doc/manual/faq.rst @@ -27,6 +27,21 @@ Substitute ``artiq-manual-pdf`` to get the LaTeX PDF version. The results will b The manual is written in `reStructured Text `_; you can find the source files in the ARTIQ repository under ``doc/manual``. If you spot a mistake, a typo, or something that's out of date or missing -- in particular, if you want to add something to this FAQ -- feel free to clone the repository, edit the source RST files, and make a pull request with your version of an improvement. (If you're not a fan of or not familiar with command-line Git, both GitHub and Gitea support making edits and pull requests directly in the web interface; tutorial materials are easy to find online.) The second best thing is to open an issue to make M-Labs aware of the problem. +roll back to older versions of ARTIQ, or obtain it through other installation methods? +-------------------------------------------------------------------------------------- + +At all times, three versions of ARTIQ are actively supported by M-Labs, released through the beta, stable, and legacy channels. See :doc:`releases`. + +If you are trying to rollback to stable or legacy, the process should be accordingly simple. See the respective :doc:`installing` page in the respective version of the manual. If you've previously used the version you are rolling back to, you can likely use the rollback methods described in :ref:`installing-upgrading`; otherwise you can always treat it as a fresh install. Remember that it will also be necessary to reflash core devices with corresponding legacy binaries. + +Regarding pre-legacy releases, note that being actively supported simply means that M-Labs makes prebuilt packages and binaries for these versions available via the supported installation methods and through AFWS. Outdated versions aren't automatically built or offered over these channels, but their source code remains available in the Git repository, and you are free to use it or adapt it in accordance with the terms of the license, including building whatever packages you prefer. In general, though, newer releases of ARTIQ offer more features, more stability, better performance, and better support. The legacy release is supported simply as a convenience for users who haven't been able to upgrade yet. For normal purposes, it is recommended to use the current stable release of ARTIQ if at all possible, or the beta to gain access to new features and improvements that are still in development. + +For more details, see also `Clarifications regarding the ARTIQ release model and AFWS `_. + +.. tip:: + + If you're particularly concerned with being able to precisely reproduce older experiments, even when you've moved on to newer ARTIQ versions, upgrade carefully and make your own local backups to be able to rollback to older versions of your system. Make sure to keep copies of older firmware binaries in order to be able to reflash your hardware. Older versions of ARTIQ will always continue working if left untouched, and you won't need to worry about rebuilding from the source if you keep your own prebuilt versions around. + .. _faq-networking: troubleshoot networking problems? @@ -293,4 +308,4 @@ For more advanced questions, sometimes the `list of publications `_ yourself, following one of the provided templates. - In some odd cases, you may want to see the `mailing list archive `_; the ARTIQ mailing list was shut down at the end of 2020 and was last regularly used during the time of ARTIQ-2 and 3, but for some older ARTIQ features, or to understand a development thought process, you may still find relevant information there. -In any situation, if you found the manual unclear or unhelpful, you might consider following the :ref:`directions for contribution ` and editing it to be more helpful for future readers. \ No newline at end of file +In any situation, if you found the manual unclear or unhelpful, you might consider following the :ref:`directions for contribution ` and editing it to be more helpful for future readers. From 40ab4fee5bc6a90f327867b728f78c00b5637cf1 Mon Sep 17 00:00:00 2001 From: occheung Date: Wed, 23 Oct 2024 13:22:32 +0800 Subject: [PATCH 048/111] drtio_eem: move rx_ready out of init --- artiq/firmware/libboard_artiq/drtio_eem.rs | 5 +---- artiq/firmware/satman/main.rs | 7 ++++++- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/artiq/firmware/libboard_artiq/drtio_eem.rs b/artiq/firmware/libboard_artiq/drtio_eem.rs index f1cd6afc8..08190ded3 100644 --- a/artiq/firmware/libboard_artiq/drtio_eem.rs +++ b/artiq/firmware/libboard_artiq/drtio_eem.rs @@ -211,9 +211,6 @@ pub fn init() { } }); - unsafe { - align_comma(); - csr::eem_transceiver::rx_ready_write(1); - } + unsafe { align_comma(); } } } diff --git a/artiq/firmware/satman/main.rs b/artiq/firmware/satman/main.rs index 11da3e1a1..8e265999e 100644 --- a/artiq/firmware/satman/main.rs +++ b/artiq/firmware/satman/main.rs @@ -777,7 +777,12 @@ pub extern fn main() -> i32 { }); #[cfg(has_drtio_eem)] - drtio_eem::init(); + { + drtio_eem::init(); + unsafe { + csr::eem_transceiver::rx_ready_write(1) + } + } #[cfg(has_drtio_routing)] let mut repeaters = [repeater::Repeater::default(); csr::DRTIOREP.len()]; From bd7b07b0fd284a627c69b8bb7cdd2f58c7d4f14f Mon Sep 17 00:00:00 2001 From: occheung Date: Wed, 23 Oct 2024 13:47:48 +0800 Subject: [PATCH 049/111] master: implement DRTIO-EEM satellite disconnection --- artiq/firmware/libboard_artiq/drtio_eem.rs | 18 +++++++++++ artiq/firmware/runtime/rtio_mgt.rs | 35 ++++++++++++++++++++++ 2 files changed, 53 insertions(+) diff --git a/artiq/firmware/libboard_artiq/drtio_eem.rs b/artiq/firmware/libboard_artiq/drtio_eem.rs index 08190ded3..9da755b83 100644 --- a/artiq/firmware/libboard_artiq/drtio_eem.rs +++ b/artiq/firmware/libboard_artiq/drtio_eem.rs @@ -187,6 +187,24 @@ unsafe fn align_comma() { } } +pub unsafe fn align_wordslip(trx_no: u8) -> bool { + csr::eem_transceiver::transceiver_sel_write(trx_no); + + for slip in 0..=1 { + csr::eem_transceiver::wordslip_write(slip as u8); + clock::spin_us(1); + csr::eem_transceiver::comma_align_reset_write(1); + clock::spin_us(100); + + if csr::eem_transceiver::comma_read() == 1 { + debug!("comma alignment completed with {} wordslip", slip); + return true; + } + } + + false +} + pub fn init() { for trx_no in 0..csr::CONFIG_EEM_DRTIO_COUNT { unsafe { diff --git a/artiq/firmware/runtime/rtio_mgt.rs b/artiq/firmware/runtime/rtio_mgt.rs index d065214a7..120a4a1c6 100644 --- a/artiq/firmware/runtime/rtio_mgt.rs +++ b/artiq/firmware/runtime/rtio_mgt.rs @@ -16,6 +16,8 @@ const ASYNC_ERROR_SEQUENCE_ERROR: u8 = 1 << 2; pub mod drtio { use super::*; use alloc::vec::Vec; + #[cfg(has_drtio_eem)] + use board_artiq::drtio_eem; use drtioaux; use proto_artiq::drtioaux_proto::{MASTER_PAYLOAD_MAX_SIZE, PayloadStatus}; use rtio_dma::remote_dma; @@ -24,6 +26,9 @@ pub mod drtio { use kernel::subkernel; use sched::Error as SchedError; + #[cfg(has_drtio_eem)] + const DRTIO_EEM_LINKNOS: core::ops::Range = (csr::DRTIO.len()-csr::CONFIG_EEM_DRTIO_COUNT as usize)..csr::DRTIO.len(); + #[derive(Fail, Debug)] pub enum Error { #[fail(display = "timed out")] @@ -73,6 +78,18 @@ pub mod drtio { fn link_rx_up(linkno: u8) -> bool { let linkno = linkno as usize; + #[cfg(has_drtio_eem)] + if DRTIO_EEM_LINKNOS.contains(&linkno) { + let eem_trx_no = linkno - DRTIO_EEM_LINKNOS.start; + unsafe { + csr::eem_transceiver::transceiver_sel_write(eem_trx_no as u8); + csr::eem_transceiver::comma_align_reset_write(1); + } + clock::spin_us(100); + return unsafe { + csr::eem_transceiver::comma_read() == 1 + }; + } unsafe { (csr::DRTIO[linkno].rx_up_read)() == 1 } @@ -414,9 +431,27 @@ pub mod drtio { } else { info!("[LINK#{}] link is down", linkno); up_links[linkno as usize] = false; + + #[cfg(has_drtio_eem)] + if DRTIO_EEM_LINKNOS.contains(&(linkno as usize)) { + unsafe { csr::eem_transceiver::rx_ready_write(0); } + // Clear DRTIOAUX buffer + while !matches!(drtioaux::recv(linkno), Ok(None)) {} + } } } else { /* link was previously down */ + #[cfg(has_drtio_eem)] + if DRTIO_EEM_LINKNOS.contains(&(linkno as usize)) { + let eem_trx_no = linkno - DRTIO_EEM_LINKNOS.start as u8; + if !unsafe { drtio_eem::align_wordslip(eem_trx_no) } { + continue; + } + unsafe { + csr::eem_transceiver::rx_ready_write(1); + } + } + if link_rx_up(linkno) { info!("[LINK#{}] link RX became up, pinging", linkno); let ping_count = ping_remote(&io, aux_mutex, linkno); From 292a07d830c83a399ca1ba3dbf016e29d88d7e40 Mon Sep 17 00:00:00 2001 From: newell Date: Thu, 14 Nov 2024 14:00:52 -0800 Subject: [PATCH 050/111] doc: EBAZ4205 SD Boot --- doc/manual/core_device.rst | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/doc/manual/core_device.rst b/doc/manual/core_device.rst index 5a405ac46..762aa438b 100644 --- a/doc/manual/core_device.rst +++ b/doc/manual/core_device.rst @@ -166,6 +166,14 @@ EBAZ4205 The `EBAZ4205 `_ Zynq-SoC control card, originally used in the Ebit E9+ BTC miner, is a low-cost development board (around $20-$30 USD), making it an ideal option for experimenting with ARTIQ. To use the EBAZ4205, it's important to carefully follow the board documentation to configure it to boot from the SD card, as network booting via ``artiq_netboot`` is currently unsupported. This is because the Ethernet PHY is routed through the EMIO, requiring the FPGA to be programmed before the board can establish a network connection. +SD BOOT +""""""" + +To enable the EBAZ4205 to boot from an SD card, you will need to modify the board's boot select resistors. By default, the board is set to boot from NAND, with a resistor placed on ``R2584``. To change the boot mode to SD card, move the resistor from ``R2584`` to ``R2577``. Be sure to carefully consult the `EBAZ4205 documentation `_ to confirm resistor locations and proper handling of the board. + +AD9834 DDS +"""""""""" + One useful application of the EBAZ4205 is controlling external devices like the AD9834 DDS Module from ZonRi Technology Co., Ltd. To establish communication between the EBAZ4205 and the AD9834 module, proper configuration of the SPI interface pins is essential. The board's flexibility allows for straightforward control of the DDS once the correct pinout is known. The table below details the necessary connections between the EBAZ4205 and the AD9834 module, including power, ground, and SPI signals. +--------------------------+---------------------+----------------------------+ @@ -188,7 +196,7 @@ One useful application of the EBAZ4205 is controlling external devices like the | RESET (Unused) | RESET | N/A: Bit Controlled | +--------------------------+---------------------+----------------------------+ -For a step-by-step guide, see the `EBAZ4205 and AD9834 setup guide `_. +For a guide, see the `EBAZ4205 and AD9834 setup guide `_. Variant details --------------- From 52c07a2b145b436053f8072eca7300759b690801 Mon Sep 17 00:00:00 2001 From: Florian Agbuya Date: Tue, 8 Oct 2024 12:23:23 +0800 Subject: [PATCH 051/111] gui/experiments: add custom colors for experiment windows Signed-off-by: Florian Agbuya --- artiq/dashboard/experiments.py | 93 +++++++++++++++++++++++++++++++++- 1 file changed, 91 insertions(+), 2 deletions(-) diff --git a/artiq/dashboard/experiments.py b/artiq/dashboard/experiments.py index 4be556d79..00a44f620 100644 --- a/artiq/dashboard/experiments.py +++ b/artiq/dashboard/experiments.py @@ -28,6 +28,7 @@ class _ArgumentEditor(EntryTreeWidget): def __init__(self, manager, dock, expurl): self.manager = manager self.expurl = expurl + self.dock = dock EntryTreeWidget.__init__(self) @@ -78,6 +79,18 @@ class _ArgumentEditor(EntryTreeWidget): argument["desc"] = procdesc argument["state"] = state self.update_argument(name, argument) + self.dock.apply_colors() + + def apply_color(self, palette, color): + self.setPalette(palette) + for child in self.findChildren(QtWidgets.QWidget): + child.setPalette(palette) + child.setAutoFillBackground(True) + items = self.findItems("*", + QtCore.Qt.MatchFlag.MatchWildcard | QtCore.Qt.MatchFlag.MatchRecursive) + for item in items: + for column in range(item.columnCount()): + item.setBackground(column, QtGui.QColor(color)) # Hooks that allow user-supplied argument editors to react to imminent user # actions. Here, we always keep the manager-stored submission arguments @@ -92,6 +105,19 @@ class _ArgumentEditor(EntryTreeWidget): log_levels = ["DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL"] +class _ColoredTitleBar(QtWidgets.QProxyStyle): + def __init__(self, color, style=None): + super().__init__(style) + self.color = color + + def drawComplexControl(self, control, option, painter, widget=None): + if control == QtWidgets.QStyle.ComplexControl.CC_TitleBar: + option = QtWidgets.QStyleOptionTitleBar(option) + option.palette.setColor(QtGui.QPalette.ColorRole.Window, QtGui.QColor(self.color)) + option.palette.setColor(QtGui.QPalette.ColorRole.Highlight, QtGui.QColor(self.color)) + self.baseStyle().drawComplexControl(control, option, painter, widget) + + class _ExperimentDock(QtWidgets.QMdiSubWindow): sigClosed = QtCore.pyqtSignal() @@ -303,14 +329,61 @@ class _ExperimentDock(QtWidgets.QMdiSubWindow): self.argeditor = editor_class(self.manager, self, self.expurl) self.layout.addWidget(self.argeditor, 0, 0, 1, 5) self.argeditor.restore_state(argeditor_state) + self.apply_colors() def contextMenuEvent(self, event): menu = QtWidgets.QMenu(self) + select_title_bar_color = menu.addAction("Select title bar color") + select_window_color = menu.addAction("Select window color") + reset_colors = menu.addAction("Reset to default colors") + menu.addSeparator() reset_sched = menu.addAction("Reset scheduler settings") action = menu.exec(self.mapToGlobal(event.pos())) - if action == reset_sched: + if action == select_title_bar_color: + self.select_color("title_bar") + elif action == select_window_color: + self.select_color("window") + elif action == reset_colors: + self.reset_colors() + elif action == reset_sched: asyncio.ensure_future(self._recompute_sched_options_task()) + def select_color(self, key): + color = QtWidgets.QColorDialog.getColor( + title=f"Select {key.replace('_', ' ').title()} color") + if color.isValid(): + self.manager.set_color(self.expurl, key, color.name()) + self.apply_colors() + + def apply_colors(self): + colors = self.manager.get_colors(self.expurl) + if colors is None: + palette = QtWidgets.QApplication.palette() + colors = { + "window": palette.color(QtGui.QPalette.ColorRole.Window).name(), + "title_bar": palette.color(QtGui.QPalette.ColorRole.Highlight).name(), + } + self.manager.colors[self.expurl] = colors + colors["window_text"] = "#000000" if QtGui.QColor( + colors["window"]).lightness() > 128 else "#FFFFFF" + self.modify_palette(colors) + self.setStyle(_ColoredTitleBar(colors["title_bar"])) + self.argeditor.apply_color(self.palette(), (colors["window"])) + + def modify_palette(self, colors): + palette = self.palette() + palette.setColor(QtGui.QPalette.ColorRole.Window, QtGui.QColor(colors["window"])) + palette.setColor(QtGui.QPalette.ColorRole.Base, QtGui.QColor(colors["window"])) + palette.setColor(QtGui.QPalette.ColorRole.Button, QtGui.QColor(colors["window"])) + palette.setColor(QtGui.QPalette.ColorRole.Text, QtGui.QColor(colors["window_text"])) + palette.setColor(QtGui.QPalette.ColorRole.ButtonText, QtGui.QColor(colors["window_text"])) + palette.setColor(QtGui.QPalette.ColorRole.WindowText, QtGui.QColor(colors["window_text"])) + self.setPalette(palette) + + def reset_colors(self): + self.manager.reset_colors(self.expurl) + self.apply_colors() + async def _recompute_sched_options_task(self): try: expdesc, _ = await self.manager.compute_expdesc(self.expurl) @@ -457,6 +530,7 @@ class ExperimentManager: self.submission_options = dict() self.submission_arguments = dict() self.argument_ui_names = dict() + self.colors = dict() self.datasets = dict() dataset_sub.add_setmodel_callback(self.set_dataset_model) @@ -483,6 +557,18 @@ class ExperimentManager: def set_schedule_model(self, model): self.schedule = model.backing_store + def set_color(self, expurl, key, value): + if expurl not in self.colors: + self.colors[expurl] = {} + self.colors[expurl][key] = value + + def get_colors(self, expurl): + return self.colors.get(expurl) + + def reset_colors(self, expurl): + if expurl in self.colors: + del self.colors[expurl] + def resolve_expurl(self, expurl): if expurl[:5] == "repo:": expinfo = self.explist[expurl[5:]] @@ -592,6 +678,7 @@ class ExperimentManager: self.open_experiments[expurl] = dock dock.setAttribute(QtCore.Qt.WidgetAttribute.WA_DeleteOnClose) self.main_window.centralWidget().addSubWindow(dock) + dock.apply_colors() dock.show() dock.sigClosed.connect(partial(self.on_dock_closed, expurl)) if expurl in self.dock_states: @@ -708,7 +795,8 @@ class ExperimentManager: "arguments": self.submission_arguments, "docks": self.dock_states, "argument_uis": self.argument_ui_names, - "open_docks": set(self.open_experiments.keys()) + "open_docks": set(self.open_experiments.keys()), + "colors": self.colors } def restore_state(self, state): @@ -719,6 +807,7 @@ class ExperimentManager: self.submission_options = state["options"] self.submission_arguments = state["arguments"] self.argument_ui_names = state.get("argument_uis", {}) + self.colors = state.get("colors", {}) for expurl in state["open_docks"]: self.open_experiment(expurl) From 673fe29a12c097f59b9cc6abc056bb3791747489 Mon Sep 17 00:00:00 2001 From: Sebastien Bourdeauducq Date: Sat, 16 Nov 2024 17:43:56 +0800 Subject: [PATCH 052/111] RELEASE_NOTES: update --- RELEASE_NOTES.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/RELEASE_NOTES.rst b/RELEASE_NOTES.rst index eb97387e8..7d1c160c8 100644 --- a/RELEASE_NOTES.rst +++ b/RELEASE_NOTES.rst @@ -7,6 +7,7 @@ ARTIQ-9 (Unreleased) -------------------- * Dashboard: + - Experiment windows can have different colors, selected by the user. - Zotino monitoring now displays the values in volts. - Schedule display columns can now be reordered and shown/hidden using the table header context menu. From 1836ab5196a6524e61ed6f5215aceec6bb9dc840 Mon Sep 17 00:00:00 2001 From: newell Date: Sat, 16 Nov 2024 23:33:38 -0800 Subject: [PATCH 053/111] doc: Add note that 150 MHz internal reference is not supported for EBAZ4205 (#2617) --- doc/manual/core_device.rst | 3 +++ 1 file changed, 3 insertions(+) diff --git a/doc/manual/core_device.rst b/doc/manual/core_device.rst index 762aa438b..e72f80153 100644 --- a/doc/manual/core_device.rst +++ b/doc/manual/core_device.rst @@ -166,6 +166,9 @@ EBAZ4205 The `EBAZ4205 `_ Zynq-SoC control card, originally used in the Ebit E9+ BTC miner, is a low-cost development board (around $20-$30 USD), making it an ideal option for experimenting with ARTIQ. To use the EBAZ4205, it's important to carefully follow the board documentation to configure it to boot from the SD card, as network booting via ``artiq_netboot`` is currently unsupported. This is because the Ethernet PHY is routed through the EMIO, requiring the FPGA to be programmed before the board can establish a network connection. +.. note:: + Although both ``int_100`` and ``int_125`` are supported, ``int_150`` -- used to synthesize a 150MHz RTIO clock -- is not currently compatible with the EBAZ4205. + SD BOOT """"""" From 12682a277e9be7e1c9c897d4317f4f0c9ff8fa00 Mon Sep 17 00:00:00 2001 From: occheung Date: Mon, 26 Aug 2024 17:37:32 +0800 Subject: [PATCH 054/111] config: impl flashing over mgmt --- artiq/firmware/libboard_misoc/config.rs | 37 +++++++++++++++++++++---- 1 file changed, 32 insertions(+), 5 deletions(-) diff --git a/artiq/firmware/libboard_misoc/config.rs b/artiq/firmware/libboard_misoc/config.rs index e5e62ad0c..a7c3d4a63 100644 --- a/artiq/firmware/libboard_misoc/config.rs +++ b/artiq/firmware/libboard_misoc/config.rs @@ -259,12 +259,39 @@ mod imp { } pub fn write(key: &str, value: &[u8]) -> Result<(), Error> { - match append(key, value) { - Err(Error::SpaceExhausted) => { - compact()?; - append(key, value) + fn flash_binary(origin: usize, payload: &[u8]) { + let mut offset = 0; + while offset < payload.len() { + unsafe { + spiflash::erase_sector(origin + offset); + } + offset += spiflash::SECTOR_SIZE; + } + unsafe { + spiflash::write(origin, payload); + } + } + + match key { + "gateware" => { + flash_binary(0, value); + Ok(()) + } + "bootloader" => { + flash_binary(::mem::ROM_BASE, value); + Ok(()) + } + "firmware" => { + flash_binary(::mem::FLASH_BOOT_ADDRESS, value); + Ok(()) + } + _ => match append(key, value) { + Err(Error::SpaceExhausted) => { + compact()?; + append(key, value) + } + res => res } - res => res } } From b60a616e78ec19654fba6778eca3726924e3920b Mon Sep 17 00:00:00 2001 From: occheung Date: Mon, 26 Aug 2024 17:39:02 +0800 Subject: [PATCH 055/111] drtio: add new messages for remote mgmt --- .../firmware/libproto_artiq/drtioaux_proto.rs | 185 ++++++++++++++++++ 1 file changed, 185 insertions(+) diff --git a/artiq/firmware/libproto_artiq/drtioaux_proto.rs b/artiq/firmware/libproto_artiq/drtioaux_proto.rs index a1b17e888..ba826873a 100644 --- a/artiq/firmware/libproto_artiq/drtioaux_proto.rs +++ b/artiq/firmware/libproto_artiq/drtioaux_proto.rs @@ -127,6 +127,22 @@ pub enum Packet { SubkernelException { destination: u8, last: bool, length: u16, data: [u8; MASTER_PAYLOAD_MAX_SIZE] }, SubkernelMessage { source: u8, destination: u8, id: u32, status: PayloadStatus, length: u16, data: [u8; MASTER_PAYLOAD_MAX_SIZE] }, SubkernelMessageAck { destination: u8 }, + + CoreMgmtGetLogRequest { destination: u8, clear: bool }, + CoreMgmtClearLogRequest { destination: u8 }, + CoreMgmtSetLogLevelRequest { destination: u8, log_level: u8 }, + CoreMgmtSetUartLogLevelRequest { destination: u8, log_level: u8 }, + CoreMgmtConfigReadRequest { destination: u8, length: u16, key: [u8; MASTER_PAYLOAD_MAX_SIZE] }, + CoreMgmtConfigReadContinue { destination: u8 }, + CoreMgmtConfigWriteRequest { destination: u8, last: bool, length: u16, data: [u8; MASTER_PAYLOAD_MAX_SIZE] }, + CoreMgmtConfigRemoveRequest { destination: u8, length: u16, key: [u8; MASTER_PAYLOAD_MAX_SIZE] }, + CoreMgmtConfigEraseRequest { destination: u8 }, + CoreMgmtRebootRequest { destination: u8 }, + CoreMgmtAllocatorDebugRequest { destination: u8 }, + CoreMgmtGetLogReply { last: bool, length: u16, data: [u8; SAT_PAYLOAD_MAX_SIZE] }, + CoreMgmtConfigReadReply { last: bool, length: u16, value: [u8; SAT_PAYLOAD_MAX_SIZE] }, + CoreMgmtAck, + CoreMgmtNack, } impl Packet { @@ -405,6 +421,93 @@ impl Packet { destination: reader.read_u8()? }, + 0xd0 => Packet::CoreMgmtGetLogRequest { + destination: reader.read_u8()?, + clear: reader.read_bool()?, + }, + 0xd1 => Packet::CoreMgmtClearLogRequest { + destination: reader.read_u8()?, + }, + 0xd2 => Packet::CoreMgmtSetLogLevelRequest { + destination: reader.read_u8()?, + log_level: reader.read_u8()?, + }, + 0xd3 => Packet::CoreMgmtSetUartLogLevelRequest { + destination: reader.read_u8()?, + log_level: reader.read_u8()?, + }, + 0xd4 => { + let destination = reader.read_u8()?; + let length = reader.read_u16()?; + let mut key: [u8; MASTER_PAYLOAD_MAX_SIZE] = [0; MASTER_PAYLOAD_MAX_SIZE]; + reader.read_exact(&mut key[0..length as usize])?; + Packet::CoreMgmtConfigReadRequest { + destination: destination, + length: length, + key: key, + } + }, + 0xd5 => Packet::CoreMgmtConfigReadContinue { + destination: reader.read_u8()?, + }, + 0xd6 => { + let destination = reader.read_u8()?; + let last = reader.read_bool()?; + let length = reader.read_u16()?; + let mut data: [u8; MASTER_PAYLOAD_MAX_SIZE] = [0; MASTER_PAYLOAD_MAX_SIZE]; + reader.read_exact(&mut data[0..length as usize])?; + Packet::CoreMgmtConfigWriteRequest { + destination: destination, + last: last, + length: length, + data: data, + } + }, + 0xd7 => { + let destination = reader.read_u8()?; + let length = reader.read_u16()?; + let mut key: [u8; MASTER_PAYLOAD_MAX_SIZE] = [0; MASTER_PAYLOAD_MAX_SIZE]; + reader.read_exact(&mut key[0..length as usize])?; + Packet::CoreMgmtConfigRemoveRequest { + destination: destination, + length: length, + key: key, + } + }, + 0xd8 => Packet::CoreMgmtConfigEraseRequest { + destination: reader.read_u8()?, + }, + 0xd9 => Packet::CoreMgmtRebootRequest { + destination: reader.read_u8()?, + }, + 0xda => Packet::CoreMgmtAllocatorDebugRequest { + destination: reader.read_u8()?, + }, + 0xdb => { + let last = reader.read_bool()?; + let length = reader.read_u16()?; + let mut data: [u8; SAT_PAYLOAD_MAX_SIZE] = [0; SAT_PAYLOAD_MAX_SIZE]; + reader.read_exact(&mut data[0..length as usize])?; + Packet::CoreMgmtGetLogReply { + last: last, + length: length, + data: data, + } + }, + 0xdc => { + let last = reader.read_bool()?; + let length = reader.read_u16()?; + let mut value: [u8; SAT_PAYLOAD_MAX_SIZE] = [0; SAT_PAYLOAD_MAX_SIZE]; + reader.read_exact(&mut value[0..length as usize])?; + Packet::CoreMgmtConfigReadReply { + last: last, + length: length, + value: value, + } + }, + 0xdd => Packet::CoreMgmtAck, + 0xde => Packet::CoreMgmtNack, + ty => return Err(Error::UnknownPacket(ty)) }) } @@ -693,6 +796,88 @@ impl Packet { writer.write_u8(0xcc)?; writer.write_u8(destination)?; }, + + Packet::CoreMgmtGetLogRequest { destination, clear } => { + writer.write_u8(0xd0)?; + writer.write_u8(destination)?; + writer.write_bool(clear)?; + }, + Packet::CoreMgmtClearLogRequest { destination } => { + writer.write_u8(0xd1)?; + writer.write_u8(destination)?; + }, + Packet::CoreMgmtSetLogLevelRequest { destination, log_level } => { + writer.write_u8(0xd2)?; + writer.write_u8(destination)?; + writer.write_u8(log_level)?; + }, + Packet::CoreMgmtSetUartLogLevelRequest { destination, log_level } => { + writer.write_u8(0xd3)?; + writer.write_u8(destination)?; + writer.write_u8(log_level)?; + }, + Packet::CoreMgmtConfigReadRequest { + destination, + length, + key, + } => { + writer.write_u8(0xd4)?; + writer.write_u8(destination)?; + writer.write_u16(length)?; + writer.write_all(&key[0..length as usize])?; + }, + Packet::CoreMgmtConfigReadContinue { destination } => { + writer.write_u8(0xd5)?; + writer.write_u8(destination)?; + }, + Packet::CoreMgmtConfigWriteRequest { + destination, + last, + length, + data, + } => { + writer.write_u8(0xd6)?; + writer.write_u8(destination)?; + writer.write_bool(last)?; + writer.write_u16(length)?; + writer.write_all(&data[0..length as usize])?; + }, + Packet::CoreMgmtConfigRemoveRequest { + destination, + length, + key, + } => { + writer.write_u8(0xd7)?; + writer.write_u8(destination)?; + writer.write_u16(length)?; + writer.write_all(&key[0..length as usize])?; + }, + Packet::CoreMgmtConfigEraseRequest { destination } => { + writer.write_u8(0xd8)?; + writer.write_u8(destination)?; + }, + Packet::CoreMgmtRebootRequest { destination } => { + writer.write_u8(0xd9)?; + writer.write_u8(destination)?; + }, + Packet::CoreMgmtAllocatorDebugRequest { destination } => { + writer.write_u8(0xda)?; + writer.write_u8(destination)?; + }, + Packet::CoreMgmtGetLogReply { last, length, data } => { + writer.write_u8(0xdb)?; + writer.write_bool(last)?; + writer.write_u16(length)?; + writer.write_all(&data[0..length as usize])?; + }, + Packet::CoreMgmtConfigReadReply { last, length, value } => { + writer.write_u8(0xdc)?; + writer.write_bool(last)?; + writer.write_u16(length)?; + writer.write_all(&value[0..length as usize])?; + }, + Packet::CoreMgmtAck => writer.write_u8(0xdd)?, + Packet::CoreMgmtNack => writer.write_u8(0xde)?, } Ok(()) } From e57d18a41f8b8c12bd77855715c2ed7679d2d2d2 Mon Sep 17 00:00:00 2001 From: occheung Date: Mon, 26 Aug 2024 17:40:16 +0800 Subject: [PATCH 056/111] runtime: support mgmt over drtio --- artiq/firmware/libproto_artiq/mgmt_proto.rs | 4 +- artiq/firmware/runtime/main.rs | 6 +- artiq/firmware/runtime/mgmt.rs | 657 ++++++++++++++++---- artiq/firmware/runtime/rtio_mgt.rs | 2 +- 4 files changed, 547 insertions(+), 122 deletions(-) diff --git a/artiq/firmware/libproto_artiq/mgmt_proto.rs b/artiq/firmware/libproto_artiq/mgmt_proto.rs index 911799ef4..74724df13 100644 --- a/artiq/firmware/libproto_artiq/mgmt_proto.rs +++ b/artiq/firmware/libproto_artiq/mgmt_proto.rs @@ -16,7 +16,9 @@ pub enum Error { #[fail(display = "invalid UTF-8: {}", _0)] Utf8(Utf8Error), #[fail(display = "{}", _0)] - Io(#[cause] IoError) + Io(#[cause] IoError), + #[fail(display = "drtio error")] + DrtioError, } impl From> for Error { diff --git a/artiq/firmware/runtime/main.rs b/artiq/firmware/runtime/main.rs index dd52aabaf..78bf9b594 100644 --- a/artiq/firmware/runtime/main.rs +++ b/artiq/firmware/runtime/main.rs @@ -209,7 +209,11 @@ fn startup() { rtio_mgt::startup(&io, &aux_mutex, &drtio_routing_table, &up_destinations, &ddma_mutex, &subkernel_mutex); { let restart_idle = restart_idle.clone(); - io.spawn(4096, move |io| { mgmt::thread(io, &restart_idle) }); + let aux_mutex = aux_mutex.clone(); + let ddma_mutex = ddma_mutex.clone(); + let subkernel_mutex = subkernel_mutex.clone(); + let drtio_routing_table = drtio_routing_table.clone(); + io.spawn(4096, move |io| { mgmt::thread(io, &restart_idle, &aux_mutex, &ddma_mutex, &subkernel_mutex, &drtio_routing_table) }); } { let aux_mutex = aux_mutex.clone(); diff --git a/artiq/firmware/runtime/mgmt.rs b/artiq/firmware/runtime/mgmt.rs index 5b6fc6d01..0962fcc97 100644 --- a/artiq/firmware/runtime/mgmt.rs +++ b/artiq/firmware/runtime/mgmt.rs @@ -1,11 +1,12 @@ use log::{self, LevelFilter}; use core::cell::Cell; +use core::cell::RefCell; -use io::{Write, ProtoWrite, Error as IoError}; -use board_misoc::{config, spiflash}; -use logger_artiq::BufferLogger; +use board_artiq::drtio_routing; +use io::{ProtoRead, Write, Error as IoError}; use mgmt_proto::*; use sched::{Io, TcpListener, TcpStream, Error as SchedError}; +use sched::{Io, Mutex, TcpListener, TcpStream, Error as SchedError}; use urc::Urc; impl From for Error { @@ -14,138 +15,556 @@ impl From for Error { } } -fn worker(io: &Io, stream: &mut TcpStream, restart_idle: &Urc>) -> Result<(), Error> { +mod local_coremgmt { + use alloc::{string::String, vec::Vec}; + use log::LevelFilter; + + use board_misoc::{config, spiflash}; + use io::{Write, ProtoWrite, Error as IoError}; + use logger_artiq::BufferLogger; + use mgmt_proto::{Error, Reply}; + use sched::{Io, TcpStream, Error as SchedError}; + + + pub fn get_log(io: &Io, stream: &mut TcpStream) -> Result<(), Error> { + BufferLogger::with(|logger| { + let mut buffer = io.until_ok(|| logger.buffer())?; + Reply::LogContent(buffer.extract()).write_to(stream) + })?; + Ok(()) + } + + pub fn clear_log(io: &Io, stream: &mut TcpStream) -> Result<(), Error> { + BufferLogger::with(|logger| -> Result<(), IoError> { + let mut buffer = io.until_ok(|| logger.buffer())?; + Ok(buffer.clear()) + })?; + + Reply::Success.write_to(stream)?; + Ok(()) + } + + pub fn pull_log(io: &Io, stream: &mut TcpStream) -> Result<(), Error> { + BufferLogger::with(|logger| -> Result<(), IoError> { + loop { + // Do this *before* acquiring the buffer, since that sets the log level + // to OFF. + let log_level = log::max_level(); + + let mut buffer = io.until_ok(|| logger.buffer())?; + if buffer.is_empty() { continue } + + stream.write_string(buffer.extract())?; + + if log_level == LevelFilter::Trace { + // Hold exclusive access over the logger until we get positive + // acknowledgement; otherwise we get an infinite loop of network + // trace messages being transmitted and causing more network + // trace messages to be emitted. + // + // Any messages unrelated to this management socket that arrive + // while it is flushed are lost, but such is life. + stream.flush()?; + } + + // Clear the log *after* flushing the network buffers, or we're just + // going to resend all the trace messages on the next iteration. + buffer.clear(); + } + })?; + Ok(()) + } + + pub fn set_log_filter(_io: &Io, stream: &mut TcpStream, level: LevelFilter) -> Result<(), Error> { + info!("changing log level to {}", level); + log::set_max_level(level); + Reply::Success.write_to(stream)?; + Ok(()) + } + + pub fn set_uart_log_filter(_io: &Io, stream: &mut TcpStream, level: LevelFilter) -> Result<(), Error> { + info!("changing UART log level to {}", level); + BufferLogger::with(|logger| + logger.set_uart_log_level(level)); + Reply::Success.write_to(stream)?; + Ok(()) + } + + pub fn config_read(_io: &Io, stream: &mut TcpStream, key: &String) -> Result<(), Error>{ + config::read(key, |result| { + match result { + Ok(value) => Reply::ConfigData(&value).write_to(stream), + Err(_) => Reply::Error.write_to(stream) + } + })?; + Ok(()) + } + + pub fn config_write(_io: &Io, stream: &mut TcpStream, key: &String, value: &Vec, restart_idle: &Urc>) -> Result<(), Error> { + match config::write(key, value) { + Ok(_) => { + if key == "idle_kernel" { + io.until(|| !restart_idle.get())?; + restart_idle.set(true); + } + Reply::Success.write_to(stream) + }, + Err(_) => Reply::Error.write_to(stream) + }?; + Ok(()) + } + + pub fn config_remove(_io: &Io, stream: &mut TcpStream, key: &String, restart_idle: &Urc>) -> Result<(), Error> { + match config::remove(key) { + Ok(()) => { + if key == "idle_kernel" { + io.until(|| !restart_idle.get())?; + restart_idle.set(true); + } + Reply::Success.write_to(stream) + }, + Err(_) => Reply::Error.write_to(stream) + }?; + Ok(()) + } + + pub fn config_erase(_io: &Io, stream: &mut TcpStream, restart_idle: &Urc>) -> Result<(), Error> { + match config::erase() { + Ok(()) => { + io.until(|| !restart_idle.get())?; + restart_idle.set(true); + Reply::Success.write_to(stream) + }, + Err(_) => Reply::Error.write_to(stream) + }?; + Ok(()) + } + + pub fn reboot(_io: &Io, stream: &mut TcpStream) -> Result<(), Error> { + Reply::RebootImminent.write_to(stream)?; + stream.close()?; + stream.flush()?; + + warn!("restarting"); + unsafe { spiflash::reload(); } + } + + pub fn debug_allocator(_io: &Io, _stream: &mut TcpStream) -> Result<(), Error> { + unsafe { println!("{}", ::ALLOC) } + Ok(()) + } +} + +#[cfg(has_drtio)] +mod remote_coremgmt { + use alloc::{string::String, vec::Vec}; + use log::LevelFilter; + + use board_artiq::{drtioaux::Packet, drtio_routing}; + use io::{Cursor, ProtoWrite}; + use mgmt_proto::{Error, Reply}; + use rtio_mgt::drtio; + use sched::{Io, Mutex, TcpStream, Error as SchedError}; + use proto_artiq::drtioaux_proto::MASTER_PAYLOAD_MAX_SIZE; + + + impl From for Error { + fn from(_value: drtio::Error) -> Error { + Error::DrtioError + } + } + + pub fn get_log(io: &Io, aux_mutex: &Mutex, + ddma_mutex: &Mutex, subkernel_mutex: &Mutex, + routing_table: &drtio_routing::RoutingTable, linkno: u8, + destination: u8, stream: &mut TcpStream) -> Result<(), Error> { + let mut buffer = String::new(); + loop { + let reply = drtio::aux_transact(io, aux_mutex, ddma_mutex, subkernel_mutex, routing_table, linkno, + &Packet::CoreMgmtGetLogRequest { destination, clear: false } + ); + + match reply { + Ok(Packet::CoreMgmtGetLogReply { last, length, data }) => { + buffer.push_str( + core::str::from_utf8(&data[..length as usize]).map_err(|_| Error::DrtioError)?); + if last { + Reply::LogContent(&buffer).write_to(stream)?; + return Ok(()); + } + } + Ok(packet) => { + error!("received unexpected aux packet: {:?}", packet); + Reply::Error.write_to(stream)?; + return Err(drtio::Error::UnexpectedReply.into()); + } + Err(e) => { + error!("aux packet error ({})", e); + Reply::Error.write_to(stream)?; + return Err(e.into()); + } + } + } + } + + pub fn clear_log(io: &Io, aux_mutex: &Mutex, + ddma_mutex: &Mutex, subkernel_mutex: &Mutex, + routing_table: &drtio_routing::RoutingTable, linkno: u8, + destination: u8, stream: &mut TcpStream) -> Result<(), Error> { + let reply = drtio::aux_transact(io, aux_mutex, ddma_mutex, subkernel_mutex, routing_table, linkno, + &Packet::CoreMgmtClearLogRequest { destination } + ); + + match reply { + Ok(Packet::CoreMgmtAck) => { + Reply::Success.write_to(stream)?; + Ok(()) + } + Ok(packet) => { + error!("received unexpected aux packet: {:?}", packet); + Reply::Error.write_to(stream)?; + Err(drtio::Error::UnexpectedReply.into()) + } + Err(e) => { + error!("aux packet error ({})", e); + Reply::Error.write_to(stream)?; + Err(e.into()) + } + } + } + + pub fn pull_log(io: &Io, aux_mutex: &Mutex, + ddma_mutex: &Mutex, subkernel_mutex: &Mutex, + routing_table: &drtio_routing::RoutingTable, linkno: u8, + destination: u8, stream: &mut TcpStream) -> Result<(), Error> { + loop { + let reply = drtio::aux_transact(io, aux_mutex, ddma_mutex, subkernel_mutex, routing_table, linkno, + &Packet::CoreMgmtGetLogRequest { destination, clear: true } + ); + + match reply { + Ok(Packet::CoreMgmtGetLogReply { last: _, length, data }) => { + stream.write_bytes(&data[..length as usize])?; + } + Ok(packet) => { + error!("received unexpected aux packet: {:?}", packet); + return Err(drtio::Error::UnexpectedReply.into()); + } + Err(e) => { + error!("aux packet error ({})", e); + return Err(e.into()); + } + } + } + } + + pub fn set_log_filter(io: &Io, aux_mutex: &Mutex, + ddma_mutex: &Mutex, subkernel_mutex: &Mutex, + routing_table: &drtio_routing::RoutingTable, linkno: u8, + destination: u8, stream: &mut TcpStream, level: LevelFilter) -> Result<(), Error> { + let reply = drtio::aux_transact(io, aux_mutex, ddma_mutex, subkernel_mutex, routing_table, linkno, + &Packet::CoreMgmtSetLogLevelRequest { destination, log_level: level as u8 } + ); + + match reply { + Ok(Packet::CoreMgmtAck) => { + Reply::Success.write_to(stream)?; + Ok(()) + } + Ok(packet) => { + error!("received unexpected aux packet: {:?}", packet); + Reply::Error.write_to(stream)?; + Err(drtio::Error::UnexpectedReply.into()) + } + Err(e) => { + error!("aux packet error ({})", e); + Reply::Error.write_to(stream)?; + Err(e.into()) + } + } + } + + pub fn set_uart_log_filter(io: &Io, aux_mutex: &Mutex, + ddma_mutex: &Mutex, subkernel_mutex: &Mutex, + routing_table: &drtio_routing::RoutingTable, linkno: u8, + destination: u8, stream: &mut TcpStream, level: LevelFilter) -> Result<(), Error> { + let reply = drtio::aux_transact(io, aux_mutex, ddma_mutex, subkernel_mutex, routing_table, linkno, + &Packet::CoreMgmtSetUartLogLevelRequest { destination, log_level: level as u8 } + ); + + match reply { + Ok(Packet::CoreMgmtAck) => { + Reply::Success.write_to(stream)?; + Ok(()) + } + Ok(packet) => { + error!("received unexpected aux packet: {:?}", packet); + Reply::Error.write_to(stream)?; + Err(drtio::Error::UnexpectedReply.into()) + } + Err(e) => { + error!("aux packet error ({})", e); + Reply::Error.write_to(stream)?; + Err(e.into()) + } + } + } + + pub fn config_read(io: &Io, aux_mutex: &Mutex, + ddma_mutex: &Mutex, subkernel_mutex: &Mutex, + routing_table: &drtio_routing::RoutingTable, linkno: u8, + destination: u8, stream: &mut TcpStream, key: &String) -> Result<(), Error> { + let mut config_key: [u8; MASTER_PAYLOAD_MAX_SIZE] = [0; MASTER_PAYLOAD_MAX_SIZE]; + let len = key.len(); + config_key[..len].clone_from_slice(key.as_bytes()); + + let mut reply = drtio::aux_transact(io, aux_mutex, ddma_mutex, subkernel_mutex, routing_table, linkno, + &Packet::CoreMgmtConfigReadRequest { + destination: destination, + length: len as u16, + key: config_key, + } + ); + + let mut buffer = Vec::::new(); + loop { + match reply { + Ok(Packet::CoreMgmtConfigReadReply { length, last, value }) => { + buffer.extend(&value[..length as usize]); + + if last { + Reply::ConfigData(&value[..length as usize]).write_to(stream)?; + return Ok(()); + } + + reply = drtio::aux_transact(io, aux_mutex, ddma_mutex, subkernel_mutex, routing_table, linkno, + &Packet::CoreMgmtConfigReadContinue { + destination: destination, + } + ); + } + Ok(packet) => { + error!("received unexpected aux packet: {:?}", packet); + Reply::Error.write_to(stream)?; + return Err(drtio::Error::UnexpectedReply.into()); + } + Err(e) => { + error!("aux packet error ({})", e); + Reply::Error.write_to(stream)?; + return Err(e.into()); + } + } + } + } + + pub fn config_write(io: &Io, aux_mutex: &Mutex, + ddma_mutex: &Mutex, subkernel_mutex: &Mutex, + routing_table: &drtio_routing::RoutingTable, linkno: u8, + destination: u8, stream: &mut TcpStream, key: &String, value: &Vec, + _restart_idle: &Urc>) -> Result<(), Error> { + let mut message = Cursor::new(Vec::with_capacity(key.len() + value.len() + 4 * 2)); + message.write_string(key).unwrap(); + message.write_bytes(value).unwrap(); + + match drtio::partition_data(message.get_ref(), |slice, status, len: usize| { + let reply = drtio::aux_transact(io, aux_mutex, ddma_mutex, subkernel_mutex, routing_table, linkno, + &Packet::CoreMgmtConfigWriteRequest { + destination: destination, length: len as u16, last: status.is_last(), data: *slice}); + match reply { + Ok(Packet::CoreMgmtAck) => Ok(()), + Ok(packet) => { + error!("received unexpected aux packet: {:?}", packet); + Err(drtio::Error::UnexpectedReply) + } + Err(e) => { + error!("aux packet error ({})", e); + Err(e) + } + } + }) { + Ok(()) => { + Reply::Success.write_to(stream)?; + Ok(()) + }, + Err(e) => { + Reply::Error.write_to(stream)?; + Err(e.into()) + }, + } + } + + pub fn config_remove(io: &Io, aux_mutex: &Mutex, + ddma_mutex: &Mutex, subkernel_mutex: &Mutex, + routing_table: &drtio_routing::RoutingTable, linkno: u8, + destination: u8, stream: &mut TcpStream, key: &String, + _restart_idle: &Urc>) -> Result<(), Error> { + let mut config_key: [u8; MASTER_PAYLOAD_MAX_SIZE] = [0; MASTER_PAYLOAD_MAX_SIZE]; + let len = key.len(); + config_key[..len].clone_from_slice(key.as_bytes()); + + let reply = drtio::aux_transact(io, aux_mutex, ddma_mutex, subkernel_mutex, routing_table, linkno, + &Packet::CoreMgmtConfigRemoveRequest { + destination: destination, + length: key.len() as u16, + key: config_key, + }); + + match reply { + Ok(Packet::CoreMgmtAck) => { + Reply::Success.write_to(stream)?; + Ok(()) + } + Ok(packet) => { + error!("received unexpected aux packet: {:?}", packet); + Reply::Error.write_to(stream)?; + Err(drtio::Error::UnexpectedReply.into()) + } + Err(e) => { + error!("aux packet error ({})", e); + Reply::Error.write_to(stream)?; + Err(e.into()) + } + } + } + + pub fn config_erase(io: &Io, aux_mutex: &Mutex, + ddma_mutex: &Mutex, subkernel_mutex: &Mutex, + routing_table: &drtio_routing::RoutingTable, linkno: u8, + destination: u8, stream: &mut TcpStream, _restart_idle: &Urc>) -> Result<(), Error> { + let reply = drtio::aux_transact(io, aux_mutex, ddma_mutex, subkernel_mutex, routing_table, linkno, + &Packet::CoreMgmtConfigEraseRequest { + destination: destination, + }); + + match reply { + Ok(Packet::CoreMgmtAck) => { + Reply::Success.write_to(stream)?; + Ok(()) + } + Ok(packet) => { + error!("received unexpected aux packet: {:?}", packet); + Reply::Error.write_to(stream)?; + Err(drtio::Error::UnexpectedReply.into()) + } + Err(e) => { + error!("aux packet error ({})", e); + Reply::Error.write_to(stream)?; + Err(e.into()) + } + } + } + + pub fn reboot(io: &Io, aux_mutex: &Mutex, + ddma_mutex: &Mutex, subkernel_mutex: &Mutex, + routing_table: &drtio_routing::RoutingTable, linkno: u8, + destination: u8, stream: &mut TcpStream) -> Result<(), Error> { + let reply = drtio::aux_transact(io, aux_mutex, ddma_mutex, subkernel_mutex, routing_table, linkno, + &Packet::CoreMgmtRebootRequest { + destination: destination, + }); + + match reply { + Ok(Packet::CoreMgmtAck) => { + Reply::RebootImminent.write_to(stream)?; + Ok(()) + } + Ok(packet) => { + error!("received unexpected aux packet: {:?}", packet); + Reply::Error.write_to(stream)?; + Err(drtio::Error::UnexpectedReply.into()) + } + Err(e) => { + error!("aux packet error ({})", e); + Reply::Error.write_to(stream)?; + Err(e.into()) + } + } + } + + pub fn debug_allocator(io: &Io, aux_mutex: &Mutex, + ddma_mutex: &Mutex, subkernel_mutex: &Mutex, + routing_table: &drtio_routing::RoutingTable, linkno: u8, + destination: u8, _stream: &mut TcpStream) -> Result<(), Error> { + let reply = drtio::aux_transact(io, aux_mutex, ddma_mutex, subkernel_mutex, routing_table, linkno, + &Packet::CoreMgmtAllocatorDebugRequest { + destination: destination, + }); + + match reply { + Ok(Packet::CoreMgmtAck) => Ok(()), + Ok(packet) => { + error!("received unexpected aux packet: {:?}", packet); + Err(drtio::Error::UnexpectedReply.into()) + } + Err(e) => { + error!("aux packet error ({})", e); + Err(e.into()) + } + } + } +} + +#[cfg(has_drtio)] +macro_rules! process { + ($io:ident, $aux_mutex:ident, $ddma_mutex:ident, $subkernel_mutex:ident, $routing_table:ident, $tcp_stream:ident, $destination: ident, $func:ident $(, $param:expr)*) => {{ + let hop = $routing_table.0[$destination as usize][0]; + if hop == 0 { + local_coremgmt::$func($io, $tcp_stream, $($param, )*) + } else { + let linkno = hop - 1; + remote_coremgmt::$func($io, $aux_mutex, $ddma_mutex, $subkernel_mutex, $routing_table, linkno, $destination, $tcp_stream, $($param, )*) + } + }} +} + +#[cfg(not(has_drtio))] +macro_rules! process { + ($io:ident, $aux_mutex:ident, $ddma_mutex:ident, $subkernel_mutex:ident, $routing_table:ident, $tcp_stream:ident, $_destination: ident, $func:ident $(, $param:expr)*) => {{ + local_coremgmt::$func($io, $tcp_stream, $($param, )*) + }} +} + +fn worker(io: &Io, stream: &mut TcpStream, restart_idle: &Urc>, + _aux_mutex: &Mutex, _ddma_mutex: &Mutex, _subkernel_mutex: &Mutex, + _routing_table: &drtio_routing::RoutingTable, stream: &mut TcpStream) -> Result<(), Error> { read_magic(stream)?; + let _destination = stream.read_u8()?; Write::write_all(stream, "e".as_bytes())?; info!("new connection from {}", stream.remote_endpoint()); loop { match Request::read_from(stream)? { - Request::GetLog => { - BufferLogger::with(|logger| { - let mut buffer = io.until_ok(|| logger.buffer())?; - Reply::LogContent(buffer.extract()).write_to(stream) - })?; - } - Request::ClearLog => { - BufferLogger::with(|logger| -> Result<(), Error> { - let mut buffer = io.until_ok(|| logger.buffer())?; - Ok(buffer.clear()) - })?; - - Reply::Success.write_to(stream)?; - } - Request::PullLog => { - BufferLogger::with(|logger| -> Result<(), Error> { - loop { - // Do this *before* acquiring the buffer, since that sets the log level - // to OFF. - let log_level = log::max_level(); - - let mut buffer = io.until_ok(|| logger.buffer())?; - if buffer.is_empty() { continue } - - stream.write_string(buffer.extract())?; - - if log_level == LevelFilter::Trace { - // Hold exclusive access over the logger until we get positive - // acknowledgement; otherwise we get an infinite loop of network - // trace messages being transmitted and causing more network - // trace messages to be emitted. - // - // Any messages unrelated to this management socket that arrive - // while it is flushed are lost, but such is life. - stream.flush()?; - } - - // Clear the log *after* flushing the network buffers, or we're just - // going to resend all the trace messages on the next iteration. - buffer.clear(); - } - })?; - } - Request::SetLogFilter(level) => { - info!("changing log level to {}", level); - log::set_max_level(level); - Reply::Success.write_to(stream)?; - } - Request::SetUartLogFilter(level) => { - info!("changing UART log level to {}", level); - BufferLogger::with(|logger| - logger.set_uart_log_level(level)); - Reply::Success.write_to(stream)?; - } - - Request::ConfigRead { ref key } => { - config::read(key, |result| { - match result { - Ok(value) => Reply::ConfigData(&value).write_to(stream), - Err(_) => Reply::Error.write_to(stream) - } - })?; - } - Request::ConfigWrite { ref key, ref value } => { - match config::write(key, value) { - Ok(_) => { - if key == "idle_kernel" { - io.until(|| !restart_idle.get())?; - restart_idle.set(true); - } - Reply::Success.write_to(stream) - }, - Err(_) => Reply::Error.write_to(stream) - }?; - } - Request::ConfigRemove { ref key } => { - match config::remove(key) { - Ok(()) => { - if key == "idle_kernel" { - io.until(|| !restart_idle.get())?; - restart_idle.set(true); - } - Reply::Success.write_to(stream) - }, - Err(_) => Reply::Error.write_to(stream) - }?; - - } - Request::ConfigErase => { - match config::erase() { - Ok(()) => { - io.until(|| !restart_idle.get())?; - restart_idle.set(true); - Reply::Success.write_to(stream) - }, - Err(_) => Reply::Error.write_to(stream) - }?; - } - - Request::Reboot => { - Reply::RebootImminent.write_to(stream)?; - stream.close()?; - stream.flush()?; - - warn!("restarting"); - unsafe { spiflash::reload(); } - } - - Request::DebugAllocator => - unsafe { println!("{}", ::ALLOC) }, - }; + Request::GetLog => process!(io, _aux_mutex, _ddma_mutex, _subkernel_mutex, _routing_table, stream, _destination, get_log), + Request::ClearLog => process!(io, _aux_mutex, _ddma_mutex, _subkernel_mutex, _routing_table, stream, _destination, clear_log), + Request::PullLog => process!(io, _aux_mutex, _ddma_mutex, _subkernel_mutex, _routing_table, stream, _destination, pull_log), + Request::SetLogFilter(level) => process!(io, _aux_mutex, _ddma_mutex, _subkernel_mutex, _routing_table, stream, _destination, set_log_filter, level), + Request::SetUartLogFilter(level) => process!(io, _aux_mutex, _ddma_mutex, _subkernel_mutex, _routing_table, stream, _destination, set_uart_log_filter, level), + Request::ConfigRead { ref key } => process!(io, _aux_mutex, _ddma_mutex, _subkernel_mutex, _routing_table, stream, _destination, config_read, key), + Request::ConfigWrite { ref key, ref value } => process!(io, _aux_mutex, _ddma_mutex, _subkernel_mutex, _routing_table, stream, _destination, config_write, key, value, restart_idle), + Request::ConfigRemove { ref key } => process!(io, _aux_mutex, _ddma_mutex, _subkernel_mutex, _routing_table, stream, _destination, config_remove, key, restart_idle), + Request::ConfigErase => process!(io, _aux_mutex, _ddma_mutex, _subkernel_mutex, _routing_table, stream, _destination, config_erase, restart_idle), + Request::Reboot => process!(io, _aux_mutex, _ddma_mutex, _subkernel_mutex, _routing_table, stream, _destination, reboot), + Request::DebugAllocator => process!(io, _aux_mutex, _ddma_mutex, _subkernel_mutex, _routing_table, stream, _destination, debug_allocator), + }?; } } -pub fn thread(io: Io, restart_idle: &Urc>) { +pub fn thread(io: Io, restart_idle: &Urc>, aux_mutex: &Mutex, ddma_mutex: &Mutex, subkernel_mutex: &Mutex, routing_table: &Urc>) { let listener = TcpListener::new(&io, 8192); listener.listen(1380).expect("mgmt: cannot listen"); info!("management interface active"); loop { - let stream = listener.accept().expect("mgmt: cannot accept").into_handle(); let restart_idle = restart_idle.clone(); - io.spawn(4096, move |io| { + let aux_mutex = aux_mutex.clone(); + let ddma_mutex = ddma_mutex.clone(); + let subkernel_mutex = subkernel_mutex.clone(); + let routing_table = routing_table.clone(); + let stream = listener.accept().expect("mgmt: cannot accept").into_handle(); + io.spawn(16384, move |io| { + let routing_table = routing_table.borrow(); let mut stream = TcpStream::from_handle(&io, stream); - match worker(&io, &mut stream, &restart_idle) { + match worker(&io, &mut stream, &restart_idle, &aux_mutex, &ddma_mutex, &subkernel_mutex, &routing_table) { Ok(()) => (), Err(Error::Io(IoError::UnexpectedEnd)) => (), Err(err) => error!("aborted: {}", err) diff --git a/artiq/firmware/runtime/rtio_mgt.rs b/artiq/firmware/runtime/rtio_mgt.rs index 120a4a1c6..bc3ac7080 100644 --- a/artiq/firmware/runtime/rtio_mgt.rs +++ b/artiq/firmware/runtime/rtio_mgt.rs @@ -506,7 +506,7 @@ pub mod drtio { } } - fn partition_data(data: &[u8], send_f: F) -> Result<(), Error> + pub fn partition_data(data: &[u8], send_f: F) -> Result<(), Error> where F: Fn(&[u8; MASTER_PAYLOAD_MAX_SIZE], PayloadStatus, usize) -> Result<(), Error> { let mut i = 0; while i < data.len() { From 05578b282ebc8551251ebf4a8c82833b2e5081dc Mon Sep 17 00:00:00 2001 From: occheung Date: Mon, 26 Aug 2024 17:40:43 +0800 Subject: [PATCH 057/111] satman: support remote drtio instruction for core mgmt --- artiq/firmware/satman/main.rs | 111 ++++++++++++++++++++++++++++++++-- artiq/firmware/satman/mgmt.rs | 52 ++++++++++++++++ 2 files changed, 159 insertions(+), 4 deletions(-) create mode 100644 artiq/firmware/satman/mgmt.rs diff --git a/artiq/firmware/satman/main.rs b/artiq/firmware/satman/main.rs index 8e265999e..9513f1a81 100644 --- a/artiq/firmware/satman/main.rs +++ b/artiq/firmware/satman/main.rs @@ -21,6 +21,7 @@ use board_artiq::si5324; use board_artiq::si549; #[cfg(soc_platform = "kasli")] use board_misoc::irq; +use board_misoc::spiflash; use board_artiq::{spi, drtioaux, drtio_routing}; #[cfg(soc_platform = "efc")] use board_artiq::ad9117; @@ -30,6 +31,7 @@ use board_artiq::drtio_eem; use riscv::register::{mcause, mepc, mtval}; use dma::Manager as DmaManager; use kernel::Manager as KernelManager; +use mgmt::Manager as CoreManager; use analyzer::Analyzer; #[global_allocator] @@ -41,6 +43,7 @@ mod dma; mod analyzer; mod kernel; mod cache; +mod mgmt; fn drtiosat_reset(reset: bool) { unsafe { @@ -129,7 +132,7 @@ macro_rules! forward { ($router:expr, $routing_table:expr, $destination:expr, $rank:expr, $self_destination:expr, $repeaters:expr, $packet:expr) => {} } -fn process_aux_packet(dmamgr: &mut DmaManager, analyzer: &mut Analyzer, kernelmgr: &mut KernelManager, +fn process_aux_packet(dmamgr: &mut DmaManager, analyzer: &mut Analyzer, kernelmgr: &mut KernelManager, coremgr: &mut CoreManager, _repeaters: &mut [repeater::Repeater], _routing_table: &mut drtio_routing::RoutingTable, rank: &mut u8, router: &mut routing::Router, self_destination: &mut u8, packet: drtioaux::Packet ) -> Result<(), drtioaux::Error> { @@ -495,6 +498,105 @@ fn process_aux_packet(dmamgr: &mut DmaManager, analyzer: &mut Analyzer, kernelmg Ok(()) } + drtioaux::Packet::CoreMgmtGetLogRequest { destination: _destination, .. } | + drtioaux::Packet::CoreMgmtClearLogRequest { destination: _destination } | + drtioaux::Packet::CoreMgmtSetLogLevelRequest {destination: _destination, .. } => { + forward!(router, _routing_table, _destination, *rank, *self_destination, _repeaters, &packet); + + error!("RISC-V satellite devices do not support buffered logging"); + drtioaux::send(0, &drtioaux::Packet::CoreMgmtNack) + } + drtioaux::Packet::CoreMgmtSetUartLogLevelRequest { destination: _destination, .. } => { + forward!(router, _routing_table, _destination, *rank, *self_destination, _repeaters, &packet); + + error!("RISC-V satellite devices has fixed UART log level fixed at TRACE"); + drtioaux::send(0, &drtioaux::Packet::CoreMgmtNack) + } + drtioaux::Packet::CoreMgmtConfigReadRequest { + destination: _destination, + length, + key, + } => { + forward!(router, _routing_table, _destination, *rank, *self_destination, _repeaters, &packet); + + let mut value_slice = [0; SAT_PAYLOAD_MAX_SIZE]; + + let key_slice = &key[..length as usize]; + if !key_slice.is_ascii() { + error!("invalid key"); + drtioaux::send(0, &drtioaux::Packet::CoreMgmtNack) + } else { + let key = core::str::from_utf8(key_slice).unwrap(); + if coremgr.fetch_config_value(key).is_ok() { + let meta = coremgr.get_config_value_slice(&mut value_slice); + drtioaux::send( + 0, + &drtioaux::Packet::CoreMgmtConfigReadReply { + length: meta.len as u16, + last: meta.status.is_last(), + value: value_slice, + }, + ) + } else { + drtioaux::send(0, &drtioaux::Packet::CoreMgmtNack) + } + } + } + drtioaux::Packet::CoreMgmtConfigReadContinue { + destination: _destination, + } => { + forward!(router, _routing_table, _destination, *rank, *self_destination, _repeaters, &packet); + + let mut value_slice = [0; SAT_PAYLOAD_MAX_SIZE]; + let meta = coremgr.get_config_value_slice(&mut value_slice); + drtioaux::send( + 0, + &drtioaux::Packet::CoreMgmtConfigReadReply { + length: meta.len as u16, + last: meta.status.is_last(), + value: value_slice, + }, + ) + } + drtioaux::Packet::CoreMgmtConfigWriteRequest { destination: _destination, length, last, data } => { + forward!(router, _routing_table, _destination, *rank, *self_destination, _repeaters, &packet); + + coremgr.add_data(&data, length as usize); + let mut succeeded = true; + if last { + succeeded = coremgr.write_config().is_ok(); + debug!("Write succeeded: {}", succeeded); + coremgr.clear_data(); + } + + if succeeded { + drtioaux::send(0, &drtioaux::Packet::CoreMgmtAck) + } else { + drtioaux::send(0, &drtioaux::Packet::CoreMgmtNack) + } + } + drtioaux::Packet::CoreMgmtConfigRemoveRequest { destination: _destination, length, key } => { + forward!(router, _routing_table, _destination, *rank, *self_destination, _repeaters, &packet); + + let key = core::str::from_utf8(&key[..length as usize]).unwrap(); + let succeeded = config::remove(key) + .map_err(|err| warn!("error on removing config: {:?}", err)) + .is_ok(); + + if succeeded { + drtioaux::send(0, &drtioaux::Packet::CoreMgmtAck) + } else { + drtioaux::send(0, &drtioaux::Packet::CoreMgmtNack) + } + } + drtioaux::Packet::CoreMgmtRebootRequest { destination: _destination } => { + forward!(router, _routing_table, _destination, *rank, *self_destination, _repeaters, &packet); + + drtioaux::send(0, &drtioaux::Packet::CoreMgmtAck)?; + warn!("restarting"); + unsafe { spiflash::reload(); } + } + _ => { warn!("received unexpected aux packet"); Ok(()) @@ -503,13 +605,13 @@ fn process_aux_packet(dmamgr: &mut DmaManager, analyzer: &mut Analyzer, kernelmg } fn process_aux_packets(dma_manager: &mut DmaManager, analyzer: &mut Analyzer, - kernelmgr: &mut KernelManager, repeaters: &mut [repeater::Repeater], + kernelmgr: &mut KernelManager, coremgr: &mut CoreManager, repeaters: &mut [repeater::Repeater], routing_table: &mut drtio_routing::RoutingTable, rank: &mut u8, router: &mut routing::Router, destination: &mut u8) { let result = drtioaux::recv(0).and_then(|packet| { if let Some(packet) = packet.or_else(|| router.get_local_packet()) { - process_aux_packet(dma_manager, analyzer, kernelmgr, + process_aux_packet(dma_manager, analyzer, kernelmgr, coremgr, repeaters, routing_table, rank, router, destination, packet) } else { Ok(()) @@ -834,6 +936,7 @@ pub extern fn main() -> i32 { let mut dma_manager = DmaManager::new(); let mut analyzer = Analyzer::new(); let mut kernelmgr = KernelManager::new(); + let mut coremgr = CoreManager::new(); cricon_select(RtioMaster::Drtio); drtioaux::reset(0); @@ -843,7 +946,7 @@ pub extern fn main() -> i32 { while drtiosat_link_rx_up() { drtiosat_process_errors(); process_aux_packets(&mut dma_manager, &mut analyzer, - &mut kernelmgr, &mut repeaters, &mut routing_table, + &mut kernelmgr, &mut coremgr, &mut repeaters, &mut routing_table, &mut rank, &mut router, &mut destination); for rep in repeaters.iter_mut() { rep.service(&routing_table, rank, destination, &mut router); diff --git a/artiq/firmware/satman/mgmt.rs b/artiq/firmware/satman/mgmt.rs new file mode 100644 index 000000000..01c64dcef --- /dev/null +++ b/artiq/firmware/satman/mgmt.rs @@ -0,0 +1,52 @@ +use alloc::vec::Vec; + +use routing::{Sliceable, SliceMeta}; +use board_misoc::config; +use io::{Cursor, ProtoRead, ProtoWrite}; +use proto_artiq::drtioaux_proto::SAT_PAYLOAD_MAX_SIZE; + + +type Result = core::result::Result; + +pub struct Manager { + current_payload: Cursor>, + last_value: Sliceable, +} + +impl Manager { + pub fn new() -> Manager { + Manager { + current_payload: Cursor::new(Vec::new()), + last_value: Sliceable::new(0, Vec::new()), + } + } + + pub fn fetch_config_value(&mut self, key: &str) -> Result<()> { + config::read(key, |result| result.map( + |value| self.last_value = Sliceable::new(0, value.to_vec()) + )).map_err(|_err| warn!("read error: no such key")) + } + + pub fn get_config_value_slice(&mut self, data_slice: &mut [u8; SAT_PAYLOAD_MAX_SIZE]) -> SliceMeta { + self.last_value.get_slice_sat(data_slice) + } + + pub fn add_data(&mut self, data: &[u8], data_len: usize) { + self.current_payload.write_all(&data[..data_len]).unwrap(); + } + + pub fn clear_data(&mut self) { + self.current_payload.get_mut().clear(); + self.current_payload.set_position(0); + } + + pub fn write_config(&mut self) -> Result<()> { + let key = self.current_payload.read_string().map_err( + |err| error!("error on reading key: {:?}", err))?; + let value = self.current_payload.read_bytes().unwrap(); + + config::write(&key, &value).map_err(|err| { + error!("error on writing config: {:?}", err); + }) + } +} \ No newline at end of file From cbcf2bd84a174df9fb7fe17a9fcbf34efee37e9f Mon Sep 17 00:00:00 2001 From: occheung Date: Mon, 26 Aug 2024 17:41:27 +0800 Subject: [PATCH 058/111] frontend: pass drtio destination during communication --- artiq/coredevice/comm_mgmt.py | 4 +++- artiq/frontend/aqctl_corelog.py | 8 ++++++-- artiq/frontend/artiq_coremgmt.py | 7 ++++++- 3 files changed, 15 insertions(+), 4 deletions(-) diff --git a/artiq/coredevice/comm_mgmt.py b/artiq/coredevice/comm_mgmt.py index 870e2759d..1b218afec 100644 --- a/artiq/coredevice/comm_mgmt.py +++ b/artiq/coredevice/comm_mgmt.py @@ -46,15 +46,17 @@ class LogLevel(Enum): class CommMgmt: - def __init__(self, host, port=1380): + def __init__(self, host, port=1380, drtio_dest=0): self.host = host self.port = port + self.drtio_dest = drtio_dest def open(self): if hasattr(self, "socket"): return self.socket = create_connection(self.host, self.port) self.socket.sendall(b"ARTIQ management\n") + self._write_int8(self.drtio_dest) endian = self._read(1) if endian == b"e": self.endian = "<" diff --git a/artiq/frontend/aqctl_corelog.py b/artiq/frontend/aqctl_corelog.py index c5f5a8eee..d2dd33a4a 100755 --- a/artiq/frontend/aqctl_corelog.py +++ b/artiq/frontend/aqctl_corelog.py @@ -25,6 +25,9 @@ def get_argparser(): help="Simulation - does not connect to device") parser.add_argument("core_addr", metavar="CORE_ADDR", help="hostname or IP address of the core device") + parser.add_argument("-s", "--satellite", default=0, + metavar="DRTIO_ID", type=int, + help="the logged DRTIO destination") return parser @@ -39,7 +42,7 @@ async def get_logs_sim(host): log_with_name("firmware.simulation", logging.INFO, "hello " + host) -async def get_logs(host): +async def get_logs(host, drtio_dest): try: reader, writer = await async_open_connection( host, @@ -49,6 +52,7 @@ async def get_logs(host): max_fails=3, ) writer.write(b"ARTIQ management\n") + writer.write(drtio_dest.to_bytes(1)) endian = await reader.readexactly(1) if endian == b"e": endian = "<" @@ -96,7 +100,7 @@ def main(): signal_handler.setup() try: get_logs_task = asyncio.ensure_future( - get_logs_sim(args.core_addr) if args.simulation else get_logs(args.core_addr), + get_logs_sim(args.core_addr) if args.simulation else get_logs(args.core_addr, args.satellite), loop=loop) try: server = Server({"corelog": PingTarget()}, None, True) diff --git a/artiq/frontend/artiq_coremgmt.py b/artiq/frontend/artiq_coremgmt.py index 789c26bf7..2fbf450c0 100755 --- a/artiq/frontend/artiq_coremgmt.py +++ b/artiq/frontend/artiq_coremgmt.py @@ -95,6 +95,11 @@ def get_argparser(): p_allocator = subparsers.add_parser("allocator", help="show heap layout") + # manage target + p_drtio_dest = parser.add_argument("-s", "--satellite", default=0, + metavar="DRTIO ID", type=int, + help="specify DRTIO destination that receives this command") + return parser @@ -107,7 +112,7 @@ def main(): core_addr = ddb.get("core", resolve_alias=True)["arguments"]["host"] else: core_addr = args.device - mgmt = CommMgmt(core_addr) + mgmt = CommMgmt(core_addr, drtio_dest=args.satellite) if args.tool == "log": if args.action == "set_level": From 455e2a8f9d1a7988e0326906c92f4c4d1059ee1f Mon Sep 17 00:00:00 2001 From: occheung Date: Tue, 27 Aug 2024 21:36:07 +0800 Subject: [PATCH 059/111] satman coremgmt: EOF line --- artiq/firmware/satman/mgmt.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/artiq/firmware/satman/mgmt.rs b/artiq/firmware/satman/mgmt.rs index 01c64dcef..245e5f4d6 100644 --- a/artiq/firmware/satman/mgmt.rs +++ b/artiq/firmware/satman/mgmt.rs @@ -49,4 +49,4 @@ impl Manager { error!("error on writing config: {:?}", err); }) } -} \ No newline at end of file +} From 644e24be349c213e0597ab3005c858ebc0d3fe44 Mon Sep 17 00:00:00 2001 From: occheung Date: Wed, 28 Aug 2024 17:43:21 +0800 Subject: [PATCH 060/111] local coremgmt: return the whole config after read --- artiq/firmware/runtime/mgmt.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/artiq/firmware/runtime/mgmt.rs b/artiq/firmware/runtime/mgmt.rs index 0962fcc97..95ac3870b 100644 --- a/artiq/firmware/runtime/mgmt.rs +++ b/artiq/firmware/runtime/mgmt.rs @@ -333,7 +333,7 @@ mod remote_coremgmt { buffer.extend(&value[..length as usize]); if last { - Reply::ConfigData(&value[..length as usize]).write_to(stream)?; + Reply::ConfigData(&buffer).write_to(stream)?; return Ok(()); } From f6cf66966ddd28e40f56bfb95212cb536b838134 Mon Sep 17 00:00:00 2001 From: occheung Date: Thu, 29 Aug 2024 12:30:15 +0800 Subject: [PATCH 061/111] remote coremgmt: restart device gracefully after flashing --- .../firmware/libproto_artiq/drtioaux_proto.rs | 3 ++ artiq/firmware/runtime/mgmt.rs | 1 + artiq/firmware/satman/main.rs | 11 +--- artiq/firmware/satman/mgmt.rs | 50 +++++++++++++++---- 4 files changed, 46 insertions(+), 19 deletions(-) diff --git a/artiq/firmware/libproto_artiq/drtioaux_proto.rs b/artiq/firmware/libproto_artiq/drtioaux_proto.rs index ba826873a..8dbe9c98a 100644 --- a/artiq/firmware/libproto_artiq/drtioaux_proto.rs +++ b/artiq/firmware/libproto_artiq/drtioaux_proto.rs @@ -143,6 +143,7 @@ pub enum Packet { CoreMgmtConfigReadReply { last: bool, length: u16, value: [u8; SAT_PAYLOAD_MAX_SIZE] }, CoreMgmtAck, CoreMgmtNack, + CoreMgmtRebootImminent, } impl Packet { @@ -507,6 +508,7 @@ impl Packet { }, 0xdd => Packet::CoreMgmtAck, 0xde => Packet::CoreMgmtNack, + 0xdf => Packet::CoreMgmtRebootImminent, ty => return Err(Error::UnknownPacket(ty)) }) @@ -878,6 +880,7 @@ impl Packet { }, Packet::CoreMgmtAck => writer.write_u8(0xdd)?, Packet::CoreMgmtNack => writer.write_u8(0xde)?, + Packet::CoreMgmtRebootImminent => writer.write_u8(0xdf)?, } Ok(()) } diff --git a/artiq/firmware/runtime/mgmt.rs b/artiq/firmware/runtime/mgmt.rs index 95ac3870b..34a2c7d2e 100644 --- a/artiq/firmware/runtime/mgmt.rs +++ b/artiq/firmware/runtime/mgmt.rs @@ -372,6 +372,7 @@ mod remote_coremgmt { destination: destination, length: len as u16, last: status.is_last(), data: *slice}); match reply { Ok(Packet::CoreMgmtAck) => Ok(()), + Ok(Packet::CoreMgmtRebootImminent) if status.is_last() => Ok(()), Ok(packet) => { error!("received unexpected aux packet: {:?}", packet); Err(drtio::Error::UnexpectedReply) diff --git a/artiq/firmware/satman/main.rs b/artiq/firmware/satman/main.rs index 9513f1a81..c7ca63775 100644 --- a/artiq/firmware/satman/main.rs +++ b/artiq/firmware/satman/main.rs @@ -562,17 +562,10 @@ fn process_aux_packet(dmamgr: &mut DmaManager, analyzer: &mut Analyzer, kernelmg forward!(router, _routing_table, _destination, *rank, *self_destination, _repeaters, &packet); coremgr.add_data(&data, length as usize); - let mut succeeded = true; if last { - succeeded = coremgr.write_config().is_ok(); - debug!("Write succeeded: {}", succeeded); - coremgr.clear_data(); - } - - if succeeded { - drtioaux::send(0, &drtioaux::Packet::CoreMgmtAck) + coremgr.write_config() } else { - drtioaux::send(0, &drtioaux::Packet::CoreMgmtNack) + drtioaux::send(0, &drtioaux::Packet::CoreMgmtAck) } } drtioaux::Packet::CoreMgmtConfigRemoveRequest { destination: _destination, length, key } => { diff --git a/artiq/firmware/satman/mgmt.rs b/artiq/firmware/satman/mgmt.rs index 245e5f4d6..7e220fcd9 100644 --- a/artiq/firmware/satman/mgmt.rs +++ b/artiq/firmware/satman/mgmt.rs @@ -1,13 +1,12 @@ use alloc::vec::Vec; use routing::{Sliceable, SliceMeta}; -use board_misoc::config; +use board_artiq::drtioaux; +use board_misoc::{clock, config, csr, spiflash}; use io::{Cursor, ProtoRead, ProtoWrite}; use proto_artiq::drtioaux_proto::SAT_PAYLOAD_MAX_SIZE; -type Result = core::result::Result; - pub struct Manager { current_payload: Cursor>, last_value: Sliceable, @@ -21,7 +20,7 @@ impl Manager { } } - pub fn fetch_config_value(&mut self, key: &str) -> Result<()> { + pub fn fetch_config_value(&mut self, key: &str) -> Result<(), ()> { config::read(key, |result| result.map( |value| self.last_value = Sliceable::new(0, value.to_vec()) )).map_err(|_err| warn!("read error: no such key")) @@ -40,13 +39,44 @@ impl Manager { self.current_payload.set_position(0); } - pub fn write_config(&mut self) -> Result<()> { - let key = self.current_payload.read_string().map_err( - |err| error!("error on reading key: {:?}", err))?; + pub fn write_config(&mut self) -> Result<(), drtioaux::Error> { + let key = match self.current_payload.read_string() { + Ok(key) => key, + Err(err) => { + self.clear_data(); + error!("error on reading key: {:?}", err); + return drtioaux::send(0, &drtioaux::Packet::CoreMgmtNack); + } + }; + let value = self.current_payload.read_bytes().unwrap(); - config::write(&key, &value).map_err(|err| { - error!("error on writing config: {:?}", err); - }) + match key.as_str() { + "gateware" | "bootloader" | "firmware" => { + drtioaux::send(0, &drtioaux::Packet::CoreMgmtRebootImminent)?; + #[cfg(not(soc_platform = "efc"))] + unsafe { + clock::spin_us(10000); + csr::gt_drtio::txenable_write(0); + } + config::write(&key, &value).expect("failed to write to flash storage"); + warn!("restarting"); + unsafe { spiflash::reload(); } + } + + _ => { + let succeeded = config::write(&key, &value).map_err(|err| { + error!("error on writing config: {:?}", err); + }).is_ok(); + + self.clear_data(); + + if succeeded { + drtioaux::send(0, &drtioaux::Packet::CoreMgmtAck) + } else { + drtioaux::send(0, &drtioaux::Packet::CoreMgmtNack) + } + } + } } } From 7af8511de38398b986b1d373993823f4d7373dda Mon Sep 17 00:00:00 2001 From: occheung Date: Thu, 29 Aug 2024 12:40:36 +0800 Subject: [PATCH 062/111] drtio-proto: remove reboot imminent message --- artiq/firmware/libproto_artiq/drtioaux_proto.rs | 3 --- artiq/firmware/runtime/mgmt.rs | 1 - artiq/firmware/satman/mgmt.rs | 2 +- 3 files changed, 1 insertion(+), 5 deletions(-) diff --git a/artiq/firmware/libproto_artiq/drtioaux_proto.rs b/artiq/firmware/libproto_artiq/drtioaux_proto.rs index 8dbe9c98a..ba826873a 100644 --- a/artiq/firmware/libproto_artiq/drtioaux_proto.rs +++ b/artiq/firmware/libproto_artiq/drtioaux_proto.rs @@ -143,7 +143,6 @@ pub enum Packet { CoreMgmtConfigReadReply { last: bool, length: u16, value: [u8; SAT_PAYLOAD_MAX_SIZE] }, CoreMgmtAck, CoreMgmtNack, - CoreMgmtRebootImminent, } impl Packet { @@ -508,7 +507,6 @@ impl Packet { }, 0xdd => Packet::CoreMgmtAck, 0xde => Packet::CoreMgmtNack, - 0xdf => Packet::CoreMgmtRebootImminent, ty => return Err(Error::UnknownPacket(ty)) }) @@ -880,7 +878,6 @@ impl Packet { }, Packet::CoreMgmtAck => writer.write_u8(0xdd)?, Packet::CoreMgmtNack => writer.write_u8(0xde)?, - Packet::CoreMgmtRebootImminent => writer.write_u8(0xdf)?, } Ok(()) } diff --git a/artiq/firmware/runtime/mgmt.rs b/artiq/firmware/runtime/mgmt.rs index 34a2c7d2e..95ac3870b 100644 --- a/artiq/firmware/runtime/mgmt.rs +++ b/artiq/firmware/runtime/mgmt.rs @@ -372,7 +372,6 @@ mod remote_coremgmt { destination: destination, length: len as u16, last: status.is_last(), data: *slice}); match reply { Ok(Packet::CoreMgmtAck) => Ok(()), - Ok(Packet::CoreMgmtRebootImminent) if status.is_last() => Ok(()), Ok(packet) => { error!("received unexpected aux packet: {:?}", packet); Err(drtio::Error::UnexpectedReply) diff --git a/artiq/firmware/satman/mgmt.rs b/artiq/firmware/satman/mgmt.rs index 7e220fcd9..1c0641493 100644 --- a/artiq/firmware/satman/mgmt.rs +++ b/artiq/firmware/satman/mgmt.rs @@ -53,7 +53,7 @@ impl Manager { match key.as_str() { "gateware" | "bootloader" | "firmware" => { - drtioaux::send(0, &drtioaux::Packet::CoreMgmtRebootImminent)?; + drtioaux::send(0, &drtioaux::Packet::CoreMgmtAck)?; #[cfg(not(soc_platform = "efc"))] unsafe { clock::spin_us(10000); From 5502aefa393804163e3c7ad24dc8adea4bd631c4 Mon Sep 17 00:00:00 2001 From: occheung Date: Thu, 29 Aug 2024 13:08:38 +0800 Subject: [PATCH 063/111] drtio-proto: merge coremgmt ACK adn NACK --- .../firmware/libproto_artiq/drtioaux_proto.rs | 14 ++++++++------ artiq/firmware/runtime/mgmt.rs | 16 ++++++++-------- artiq/firmware/satman/main.rs | 18 +++++++----------- artiq/firmware/satman/mgmt.rs | 10 +++------- 4 files changed, 26 insertions(+), 32 deletions(-) diff --git a/artiq/firmware/libproto_artiq/drtioaux_proto.rs b/artiq/firmware/libproto_artiq/drtioaux_proto.rs index ba826873a..4aeeaec12 100644 --- a/artiq/firmware/libproto_artiq/drtioaux_proto.rs +++ b/artiq/firmware/libproto_artiq/drtioaux_proto.rs @@ -141,8 +141,7 @@ pub enum Packet { CoreMgmtAllocatorDebugRequest { destination: u8 }, CoreMgmtGetLogReply { last: bool, length: u16, data: [u8; SAT_PAYLOAD_MAX_SIZE] }, CoreMgmtConfigReadReply { last: bool, length: u16, value: [u8; SAT_PAYLOAD_MAX_SIZE] }, - CoreMgmtAck, - CoreMgmtNack, + CoreMgmtReply { succeeded: bool }, } impl Packet { @@ -505,8 +504,9 @@ impl Packet { value: value, } }, - 0xdd => Packet::CoreMgmtAck, - 0xde => Packet::CoreMgmtNack, + 0xdd => Packet::CoreMgmtReply { + succeeded: reader.read_bool()?, + }, ty => return Err(Error::UnknownPacket(ty)) }) @@ -876,8 +876,10 @@ impl Packet { writer.write_u16(length)?; writer.write_all(&value[0..length as usize])?; }, - Packet::CoreMgmtAck => writer.write_u8(0xdd)?, - Packet::CoreMgmtNack => writer.write_u8(0xde)?, + Packet::CoreMgmtReply { succeeded } => { + writer.write_u8(0xdd)?; + writer.write_bool(succeeded)?; + }, } Ok(()) } diff --git a/artiq/firmware/runtime/mgmt.rs b/artiq/firmware/runtime/mgmt.rs index 95ac3870b..3fd70079d 100644 --- a/artiq/firmware/runtime/mgmt.rs +++ b/artiq/firmware/runtime/mgmt.rs @@ -216,7 +216,7 @@ mod remote_coremgmt { ); match reply { - Ok(Packet::CoreMgmtAck) => { + Ok(Packet::CoreMgmtReply { succeeded: true }) => { Reply::Success.write_to(stream)?; Ok(()) } @@ -267,7 +267,7 @@ mod remote_coremgmt { ); match reply { - Ok(Packet::CoreMgmtAck) => { + Ok(Packet::CoreMgmtReply { succeeded: true }) => { Reply::Success.write_to(stream)?; Ok(()) } @@ -293,7 +293,7 @@ mod remote_coremgmt { ); match reply { - Ok(Packet::CoreMgmtAck) => { + Ok(Packet::CoreMgmtReply { succeeded: true }) => { Reply::Success.write_to(stream)?; Ok(()) } @@ -371,7 +371,7 @@ mod remote_coremgmt { &Packet::CoreMgmtConfigWriteRequest { destination: destination, length: len as u16, last: status.is_last(), data: *slice}); match reply { - Ok(Packet::CoreMgmtAck) => Ok(()), + Ok(Packet::CoreMgmtReply { succeeded: true }) => Ok(()), Ok(packet) => { error!("received unexpected aux packet: {:?}", packet); Err(drtio::Error::UnexpectedReply) @@ -410,7 +410,7 @@ mod remote_coremgmt { }); match reply { - Ok(Packet::CoreMgmtAck) => { + Ok(Packet::CoreMgmtReply { succeeded: true }) => { Reply::Success.write_to(stream)?; Ok(()) } @@ -437,7 +437,7 @@ mod remote_coremgmt { }); match reply { - Ok(Packet::CoreMgmtAck) => { + Ok(Packet::CoreMgmtReply { succeeded: true }) => { Reply::Success.write_to(stream)?; Ok(()) } @@ -464,7 +464,7 @@ mod remote_coremgmt { }); match reply { - Ok(Packet::CoreMgmtAck) => { + Ok(Packet::CoreMgmtReply { succeeded: true }) => { Reply::RebootImminent.write_to(stream)?; Ok(()) } @@ -491,7 +491,7 @@ mod remote_coremgmt { }); match reply { - Ok(Packet::CoreMgmtAck) => Ok(()), + Ok(Packet::CoreMgmtReply { succeeded: true }) => Ok(()), Ok(packet) => { error!("received unexpected aux packet: {:?}", packet); Err(drtio::Error::UnexpectedReply.into()) diff --git a/artiq/firmware/satman/main.rs b/artiq/firmware/satman/main.rs index c7ca63775..16ccf31fb 100644 --- a/artiq/firmware/satman/main.rs +++ b/artiq/firmware/satman/main.rs @@ -504,13 +504,13 @@ fn process_aux_packet(dmamgr: &mut DmaManager, analyzer: &mut Analyzer, kernelmg forward!(router, _routing_table, _destination, *rank, *self_destination, _repeaters, &packet); error!("RISC-V satellite devices do not support buffered logging"); - drtioaux::send(0, &drtioaux::Packet::CoreMgmtNack) + drtioaux::send(0, &drtioaux::Packet::CoreMgmtReply { succeeded: false }) } drtioaux::Packet::CoreMgmtSetUartLogLevelRequest { destination: _destination, .. } => { forward!(router, _routing_table, _destination, *rank, *self_destination, _repeaters, &packet); error!("RISC-V satellite devices has fixed UART log level fixed at TRACE"); - drtioaux::send(0, &drtioaux::Packet::CoreMgmtNack) + drtioaux::send(0, &drtioaux::Packet::CoreMgmtReply { succeeded: false }) } drtioaux::Packet::CoreMgmtConfigReadRequest { destination: _destination, @@ -524,7 +524,7 @@ fn process_aux_packet(dmamgr: &mut DmaManager, analyzer: &mut Analyzer, kernelmg let key_slice = &key[..length as usize]; if !key_slice.is_ascii() { error!("invalid key"); - drtioaux::send(0, &drtioaux::Packet::CoreMgmtNack) + drtioaux::send(0, &drtioaux::Packet::CoreMgmtReply { succeeded: false }) } else { let key = core::str::from_utf8(key_slice).unwrap(); if coremgr.fetch_config_value(key).is_ok() { @@ -538,7 +538,7 @@ fn process_aux_packet(dmamgr: &mut DmaManager, analyzer: &mut Analyzer, kernelmg }, ) } else { - drtioaux::send(0, &drtioaux::Packet::CoreMgmtNack) + drtioaux::send(0, &drtioaux::Packet::CoreMgmtReply { succeeded: false }) } } } @@ -565,7 +565,7 @@ fn process_aux_packet(dmamgr: &mut DmaManager, analyzer: &mut Analyzer, kernelmg if last { coremgr.write_config() } else { - drtioaux::send(0, &drtioaux::Packet::CoreMgmtAck) + drtioaux::send(0, &drtioaux::Packet::CoreMgmtReply { succeeded: true }) } } drtioaux::Packet::CoreMgmtConfigRemoveRequest { destination: _destination, length, key } => { @@ -576,16 +576,12 @@ fn process_aux_packet(dmamgr: &mut DmaManager, analyzer: &mut Analyzer, kernelmg .map_err(|err| warn!("error on removing config: {:?}", err)) .is_ok(); - if succeeded { - drtioaux::send(0, &drtioaux::Packet::CoreMgmtAck) - } else { - drtioaux::send(0, &drtioaux::Packet::CoreMgmtNack) - } + drtioaux::send(0, &drtioaux::Packet::CoreMgmtReply { succeeded }) } drtioaux::Packet::CoreMgmtRebootRequest { destination: _destination } => { forward!(router, _routing_table, _destination, *rank, *self_destination, _repeaters, &packet); - drtioaux::send(0, &drtioaux::Packet::CoreMgmtAck)?; + drtioaux::send(0, &drtioaux::Packet::CoreMgmtReply { succeeded: true })?; warn!("restarting"); unsafe { spiflash::reload(); } } diff --git a/artiq/firmware/satman/mgmt.rs b/artiq/firmware/satman/mgmt.rs index 1c0641493..86e5794e3 100644 --- a/artiq/firmware/satman/mgmt.rs +++ b/artiq/firmware/satman/mgmt.rs @@ -45,7 +45,7 @@ impl Manager { Err(err) => { self.clear_data(); error!("error on reading key: {:?}", err); - return drtioaux::send(0, &drtioaux::Packet::CoreMgmtNack); + return drtioaux::send(0, &drtioaux::Packet::CoreMgmtReply { succeeded: false }); } }; @@ -53,7 +53,7 @@ impl Manager { match key.as_str() { "gateware" | "bootloader" | "firmware" => { - drtioaux::send(0, &drtioaux::Packet::CoreMgmtAck)?; + drtioaux::send(0, &drtioaux::Packet::CoreMgmtReply { succeeded: true })?; #[cfg(not(soc_platform = "efc"))] unsafe { clock::spin_us(10000); @@ -71,11 +71,7 @@ impl Manager { self.clear_data(); - if succeeded { - drtioaux::send(0, &drtioaux::Packet::CoreMgmtAck) - } else { - drtioaux::send(0, &drtioaux::Packet::CoreMgmtNack) - } + drtioaux::send(0, &drtioaux::Packet::CoreMgmtReply { succeeded }) } } } From 045ebd53c4fe12e0808905d4d25f3e16f508bdaa Mon Sep 17 00:00:00 2001 From: occheung Date: Fri, 30 Aug 2024 18:41:45 +0800 Subject: [PATCH 064/111] coremgmt: implement flash --- artiq/coredevice/comm_mgmt.py | 46 ++++++++++++++++ artiq/firmware/Cargo.lock | 1 + artiq/firmware/libboard_misoc/config.rs | 37 ++----------- artiq/firmware/libboard_misoc/spiflash.rs | 44 +++++++++++++++ artiq/firmware/libproto_artiq/mgmt_proto.rs | 6 ++ artiq/firmware/runtime/Cargo.toml | 1 + artiq/firmware/runtime/main.rs | 1 + artiq/firmware/runtime/mgmt.rs | 61 ++++++++++++++++++++- artiq/frontend/artiq_coremgmt.py | 43 +++++++++++++++ 9 files changed, 206 insertions(+), 34 deletions(-) diff --git a/artiq/coredevice/comm_mgmt.py b/artiq/coredevice/comm_mgmt.py index 1b218afec..f8f95fc84 100644 --- a/artiq/coredevice/comm_mgmt.py +++ b/artiq/coredevice/comm_mgmt.py @@ -1,5 +1,7 @@ from enum import Enum +import binascii import logging +import io import struct from sipyco.keepalive import create_connection @@ -23,6 +25,8 @@ class Request(Enum): DebugAllocator = 8 + Flash = 9 + class Reply(Enum): Success = 1 @@ -196,3 +200,45 @@ class CommMgmt: def debug_allocator(self): self._write_header(Request.DebugAllocator) + + def flash(self, **bin_paths): + self._write_header(Request.Flash) + + addr_table = {} + with io.BytesIO() as image_buf, io.BytesIO() as bin_buf: + offset = 0 + # Reserve 4-bytes for CRC + image_buf.write(struct.pack(self.endian + "I", 0)) + # Reserve 4-bytes for header length + image_buf.write(struct.pack(self.endian + "I", 0)) + image_buf.write(struct.pack(self.endian + "I", len(bin_paths))) + for bin_name, filename in bin_paths.items(): + with open(filename, "rb") as fi: + bin_ = fi.read() + length = bin_buf.write(bin_) + + bin_name_str = bin_name.encode("utf-8") + image_buf.write(struct.pack(self.endian + "I", len(bin_name_str))) + image_buf.write(bin_name_str) + image_buf.write(struct.pack(self.endian + "II", offset, length)) + + offset += length + + # header = image_buf.getvalue() + # image = image_buf.getvalue() + + assert(image_buf.tell() == len(image_buf.getvalue())) + header_len = image_buf.tell() - 8 + image_buf.seek(4, 0) + image_buf.write(struct.pack(self.endian + "I", header_len)) + image_buf.seek(0, 2) + image_buf.write(bin_buf.getvalue()) + + image_buf.seek(4, 0) + crc = binascii.crc32(image_buf.read()) + image_buf.seek(0, 0) + image_buf.write(struct.pack(self.endian + "I", crc)) + + self._write_bytes(image_buf.getvalue()) + + self._read_expect(Reply.RebootImminent) diff --git a/artiq/firmware/Cargo.lock b/artiq/firmware/Cargo.lock index 527a662ee..a14353341 100644 --- a/artiq/firmware/Cargo.lock +++ b/artiq/firmware/Cargo.lock @@ -513,6 +513,7 @@ dependencies = [ "board_misoc", "build_misoc", "byteorder", + "crc", "cslice", "dyld", "eh", diff --git a/artiq/firmware/libboard_misoc/config.rs b/artiq/firmware/libboard_misoc/config.rs index a7c3d4a63..e5e62ad0c 100644 --- a/artiq/firmware/libboard_misoc/config.rs +++ b/artiq/firmware/libboard_misoc/config.rs @@ -259,39 +259,12 @@ mod imp { } pub fn write(key: &str, value: &[u8]) -> Result<(), Error> { - fn flash_binary(origin: usize, payload: &[u8]) { - let mut offset = 0; - while offset < payload.len() { - unsafe { - spiflash::erase_sector(origin + offset); - } - offset += spiflash::SECTOR_SIZE; - } - unsafe { - spiflash::write(origin, payload); - } - } - - match key { - "gateware" => { - flash_binary(0, value); - Ok(()) - } - "bootloader" => { - flash_binary(::mem::ROM_BASE, value); - Ok(()) - } - "firmware" => { - flash_binary(::mem::FLASH_BOOT_ADDRESS, value); - Ok(()) - } - _ => match append(key, value) { - Err(Error::SpaceExhausted) => { - compact()?; - append(key, value) - } - res => res + match append(key, value) { + Err(Error::SpaceExhausted) => { + compact()?; + append(key, value) } + res => res } } diff --git a/artiq/firmware/libboard_misoc/spiflash.rs b/artiq/firmware/libboard_misoc/spiflash.rs index 598d96633..0f25ecbbc 100644 --- a/artiq/firmware/libboard_misoc/spiflash.rs +++ b/artiq/firmware/libboard_misoc/spiflash.rs @@ -114,6 +114,50 @@ pub unsafe fn write(mut addr: usize, mut data: &[u8]) { } } +// pub unsafe fn write_image(image: &[u8]) { +// let image = &image[..]; +// let actual_crc = crc32::checksum_ieee(image); + +// if actual_crc == expected_crc { +// let mut reader = Cursor::new(header); +// let bin_no = reader.read_u32().unwrap() as usize; +// for _ in 0..bin_no { +// let bin_name = reader.read_string().unwrap(); +// let offset = reader.read_u32().unwrap() as usize; +// let len = reader.read_u32().unwrap() as usize; + +// let origin = match bin_name.as_str() { +// "gateware" => 0, +// "bootloader" => mem::ROM_BASE, +// "firmware" => mem::FLASH_BOOT_ADDRESS, +// _ => { +// error!("unexpected binary component {}", bin_name); +// return Ok(Reply::Error.write_to(stream)?); +// } +// }; + +// unsafe { +// spiflash::flash_binary(origin, &image[offset..offset+len]); +// } +// } + +// reboot(_io, stream)?; +// } else { +// error!("CRC failed in SDRAM (actual {:08x}, expected {:08x})", actual_crc, expected_crc); +// Reply::Error.write_to(stream)?; +// } +// } + +pub unsafe fn flash_binary(origin: usize, payload: &[u8]) { + assert!((origin & (SECTOR_SIZE - 1)) == 0); + let mut offset = 0; + while offset < payload.len() { + erase_sector(origin + offset); + offset += SECTOR_SIZE; + } + write(origin, payload); +} + #[cfg(any(soc_platform = "kasli", soc_platform = "kc705"))] pub unsafe fn reload () -> ! { csr::icap::iprog_write(1); diff --git a/artiq/firmware/libproto_artiq/mgmt_proto.rs b/artiq/firmware/libproto_artiq/mgmt_proto.rs index 74724df13..bfc8cc004 100644 --- a/artiq/firmware/libproto_artiq/mgmt_proto.rs +++ b/artiq/firmware/libproto_artiq/mgmt_proto.rs @@ -67,6 +67,8 @@ pub enum Request { Reboot, + Flash { image: Vec }, + DebugAllocator, } @@ -125,6 +127,10 @@ impl Request { 8 => Request::DebugAllocator, + 9 => Request::Flash { + image: reader.read_bytes()?, + }, + ty => return Err(Error::UnknownPacket(ty)) }) } diff --git a/artiq/firmware/runtime/Cargo.toml b/artiq/firmware/runtime/Cargo.toml index 0d132d5a9..f06b93e49 100644 --- a/artiq/firmware/runtime/Cargo.toml +++ b/artiq/firmware/runtime/Cargo.toml @@ -16,6 +16,7 @@ build_misoc = { path = "../libbuild_misoc" } failure = { version = "0.1", default-features = false } failure_derive = { version = "0.1", default-features = false } byteorder = { version = "1.0", default-features = false } +crc = { version = "1.7", default-features = false } cslice = { version = "0.3" } log = { version = "=0.4.14", default-features = false } managed = { version = "^0.7.1", default-features = false, features = ["alloc", "map"] } diff --git a/artiq/firmware/runtime/main.rs b/artiq/firmware/runtime/main.rs index 78bf9b594..a66b48fcf 100644 --- a/artiq/firmware/runtime/main.rs +++ b/artiq/firmware/runtime/main.rs @@ -1,6 +1,7 @@ #![feature(lang_items, panic_info_message, const_btree_new, iter_advance_by, never_type)] #![no_std] +extern crate crc; extern crate dyld; extern crate eh; #[macro_use] diff --git a/artiq/firmware/runtime/mgmt.rs b/artiq/firmware/runtime/mgmt.rs index 3fd70079d..e798db894 100644 --- a/artiq/firmware/runtime/mgmt.rs +++ b/artiq/firmware/runtime/mgmt.rs @@ -17,10 +17,11 @@ impl From for Error { mod local_coremgmt { use alloc::{string::String, vec::Vec}; + use crc::crc32; use log::LevelFilter; - use board_misoc::{config, spiflash}; - use io::{Write, ProtoWrite, Error as IoError}; + use board_misoc::{config, mem, spiflash}; + use io::{Cursor, Write, ProtoWrite, ProtoRead, Error as IoError}; use logger_artiq::BufferLogger; use mgmt_proto::{Error, Reply}; use sched::{Io, TcpStream, Error as SchedError}; @@ -153,6 +154,54 @@ mod local_coremgmt { unsafe { println!("{}", ::ALLOC) } Ok(()) } + + pub fn flash(_io: &Io, stream: &mut TcpStream, image: &Vec) -> Result<(), Error> { + let mut reader = Cursor::new(&image[..]); + let expected_crc = reader.read_u32().unwrap(); + + let image = &image[4..]; + let actual_crc = crc32::checksum_ieee(image); + + if actual_crc == expected_crc { + info!("Checksum matched"); + let header_size = reader.read_u32().unwrap() as usize; + let header_offset = reader.position(); + let bin_offset = header_offset + header_size; + + let header = &image[header_offset..bin_offset]; + let binaries = &image[bin_offset..]; + + info!("found header of size {}", header.len()); + + let mut reader = Cursor::new(header); + let bin_no = reader.read_u32().unwrap() as usize; + for _ in 0..bin_no { + let bin_name = reader.read_string().unwrap(); + let offset = reader.read_u32().unwrap() as usize; + let len = reader.read_u32().unwrap() as usize; + + let origin = match bin_name.as_str() { + "gateware" => 0, + "bootloader" => mem::ROM_BASE, + "firmware" => mem::FLASH_BOOT_ADDRESS, + _ => { + error!("unexpected binary component {}", bin_name); + return Ok(Reply::Error.write_to(stream)?); + } + }; + + unsafe { + spiflash::flash_binary(origin, &binaries[offset..offset+len]); + } + } + + reboot(_io, stream)?; + } else { + error!("CRC failed in SDRAM (actual {:08x}, expected {:08x})", actual_crc, expected_crc); + Reply::Error.write_to(stream)?; + } + Ok(()) + } } #[cfg(has_drtio)] @@ -502,6 +551,13 @@ mod remote_coremgmt { } } } + + pub fn flash(io: &Io, aux_mutex: &Mutex, + ddma_mutex: &Mutex, subkernel_mutex: &Mutex, + routing_table: &drtio_routing::RoutingTable, linkno: u8, + destination: u8, stream: &mut TcpStream, image: &Vec) -> Result<(), Error> { + todo!() + } } #[cfg(has_drtio)] @@ -545,6 +601,7 @@ fn worker(io: &Io, stream: &mut TcpStream, restart_idle: &Urc>, Request::ConfigErase => process!(io, _aux_mutex, _ddma_mutex, _subkernel_mutex, _routing_table, stream, _destination, config_erase, restart_idle), Request::Reboot => process!(io, _aux_mutex, _ddma_mutex, _subkernel_mutex, _routing_table, stream, _destination, reboot), Request::DebugAllocator => process!(io, _aux_mutex, _ddma_mutex, _subkernel_mutex, _routing_table, stream, _destination, debug_allocator), + Request::Flash { ref image } => process!(io, _aux_mutex, _ddma_mutex, _subkernel_mutex, _routing_table, stream, _destination, flash, image), }?; } } diff --git a/artiq/frontend/artiq_coremgmt.py b/artiq/frontend/artiq_coremgmt.py index 2fbf450c0..d6c1826ae 100755 --- a/artiq/frontend/artiq_coremgmt.py +++ b/artiq/frontend/artiq_coremgmt.py @@ -1,7 +1,10 @@ #!/usr/bin/env python3 import argparse +import os import struct +import tempfile +import atexit from sipyco import common_args @@ -9,6 +12,8 @@ from artiq import __version__ as artiq_version from artiq.master.databases import DeviceDB from artiq.coredevice.comm_kernel import CommKernel from artiq.coredevice.comm_mgmt import CommMgmt +from artiq.frontend.bit2bin import bit2bin +from misoc.tools.mkmscimg import insert_crc def get_argparser(): @@ -85,6 +90,13 @@ def get_argparser(): t_boot = tools.add_parser("reboot", help="reboot the running system") + # flashing + t_flash = tools.add_parser("flash", + help="flash the running system") + + p_directory = t_flash.add_argument("directory", metavar="DIRECTORY", type=str, + help="directory that contains the binaries") + # misc debug t_debug = tools.add_parser("debug", help="specialized debug functions") @@ -142,6 +154,37 @@ def main(): mgmt.config_remove(key) if args.action == "erase": mgmt.config_erase() + + if args.tool == "flash": + def convert_gateware(bit_filename): + bin_handle, bin_filename = tempfile.mkstemp( + prefix="artiq_", suffix="_" + os.path.basename(bit_filename)) + with open(bit_filename, "rb") as bit_file, open(bin_handle, "wb") as bin_file: + bit2bin(bit_file, bin_file) + atexit.register(lambda: os.unlink(bin_filename)) + return bin_filename + + gateware = convert_gateware(os.path.join(args.directory, "top.bit")) + bootloader = os.path.join(args.directory, "bootloader.bin") + + firmwares = [] + for firmware in "satman", "runtime": + filename = os.path.join(args.directory, firmware + ".fbi") + if os.path.exists(filename): + firmwares.append(filename) + if not firmwares: + raise FileNotFoundError("no firmware found") + if len(firmwares) > 1: + raise ValueError("more than one firmware file, please clean up your build directory. " + "Found firmware files: {}".format(" ".join(firmwares))) + firmware = firmwares[0] + + bins = { + "gateware": gateware, + "bootloader": bootloader, + "firmware": firmware, + } + mgmt.flash(**bins) if args.tool == "reboot": mgmt.reboot() From c5988ab48b64fe1a8cb99d23d61f593f6f0a3f2a Mon Sep 17 00:00:00 2001 From: occheung Date: Mon, 2 Sep 2024 16:55:34 +0800 Subject: [PATCH 065/111] mgmt flash: simplify protocol --- artiq/coredevice/comm_mgmt.py | 37 ++++------------------- artiq/firmware/runtime/mgmt.rs | 51 ++++++++++++-------------------- artiq/frontend/artiq_coremgmt.py | 8 ++--- 3 files changed, 27 insertions(+), 69 deletions(-) diff --git a/artiq/coredevice/comm_mgmt.py b/artiq/coredevice/comm_mgmt.py index f8f95fc84..b0ab94509 100644 --- a/artiq/coredevice/comm_mgmt.py +++ b/artiq/coredevice/comm_mgmt.py @@ -201,42 +201,17 @@ class CommMgmt: def debug_allocator(self): self._write_header(Request.DebugAllocator) - def flash(self, **bin_paths): + def flash(self, bin_paths): self._write_header(Request.Flash) - addr_table = {} - with io.BytesIO() as image_buf, io.BytesIO() as bin_buf: - offset = 0 - # Reserve 4-bytes for CRC - image_buf.write(struct.pack(self.endian + "I", 0)) - # Reserve 4-bytes for header length - image_buf.write(struct.pack(self.endian + "I", 0)) - image_buf.write(struct.pack(self.endian + "I", len(bin_paths))) - for bin_name, filename in bin_paths.items(): + with io.BytesIO() as image_buf: + for filename in bin_paths: with open(filename, "rb") as fi: bin_ = fi.read() - length = bin_buf.write(bin_) + image_buf.write(struct.pack(self.endian + "I", len(bin_))) + image_buf.write(bin_) - bin_name_str = bin_name.encode("utf-8") - image_buf.write(struct.pack(self.endian + "I", len(bin_name_str))) - image_buf.write(bin_name_str) - image_buf.write(struct.pack(self.endian + "II", offset, length)) - - offset += length - - # header = image_buf.getvalue() - # image = image_buf.getvalue() - - assert(image_buf.tell() == len(image_buf.getvalue())) - header_len = image_buf.tell() - 8 - image_buf.seek(4, 0) - image_buf.write(struct.pack(self.endian + "I", header_len)) - image_buf.seek(0, 2) - image_buf.write(bin_buf.getvalue()) - - image_buf.seek(4, 0) - crc = binascii.crc32(image_buf.read()) - image_buf.seek(0, 0) + crc = binascii.crc32(image_buf.getvalue()) image_buf.write(struct.pack(self.endian + "I", crc)) self._write_bytes(image_buf.getvalue()) diff --git a/artiq/firmware/runtime/mgmt.rs b/artiq/firmware/runtime/mgmt.rs index e798db894..33959f9cd 100644 --- a/artiq/firmware/runtime/mgmt.rs +++ b/artiq/firmware/runtime/mgmt.rs @@ -17,11 +17,12 @@ impl From for Error { mod local_coremgmt { use alloc::{string::String, vec::Vec}; + use byteorder::{ByteOrder, NativeEndian}; use crc::crc32; use log::LevelFilter; use board_misoc::{config, mem, spiflash}; - use io::{Cursor, Write, ProtoWrite, ProtoRead, Error as IoError}; + use io::{Write, ProtoWrite, Error as IoError}; use logger_artiq::BufferLogger; use mgmt_proto::{Error, Reply}; use sched::{Io, TcpStream, Error as SchedError}; @@ -155,44 +156,30 @@ mod local_coremgmt { Ok(()) } - pub fn flash(_io: &Io, stream: &mut TcpStream, image: &Vec) -> Result<(), Error> { - let mut reader = Cursor::new(&image[..]); - let expected_crc = reader.read_u32().unwrap(); + pub fn flash(_io: &Io, stream: &mut TcpStream, image: &[u8]) -> Result<(), Error> { + let (expected_crc, mut image) = { + let (image, crc_slice) = image.split_at(image.len() - 4); + (NativeEndian::read_u32(crc_slice), image) + }; - let image = &image[4..]; let actual_crc = crc32::checksum_ieee(image); if actual_crc == expected_crc { - info!("Checksum matched"); - let header_size = reader.read_u32().unwrap() as usize; - let header_offset = reader.position(); - let bin_offset = header_offset + header_size; + let bin_origins = [ + ("gateware" , 0 ), + ("bootloader", mem::ROM_BASE ), + ("firmware" , mem::FLASH_BOOT_ADDRESS), + ]; - let header = &image[header_offset..bin_offset]; - let binaries = &image[bin_offset..]; + for (name, origin) in bin_origins { + info!("Flashing {} binary...", name); + let size = NativeEndian::read_u32(&image[..4]) as usize; + image = &image[4..]; - info!("found header of size {}", header.len()); + let (bin, remaining) = image.split_at(size); + image = remaining; - let mut reader = Cursor::new(header); - let bin_no = reader.read_u32().unwrap() as usize; - for _ in 0..bin_no { - let bin_name = reader.read_string().unwrap(); - let offset = reader.read_u32().unwrap() as usize; - let len = reader.read_u32().unwrap() as usize; - - let origin = match bin_name.as_str() { - "gateware" => 0, - "bootloader" => mem::ROM_BASE, - "firmware" => mem::FLASH_BOOT_ADDRESS, - _ => { - error!("unexpected binary component {}", bin_name); - return Ok(Reply::Error.write_to(stream)?); - } - }; - - unsafe { - spiflash::flash_binary(origin, &binaries[offset..offset+len]); - } + unsafe { spiflash::flash_binary(origin, bin) }; } reboot(_io, stream)?; diff --git a/artiq/frontend/artiq_coremgmt.py b/artiq/frontend/artiq_coremgmt.py index d6c1826ae..8b23c7e7c 100755 --- a/artiq/frontend/artiq_coremgmt.py +++ b/artiq/frontend/artiq_coremgmt.py @@ -179,12 +179,8 @@ def main(): "Found firmware files: {}".format(" ".join(firmwares))) firmware = firmwares[0] - bins = { - "gateware": gateware, - "bootloader": bootloader, - "firmware": firmware, - } - mgmt.flash(**bins) + bins = [ gateware, bootloader, firmware ] + mgmt.flash(bins) if args.tool == "reboot": mgmt.reboot() From bb8148e554ba6744a60d8e06e624f37dc6d3f861 Mon Sep 17 00:00:00 2001 From: occheung Date: Mon, 2 Sep 2024 16:57:38 +0800 Subject: [PATCH 066/111] mgmt: implement flash via drtio --- artiq/firmware/Cargo.lock | 2 + .../firmware/libproto_artiq/drtioaux_proto.rs | 21 ++++ artiq/firmware/runtime/mgmt.rs | 31 +++++- artiq/firmware/satman/Cargo.toml | 2 + artiq/firmware/satman/main.rs | 16 ++- artiq/firmware/satman/mgmt.rs | 98 +++++++++++++------ 6 files changed, 137 insertions(+), 33 deletions(-) diff --git a/artiq/firmware/Cargo.lock b/artiq/firmware/Cargo.lock index a14353341..aa156989e 100644 --- a/artiq/firmware/Cargo.lock +++ b/artiq/firmware/Cargo.lock @@ -554,6 +554,8 @@ dependencies = [ "board_artiq", "board_misoc", "build_misoc", + "byteorder", + "crc", "cslice", "eh", "io", diff --git a/artiq/firmware/libproto_artiq/drtioaux_proto.rs b/artiq/firmware/libproto_artiq/drtioaux_proto.rs index 4aeeaec12..267b983df 100644 --- a/artiq/firmware/libproto_artiq/drtioaux_proto.rs +++ b/artiq/firmware/libproto_artiq/drtioaux_proto.rs @@ -139,6 +139,7 @@ pub enum Packet { CoreMgmtConfigEraseRequest { destination: u8 }, CoreMgmtRebootRequest { destination: u8 }, CoreMgmtAllocatorDebugRequest { destination: u8 }, + CoreMgmtFlashRequest { destination: u8, last: bool, length: u16, data: [u8; MASTER_PAYLOAD_MAX_SIZE] }, CoreMgmtGetLogReply { last: bool, length: u16, data: [u8; SAT_PAYLOAD_MAX_SIZE] }, CoreMgmtConfigReadReply { last: bool, length: u16, value: [u8; SAT_PAYLOAD_MAX_SIZE] }, CoreMgmtReply { succeeded: bool }, @@ -507,6 +508,19 @@ impl Packet { 0xdd => Packet::CoreMgmtReply { succeeded: reader.read_bool()?, }, + 0xde => { + let destination = reader.read_u8()?; + let last = reader.read_bool()?; + let length = reader.read_u16()?; + let mut data: [u8; MASTER_PAYLOAD_MAX_SIZE] = [0; MASTER_PAYLOAD_MAX_SIZE]; + reader.read_exact(&mut data[0..length as usize])?; + Packet::CoreMgmtFlashRequest { + destination: destination, + last: last, + length: length, + data: data, + } + }, ty => return Err(Error::UnknownPacket(ty)) }) @@ -880,6 +894,13 @@ impl Packet { writer.write_u8(0xdd)?; writer.write_bool(succeeded)?; }, + Packet::CoreMgmtFlashRequest { destination, last, length, data } => { + writer.write_u8(0xde)?; + writer.write_u8(destination)?; + writer.write_bool(last)?; + writer.write_u16(length)?; + writer.write_all(&data[..length as usize])?; + }, } Ok(()) } diff --git a/artiq/firmware/runtime/mgmt.rs b/artiq/firmware/runtime/mgmt.rs index 33959f9cd..65567b3bc 100644 --- a/artiq/firmware/runtime/mgmt.rs +++ b/artiq/firmware/runtime/mgmt.rs @@ -542,8 +542,33 @@ mod remote_coremgmt { pub fn flash(io: &Io, aux_mutex: &Mutex, ddma_mutex: &Mutex, subkernel_mutex: &Mutex, routing_table: &drtio_routing::RoutingTable, linkno: u8, - destination: u8, stream: &mut TcpStream, image: &Vec) -> Result<(), Error> { - todo!() + destination: u8, stream: &mut TcpStream, image: &[u8]) -> Result<(), Error> { + + match drtio::partition_data(&image, |slice, status, len: usize| { + let reply = drtio::aux_transact(io, aux_mutex, ddma_mutex, subkernel_mutex, routing_table, linkno, + &Packet::CoreMgmtFlashRequest { + destination: destination, length: len as u16, last: status.is_last(), data: *slice}); + match reply { + Ok(Packet::CoreMgmtReply { succeeded: true }) => Ok(()), + Ok(packet) => { + error!("received unexpected aux packet: {:?}", packet); + Err(drtio::Error::UnexpectedReply) + } + Err(e) => { + error!("aux packet error ({})", e); + Err(e) + } + } + }) { + Ok(()) => { + Reply::RebootImminent.write_to(stream)?; + Ok(()) + }, + Err(e) => { + Reply::Error.write_to(stream)?; + Err(e.into()) + }, + } } } @@ -588,7 +613,7 @@ fn worker(io: &Io, stream: &mut TcpStream, restart_idle: &Urc>, Request::ConfigErase => process!(io, _aux_mutex, _ddma_mutex, _subkernel_mutex, _routing_table, stream, _destination, config_erase, restart_idle), Request::Reboot => process!(io, _aux_mutex, _ddma_mutex, _subkernel_mutex, _routing_table, stream, _destination, reboot), Request::DebugAllocator => process!(io, _aux_mutex, _ddma_mutex, _subkernel_mutex, _routing_table, stream, _destination, debug_allocator), - Request::Flash { ref image } => process!(io, _aux_mutex, _ddma_mutex, _subkernel_mutex, _routing_table, stream, _destination, flash, image), + Request::Flash { ref image } => process!(io, _aux_mutex, _ddma_mutex, _subkernel_mutex, _routing_table, stream, _destination, flash, &image[..]), }?; } } diff --git a/artiq/firmware/satman/Cargo.toml b/artiq/firmware/satman/Cargo.toml index f8016f576..522ae7b32 100644 --- a/artiq/firmware/satman/Cargo.toml +++ b/artiq/firmware/satman/Cargo.toml @@ -15,6 +15,8 @@ build_misoc = { path = "../libbuild_misoc" } [dependencies] log = { version = "0.4", default-features = false } io = { path = "../libio", features = ["byteorder", "alloc"] } +byteorder = { version = "1.0", default-features = false } +crc = { version = "1.7", default-features = false } cslice = { version = "0.3" } board_misoc = { path = "../libboard_misoc", features = ["uart_console", "log"] } board_artiq = { path = "../libboard_artiq", features = ["alloc"] } diff --git a/artiq/firmware/satman/main.rs b/artiq/firmware/satman/main.rs index 16ccf31fb..9a98528a6 100644 --- a/artiq/firmware/satman/main.rs +++ b/artiq/firmware/satman/main.rs @@ -9,6 +9,8 @@ extern crate board_artiq; extern crate riscv; extern crate alloc; extern crate proto_artiq; +extern crate byteorder; +extern crate crc; extern crate cslice; extern crate io; extern crate eh; @@ -558,10 +560,10 @@ fn process_aux_packet(dmamgr: &mut DmaManager, analyzer: &mut Analyzer, kernelmg }, ) } - drtioaux::Packet::CoreMgmtConfigWriteRequest { destination: _destination, length, last, data } => { + drtioaux::Packet::CoreMgmtConfigWriteRequest { destination: _destination, last, length, data } => { forward!(router, _routing_table, _destination, *rank, *self_destination, _repeaters, &packet); - coremgr.add_data(&data, length as usize); + coremgr.add_config_data(&data, length as usize); if last { coremgr.write_config() } else { @@ -585,6 +587,16 @@ fn process_aux_packet(dmamgr: &mut DmaManager, analyzer: &mut Analyzer, kernelmg warn!("restarting"); unsafe { spiflash::reload(); } } + drtioaux::Packet::CoreMgmtFlashRequest { destination: _destination, last, length, data } => { + forward!(router, _routing_table, _destination, *rank, *self_destination, _repeaters, &packet); + + coremgr.add_image_data(&data, length as usize); + if last { + coremgr.flash_image() + } else { + drtioaux::send(0, &drtioaux::Packet::CoreMgmtReply { succeeded: true }) + } + } _ => { warn!("received unexpected aux packet"); diff --git a/artiq/firmware/satman/mgmt.rs b/artiq/firmware/satman/mgmt.rs index 86e5794e3..911a15d5d 100644 --- a/artiq/firmware/satman/mgmt.rs +++ b/artiq/firmware/satman/mgmt.rs @@ -1,21 +1,25 @@ use alloc::vec::Vec; +use byteorder::{ByteOrder, NativeEndian}; +use crc::crc32; use routing::{Sliceable, SliceMeta}; use board_artiq::drtioaux; -use board_misoc::{clock, config, csr, spiflash}; +use board_misoc::{mem, clock, config, csr, spiflash}; use io::{Cursor, ProtoRead, ProtoWrite}; use proto_artiq::drtioaux_proto::SAT_PAYLOAD_MAX_SIZE; pub struct Manager { - current_payload: Cursor>, + config_payload: Cursor>, + image_payload: Cursor>, last_value: Sliceable, } impl Manager { pub fn new() -> Manager { Manager { - current_payload: Cursor::new(Vec::new()), + config_payload: Cursor::new(Vec::new()), + image_payload: Cursor::new(Vec::new()), last_value: Sliceable::new(0, Vec::new()), } } @@ -30,49 +34,87 @@ impl Manager { self.last_value.get_slice_sat(data_slice) } - pub fn add_data(&mut self, data: &[u8], data_len: usize) { - self.current_payload.write_all(&data[..data_len]).unwrap(); + pub fn add_config_data(&mut self, data: &[u8], data_len: usize) { + self.config_payload.write_all(&data[..data_len]).unwrap(); } - pub fn clear_data(&mut self) { - self.current_payload.get_mut().clear(); - self.current_payload.set_position(0); + pub fn clear_config_data(&mut self) { + self.config_payload.get_mut().clear(); + self.config_payload.set_position(0); } pub fn write_config(&mut self) -> Result<(), drtioaux::Error> { - let key = match self.current_payload.read_string() { + let key = match self.config_payload.read_string() { Ok(key) => key, Err(err) => { - self.clear_data(); + self.clear_config_data(); error!("error on reading key: {:?}", err); return drtioaux::send(0, &drtioaux::Packet::CoreMgmtReply { succeeded: false }); } }; - let value = self.current_payload.read_bytes().unwrap(); + let value = self.config_payload.read_bytes().unwrap(); - match key.as_str() { - "gateware" | "bootloader" | "firmware" => { - drtioaux::send(0, &drtioaux::Packet::CoreMgmtReply { succeeded: true })?; - #[cfg(not(soc_platform = "efc"))] - unsafe { - clock::spin_us(10000); - csr::gt_drtio::txenable_write(0); - } - config::write(&key, &value).expect("failed to write to flash storage"); - warn!("restarting"); - unsafe { spiflash::reload(); } + let succeeded = config::write(&key, &value).map_err(|err| { + error!("error on writing config: {:?}", err); + }).is_ok(); + + self.clear_config_data(); + + drtioaux::send(0, &drtioaux::Packet::CoreMgmtReply { succeeded }) + } + + pub fn add_image_data(&mut self, data: &[u8], data_len: usize) { + self.image_payload.write_all(&data[..data_len]).unwrap(); + } + + pub fn clear_image_data(&mut self) { + self.image_payload.get_mut().clear(); + self.image_payload.set_position(0); + } + + pub fn flash_image(&mut self) -> Result<(), drtioaux::Error> { + let image = &self.image_payload.get_ref()[..]; + + let (expected_crc, mut image) = { + let (image, crc_slice) = image.split_at(image.len() - 4); + (NativeEndian::read_u32(crc_slice), image) + }; + + let actual_crc = crc32::checksum_ieee(image); + + if actual_crc == expected_crc { + drtioaux::send(0, &drtioaux::Packet::CoreMgmtReply { succeeded: true })?; + #[cfg(not(soc_platform = "efc"))] + unsafe { + clock::spin_us(10000); + csr::gt_drtio::txenable_write(0); } - _ => { - let succeeded = config::write(&key, &value).map_err(|err| { - error!("error on writing config: {:?}", err); - }).is_ok(); + let bin_origins = [ + ("gateware" , 0 ), + ("bootloader", mem::ROM_BASE ), + ("firmware" , mem::FLASH_BOOT_ADDRESS), + ]; - self.clear_data(); + for (name, origin) in bin_origins { + info!("Flashing {} binary...", name); + let size = NativeEndian::read_u32(&image[..4]) as usize; + image = &image[4..]; - drtioaux::send(0, &drtioaux::Packet::CoreMgmtReply { succeeded }) + let (bin, remaining) = image.split_at(size); + image = remaining; + + unsafe { spiflash::flash_binary(origin, bin) }; } + + warn!("restarting"); + unsafe { spiflash::reload(); } + + } else { + error!("CRC failed in SDRAM (actual {:08x}, expected {:08x})", actual_crc, expected_crc); + self.clear_image_data(); + drtioaux::send(0, &drtioaux::Packet::CoreMgmtReply { succeeded: false }) } } } From 2c0a4c0bae9acd4c0b4a5489045423b6a320de7a Mon Sep 17 00:00:00 2001 From: occheung Date: Tue, 3 Sep 2024 11:42:39 +0800 Subject: [PATCH 067/111] drtio_proto: implement reboot init handshake --- .../firmware/libproto_artiq/drtioaux_proto.rs | 12 +++++++++ artiq/firmware/runtime/mgmt.rs | 12 ++++++++- artiq/firmware/satman/main.rs | 19 +++++++++++++- artiq/firmware/satman/mgmt.rs | 25 +++---------------- 4 files changed, 45 insertions(+), 23 deletions(-) diff --git a/artiq/firmware/libproto_artiq/drtioaux_proto.rs b/artiq/firmware/libproto_artiq/drtioaux_proto.rs index 267b983df..4e269fdd1 100644 --- a/artiq/firmware/libproto_artiq/drtioaux_proto.rs +++ b/artiq/firmware/libproto_artiq/drtioaux_proto.rs @@ -140,6 +140,8 @@ pub enum Packet { CoreMgmtRebootRequest { destination: u8 }, CoreMgmtAllocatorDebugRequest { destination: u8 }, CoreMgmtFlashRequest { destination: u8, last: bool, length: u16, data: [u8; MASTER_PAYLOAD_MAX_SIZE] }, + CoreMgmtDropLinkAck { destination: u8 }, + CoreMgmtDropLink, CoreMgmtGetLogReply { last: bool, length: u16, data: [u8; SAT_PAYLOAD_MAX_SIZE] }, CoreMgmtConfigReadReply { last: bool, length: u16, value: [u8; SAT_PAYLOAD_MAX_SIZE] }, CoreMgmtReply { succeeded: bool }, @@ -521,6 +523,10 @@ impl Packet { data: data, } }, + 0xdf => Packet::CoreMgmtDropLink, + 0xe0 => Packet::CoreMgmtDropLinkAck { + destination: reader.read_u8()?, + }, ty => return Err(Error::UnknownPacket(ty)) }) @@ -901,6 +907,12 @@ impl Packet { writer.write_u16(length)?; writer.write_all(&data[..length as usize])?; }, + Packet::CoreMgmtDropLink => + writer.write_u8(0xdf)?, + Packet::CoreMgmtDropLinkAck { destination } => { + writer.write_u8(0xe0)?; + writer.write_u8(destination)?; + }, } Ok(()) } diff --git a/artiq/firmware/runtime/mgmt.rs b/artiq/firmware/runtime/mgmt.rs index 65567b3bc..0c1b423d6 100644 --- a/artiq/firmware/runtime/mgmt.rs +++ b/artiq/firmware/runtime/mgmt.rs @@ -196,7 +196,7 @@ mod remote_coremgmt { use alloc::{string::String, vec::Vec}; use log::LevelFilter; - use board_artiq::{drtioaux::Packet, drtio_routing}; + use board_artiq::{drtioaux, drtioaux::Packet, drtio_routing}; use io::{Cursor, ProtoWrite}; use mgmt_proto::{Error, Reply}; use rtio_mgt::drtio; @@ -550,6 +550,16 @@ mod remote_coremgmt { destination: destination, length: len as u16, last: status.is_last(), data: *slice}); match reply { Ok(Packet::CoreMgmtReply { succeeded: true }) => Ok(()), + Ok(Packet::CoreMgmtDropLink) => { + if status.is_last() { + drtioaux::send( + linkno, &Packet::CoreMgmtDropLinkAck { destination: destination } + ).map_err(|_| drtio::Error::AuxError) + } else { + error!("received unexpected drop link packet"); + Err(drtio::Error::UnexpectedReply) + } + } Ok(packet) => { error!("received unexpected aux packet: {:?}", packet); Err(drtio::Error::UnexpectedReply) diff --git a/artiq/firmware/satman/main.rs b/artiq/firmware/satman/main.rs index 9a98528a6..1647c7955 100644 --- a/artiq/firmware/satman/main.rs +++ b/artiq/firmware/satman/main.rs @@ -592,11 +592,28 @@ fn process_aux_packet(dmamgr: &mut DmaManager, analyzer: &mut Analyzer, kernelmg coremgr.add_image_data(&data, length as usize); if last { - coremgr.flash_image() + drtioaux::send(0, &drtioaux::Packet::CoreMgmtDropLink) } else { drtioaux::send(0, &drtioaux::Packet::CoreMgmtReply { succeeded: true }) } } + drtioaux::Packet::CoreMgmtDropLinkAck { destination: _destination } => { + forward!(router, _routing_table, _destination, *rank, *self_destination, _repeaters, &packet); + + #[cfg(not(soc_platform = "efc"))] + unsafe { + csr::gt_drtio::txenable_write(0); + } + + #[cfg(has_drtio_eem)] + unsafe { + csr::eem_transceiver::txenable_write(0); + } + + coremgr.flash_image(); + warn!("restarting"); + unsafe { spiflash::reload(); } + } _ => { warn!("received unexpected aux packet"); diff --git a/artiq/firmware/satman/mgmt.rs b/artiq/firmware/satman/mgmt.rs index 911a15d5d..2b44bc4bb 100644 --- a/artiq/firmware/satman/mgmt.rs +++ b/artiq/firmware/satman/mgmt.rs @@ -4,7 +4,7 @@ use crc::crc32; use routing::{Sliceable, SliceMeta}; use board_artiq::drtioaux; -use board_misoc::{mem, clock, config, csr, spiflash}; +use board_misoc::{mem, config, spiflash}; use io::{Cursor, ProtoRead, ProtoWrite}; use proto_artiq::drtioaux_proto::SAT_PAYLOAD_MAX_SIZE; @@ -68,12 +68,7 @@ impl Manager { self.image_payload.write_all(&data[..data_len]).unwrap(); } - pub fn clear_image_data(&mut self) { - self.image_payload.get_mut().clear(); - self.image_payload.set_position(0); - } - - pub fn flash_image(&mut self) -> Result<(), drtioaux::Error> { + pub fn flash_image(&self) { let image = &self.image_payload.get_ref()[..]; let (expected_crc, mut image) = { @@ -84,13 +79,6 @@ impl Manager { let actual_crc = crc32::checksum_ieee(image); if actual_crc == expected_crc { - drtioaux::send(0, &drtioaux::Packet::CoreMgmtReply { succeeded: true })?; - #[cfg(not(soc_platform = "efc"))] - unsafe { - clock::spin_us(10000); - csr::gt_drtio::txenable_write(0); - } - let bin_origins = [ ("gateware" , 0 ), ("bootloader", mem::ROM_BASE ), @@ -98,7 +86,7 @@ impl Manager { ]; for (name, origin) in bin_origins { - info!("Flashing {} binary...", name); + info!("flashing {} binary...", name); let size = NativeEndian::read_u32(&image[..4]) as usize; image = &image[4..]; @@ -108,13 +96,8 @@ impl Manager { unsafe { spiflash::flash_binary(origin, bin) }; } - warn!("restarting"); - unsafe { spiflash::reload(); } - } else { - error!("CRC failed in SDRAM (actual {:08x}, expected {:08x})", actual_crc, expected_crc); - self.clear_image_data(); - drtioaux::send(0, &drtioaux::Packet::CoreMgmtReply { succeeded: false }) + panic!("CRC failed in SDRAM (actual {:08x}, expected {:08x})", actual_crc, expected_crc); } } } From d935c22aad784ac1b233dff5d007ca316d7eb844 Mon Sep 17 00:00:00 2001 From: occheung Date: Tue, 3 Sep 2024 11:55:34 +0800 Subject: [PATCH 068/111] remove dead commented code --- artiq/firmware/libboard_misoc/spiflash.rs | 34 ----------------------- 1 file changed, 34 deletions(-) diff --git a/artiq/firmware/libboard_misoc/spiflash.rs b/artiq/firmware/libboard_misoc/spiflash.rs index 0f25ecbbc..7d66977f7 100644 --- a/artiq/firmware/libboard_misoc/spiflash.rs +++ b/artiq/firmware/libboard_misoc/spiflash.rs @@ -114,40 +114,6 @@ pub unsafe fn write(mut addr: usize, mut data: &[u8]) { } } -// pub unsafe fn write_image(image: &[u8]) { -// let image = &image[..]; -// let actual_crc = crc32::checksum_ieee(image); - -// if actual_crc == expected_crc { -// let mut reader = Cursor::new(header); -// let bin_no = reader.read_u32().unwrap() as usize; -// for _ in 0..bin_no { -// let bin_name = reader.read_string().unwrap(); -// let offset = reader.read_u32().unwrap() as usize; -// let len = reader.read_u32().unwrap() as usize; - -// let origin = match bin_name.as_str() { -// "gateware" => 0, -// "bootloader" => mem::ROM_BASE, -// "firmware" => mem::FLASH_BOOT_ADDRESS, -// _ => { -// error!("unexpected binary component {}", bin_name); -// return Ok(Reply::Error.write_to(stream)?); -// } -// }; - -// unsafe { -// spiflash::flash_binary(origin, &image[offset..offset+len]); -// } -// } - -// reboot(_io, stream)?; -// } else { -// error!("CRC failed in SDRAM (actual {:08x}, expected {:08x})", actual_crc, expected_crc); -// Reply::Error.write_to(stream)?; -// } -// } - pub unsafe fn flash_binary(origin: usize, payload: &[u8]) { assert!((origin & (SECTOR_SIZE - 1)) == 0); let mut offset = 0; From 18744388907e60cf96b70d7b5e1a327cb2f349ec Mon Sep 17 00:00:00 2001 From: occheung Date: Tue, 3 Sep 2024 12:37:32 +0800 Subject: [PATCH 069/111] coremgmt frontend: add artiq flash like source tree support --- artiq/frontend/artiq_coremgmt.py | 20 +++++++++++++++++--- 1 file changed, 17 insertions(+), 3 deletions(-) diff --git a/artiq/frontend/artiq_coremgmt.py b/artiq/frontend/artiq_coremgmt.py index 8b23c7e7c..b51861a95 100755 --- a/artiq/frontend/artiq_coremgmt.py +++ b/artiq/frontend/artiq_coremgmt.py @@ -97,6 +97,10 @@ def get_argparser(): p_directory = t_flash.add_argument("directory", metavar="DIRECTORY", type=str, help="directory that contains the binaries") + p_srcbuild = t_flash.add_argument("--srcbuild", + help="board binaries directory is laid out as a source build tree", + default=False, action="store_true") + # misc debug t_debug = tools.add_parser("debug", help="specialized debug functions") @@ -156,6 +160,15 @@ def main(): mgmt.config_erase() if args.tool == "flash": + def artifact_path(this_binary_dir, *path_filename): + if args.srcbuild: + # source tree - use path elements to locate file + return os.path.join(this_binary_dir, *path_filename) + else: + # flat tree - all files in the same directory, discard path elements + *_, filename = path_filename + return os.path.join(this_binary_dir, filename) + def convert_gateware(bit_filename): bin_handle, bin_filename = tempfile.mkstemp( prefix="artiq_", suffix="_" + os.path.basename(bit_filename)) @@ -164,12 +177,13 @@ def main(): atexit.register(lambda: os.unlink(bin_filename)) return bin_filename - gateware = convert_gateware(os.path.join(args.directory, "top.bit")) - bootloader = os.path.join(args.directory, "bootloader.bin") + gateware = convert_gateware( + artifact_path(args.directory, "gateware", "top.bit")) + bootloader = artifact_path(args.directory, "software", "bootloader", "bootloader.bin") firmwares = [] for firmware in "satman", "runtime": - filename = os.path.join(args.directory, firmware + ".fbi") + filename = artifact_path(args.directory, "software", firmware, firmware + ".fbi") if os.path.exists(filename): firmwares.append(filename) if not firmwares: From e85b640a838c25fd55a82511a96602ff5eaa1ab7 Mon Sep 17 00:00:00 2001 From: occheung Date: Tue, 3 Sep 2024 16:46:39 +0800 Subject: [PATCH 070/111] drtop-proto: rearrange packet assignment --- .../firmware/libproto_artiq/drtioaux_proto.rs | 94 +++++++++---------- 1 file changed, 47 insertions(+), 47 deletions(-) diff --git a/artiq/firmware/libproto_artiq/drtioaux_proto.rs b/artiq/firmware/libproto_artiq/drtioaux_proto.rs index 4e269fdd1..19a6502af 100644 --- a/artiq/firmware/libproto_artiq/drtioaux_proto.rs +++ b/artiq/firmware/libproto_artiq/drtioaux_proto.rs @@ -486,31 +486,6 @@ impl Packet { destination: reader.read_u8()?, }, 0xdb => { - let last = reader.read_bool()?; - let length = reader.read_u16()?; - let mut data: [u8; SAT_PAYLOAD_MAX_SIZE] = [0; SAT_PAYLOAD_MAX_SIZE]; - reader.read_exact(&mut data[0..length as usize])?; - Packet::CoreMgmtGetLogReply { - last: last, - length: length, - data: data, - } - }, - 0xdc => { - let last = reader.read_bool()?; - let length = reader.read_u16()?; - let mut value: [u8; SAT_PAYLOAD_MAX_SIZE] = [0; SAT_PAYLOAD_MAX_SIZE]; - reader.read_exact(&mut value[0..length as usize])?; - Packet::CoreMgmtConfigReadReply { - last: last, - length: length, - value: value, - } - }, - 0xdd => Packet::CoreMgmtReply { - succeeded: reader.read_bool()?, - }, - 0xde => { let destination = reader.read_u8()?; let last = reader.read_bool()?; let length = reader.read_u16()?; @@ -523,10 +498,35 @@ impl Packet { data: data, } }, - 0xdf => Packet::CoreMgmtDropLink, - 0xe0 => Packet::CoreMgmtDropLinkAck { + 0xdc => Packet::CoreMgmtDropLinkAck { destination: reader.read_u8()?, }, + 0xdd => Packet::CoreMgmtDropLink, + 0xde => { + let last = reader.read_bool()?; + let length = reader.read_u16()?; + let mut data: [u8; SAT_PAYLOAD_MAX_SIZE] = [0; SAT_PAYLOAD_MAX_SIZE]; + reader.read_exact(&mut data[0..length as usize])?; + Packet::CoreMgmtGetLogReply { + last: last, + length: length, + data: data, + } + }, + 0xdf => { + let last = reader.read_bool()?; + let length = reader.read_u16()?; + let mut value: [u8; SAT_PAYLOAD_MAX_SIZE] = [0; SAT_PAYLOAD_MAX_SIZE]; + reader.read_exact(&mut value[0..length as usize])?; + Packet::CoreMgmtConfigReadReply { + last: last, + length: length, + value: value, + } + }, + 0xe0 => Packet::CoreMgmtReply { + succeeded: reader.read_bool()?, + }, ty => return Err(Error::UnknownPacket(ty)) }) @@ -884,35 +884,35 @@ impl Packet { writer.write_u8(0xda)?; writer.write_u8(destination)?; }, - Packet::CoreMgmtGetLogReply { last, length, data } => { - writer.write_u8(0xdb)?; - writer.write_bool(last)?; - writer.write_u16(length)?; - writer.write_all(&data[0..length as usize])?; - }, - Packet::CoreMgmtConfigReadReply { last, length, value } => { - writer.write_u8(0xdc)?; - writer.write_bool(last)?; - writer.write_u16(length)?; - writer.write_all(&value[0..length as usize])?; - }, - Packet::CoreMgmtReply { succeeded } => { - writer.write_u8(0xdd)?; - writer.write_bool(succeeded)?; - }, Packet::CoreMgmtFlashRequest { destination, last, length, data } => { - writer.write_u8(0xde)?; + writer.write_u8(0xdb)?; writer.write_u8(destination)?; writer.write_bool(last)?; writer.write_u16(length)?; writer.write_all(&data[..length as usize])?; }, - Packet::CoreMgmtDropLink => - writer.write_u8(0xdf)?, Packet::CoreMgmtDropLinkAck { destination } => { - writer.write_u8(0xe0)?; + writer.write_u8(0xdc)?; writer.write_u8(destination)?; }, + Packet::CoreMgmtDropLink => + writer.write_u8(0xdd)?, + Packet::CoreMgmtGetLogReply { last, length, data } => { + writer.write_u8(0xde)?; + writer.write_bool(last)?; + writer.write_u16(length)?; + writer.write_all(&data[0..length as usize])?; + }, + Packet::CoreMgmtConfigReadReply { last, length, value } => { + writer.write_u8(0xdf)?; + writer.write_bool(last)?; + writer.write_u16(length)?; + writer.write_all(&value[0..length as usize])?; + }, + Packet::CoreMgmtReply { succeeded } => { + writer.write_u8(0xe0)?; + writer.write_bool(succeeded)?; + }, } Ok(()) } From a56294f9de513cca8e460613491f5775b2744cbc Mon Sep 17 00:00:00 2001 From: occheung Date: Thu, 5 Sep 2024 11:56:43 +0800 Subject: [PATCH 071/111] satman coremgmt: impl config erase --- artiq/firmware/satman/main.rs | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/artiq/firmware/satman/main.rs b/artiq/firmware/satman/main.rs index 1647c7955..3909ce6d3 100644 --- a/artiq/firmware/satman/main.rs +++ b/artiq/firmware/satman/main.rs @@ -580,6 +580,15 @@ fn process_aux_packet(dmamgr: &mut DmaManager, analyzer: &mut Analyzer, kernelmg drtioaux::send(0, &drtioaux::Packet::CoreMgmtReply { succeeded }) } + drtioaux::Packet::CoreMgmtConfigEraseRequest { destination: _destination } => { + forward!(router, _routing_table, _destination, *rank, *self_destination, _repeaters, &packet); + + let succeeded = config::erase() + .map_err(|err| warn!("error on erasing config: {:?}", err)) + .is_ok(); + + drtioaux::send(0, &drtioaux::Packet::CoreMgmtReply { succeeded }) + } drtioaux::Packet::CoreMgmtRebootRequest { destination: _destination } => { forward!(router, _routing_table, _destination, *rank, *self_destination, _repeaters, &packet); From f2f27e2d308551750eb6ab6603210b8a623c9678 Mon Sep 17 00:00:00 2001 From: occheung Date: Thu, 5 Sep 2024 11:59:10 +0800 Subject: [PATCH 072/111] runtime mgmt: remove cursor --- artiq/firmware/runtime/mgmt.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/artiq/firmware/runtime/mgmt.rs b/artiq/firmware/runtime/mgmt.rs index 0c1b423d6..176a4d7ba 100644 --- a/artiq/firmware/runtime/mgmt.rs +++ b/artiq/firmware/runtime/mgmt.rs @@ -398,11 +398,11 @@ mod remote_coremgmt { routing_table: &drtio_routing::RoutingTable, linkno: u8, destination: u8, stream: &mut TcpStream, key: &String, value: &Vec, _restart_idle: &Urc>) -> Result<(), Error> { - let mut message = Cursor::new(Vec::with_capacity(key.len() + value.len() + 4 * 2)); + let mut message = Vec::with_capacity(key.len() + value.len() + 4 * 2); message.write_string(key).unwrap(); message.write_bytes(value).unwrap(); - match drtio::partition_data(message.get_ref(), |slice, status, len: usize| { + match drtio::partition_data(&message, |slice, status, len: usize| { let reply = drtio::aux_transact(io, aux_mutex, ddma_mutex, subkernel_mutex, routing_table, linkno, &Packet::CoreMgmtConfigWriteRequest { destination: destination, length: len as u16, last: status.is_last(), data: *slice}); From 4a54241e1bc2dea500df4668cc2026eeb810a3e1 Mon Sep 17 00:00:00 2001 From: occheung Date: Thu, 5 Sep 2024 11:59:23 +0800 Subject: [PATCH 073/111] runtime mgmt: reorganize uses --- artiq/firmware/runtime/mgmt.rs | 43 +++++++++++++++++----------------- 1 file changed, 21 insertions(+), 22 deletions(-) diff --git a/artiq/firmware/runtime/mgmt.rs b/artiq/firmware/runtime/mgmt.rs index 176a4d7ba..436d1d22c 100644 --- a/artiq/firmware/runtime/mgmt.rs +++ b/artiq/firmware/runtime/mgmt.rs @@ -2,7 +2,7 @@ use log::{self, LevelFilter}; use core::cell::Cell; use core::cell::RefCell; -use board_artiq::drtio_routing; +use board_artiq::drtio_routing::RoutingTable; use io::{ProtoRead, Write, Error as IoError}; use mgmt_proto::*; use sched::{Io, TcpListener, TcpStream, Error as SchedError}; @@ -22,10 +22,10 @@ mod local_coremgmt { use log::LevelFilter; use board_misoc::{config, mem, spiflash}; - use io::{Write, ProtoWrite, Error as IoError}; + use io::ProtoWrite; use logger_artiq::BufferLogger; - use mgmt_proto::{Error, Reply}; - use sched::{Io, TcpStream, Error as SchedError}; + + use super::*; pub fn get_log(io: &Io, stream: &mut TcpStream) -> Result<(), Error> { @@ -196,13 +196,12 @@ mod remote_coremgmt { use alloc::{string::String, vec::Vec}; use log::LevelFilter; - use board_artiq::{drtioaux, drtioaux::Packet, drtio_routing}; - use io::{Cursor, ProtoWrite}; - use mgmt_proto::{Error, Reply}; + use board_artiq::{drtioaux, drtioaux::Packet}; + use io::ProtoWrite; use rtio_mgt::drtio; - use sched::{Io, Mutex, TcpStream, Error as SchedError}; use proto_artiq::drtioaux_proto::MASTER_PAYLOAD_MAX_SIZE; + use super::*; impl From for Error { fn from(_value: drtio::Error) -> Error { @@ -212,7 +211,7 @@ mod remote_coremgmt { pub fn get_log(io: &Io, aux_mutex: &Mutex, ddma_mutex: &Mutex, subkernel_mutex: &Mutex, - routing_table: &drtio_routing::RoutingTable, linkno: u8, + routing_table: &RoutingTable, linkno: u8, destination: u8, stream: &mut TcpStream) -> Result<(), Error> { let mut buffer = String::new(); loop { @@ -245,7 +244,7 @@ mod remote_coremgmt { pub fn clear_log(io: &Io, aux_mutex: &Mutex, ddma_mutex: &Mutex, subkernel_mutex: &Mutex, - routing_table: &drtio_routing::RoutingTable, linkno: u8, + routing_table: &RoutingTable, linkno: u8, destination: u8, stream: &mut TcpStream) -> Result<(), Error> { let reply = drtio::aux_transact(io, aux_mutex, ddma_mutex, subkernel_mutex, routing_table, linkno, &Packet::CoreMgmtClearLogRequest { destination } @@ -271,7 +270,7 @@ mod remote_coremgmt { pub fn pull_log(io: &Io, aux_mutex: &Mutex, ddma_mutex: &Mutex, subkernel_mutex: &Mutex, - routing_table: &drtio_routing::RoutingTable, linkno: u8, + routing_table: &RoutingTable, linkno: u8, destination: u8, stream: &mut TcpStream) -> Result<(), Error> { loop { let reply = drtio::aux_transact(io, aux_mutex, ddma_mutex, subkernel_mutex, routing_table, linkno, @@ -296,7 +295,7 @@ mod remote_coremgmt { pub fn set_log_filter(io: &Io, aux_mutex: &Mutex, ddma_mutex: &Mutex, subkernel_mutex: &Mutex, - routing_table: &drtio_routing::RoutingTable, linkno: u8, + routing_table: &RoutingTable, linkno: u8, destination: u8, stream: &mut TcpStream, level: LevelFilter) -> Result<(), Error> { let reply = drtio::aux_transact(io, aux_mutex, ddma_mutex, subkernel_mutex, routing_table, linkno, &Packet::CoreMgmtSetLogLevelRequest { destination, log_level: level as u8 } @@ -322,7 +321,7 @@ mod remote_coremgmt { pub fn set_uart_log_filter(io: &Io, aux_mutex: &Mutex, ddma_mutex: &Mutex, subkernel_mutex: &Mutex, - routing_table: &drtio_routing::RoutingTable, linkno: u8, + routing_table: &RoutingTable, linkno: u8, destination: u8, stream: &mut TcpStream, level: LevelFilter) -> Result<(), Error> { let reply = drtio::aux_transact(io, aux_mutex, ddma_mutex, subkernel_mutex, routing_table, linkno, &Packet::CoreMgmtSetUartLogLevelRequest { destination, log_level: level as u8 } @@ -348,7 +347,7 @@ mod remote_coremgmt { pub fn config_read(io: &Io, aux_mutex: &Mutex, ddma_mutex: &Mutex, subkernel_mutex: &Mutex, - routing_table: &drtio_routing::RoutingTable, linkno: u8, + routing_table: &RoutingTable, linkno: u8, destination: u8, stream: &mut TcpStream, key: &String) -> Result<(), Error> { let mut config_key: [u8; MASTER_PAYLOAD_MAX_SIZE] = [0; MASTER_PAYLOAD_MAX_SIZE]; let len = key.len(); @@ -395,7 +394,7 @@ mod remote_coremgmt { pub fn config_write(io: &Io, aux_mutex: &Mutex, ddma_mutex: &Mutex, subkernel_mutex: &Mutex, - routing_table: &drtio_routing::RoutingTable, linkno: u8, + routing_table: &RoutingTable, linkno: u8, destination: u8, stream: &mut TcpStream, key: &String, value: &Vec, _restart_idle: &Urc>) -> Result<(), Error> { let mut message = Vec::with_capacity(key.len() + value.len() + 4 * 2); @@ -431,7 +430,7 @@ mod remote_coremgmt { pub fn config_remove(io: &Io, aux_mutex: &Mutex, ddma_mutex: &Mutex, subkernel_mutex: &Mutex, - routing_table: &drtio_routing::RoutingTable, linkno: u8, + routing_table: &RoutingTable, linkno: u8, destination: u8, stream: &mut TcpStream, key: &String, _restart_idle: &Urc>) -> Result<(), Error> { let mut config_key: [u8; MASTER_PAYLOAD_MAX_SIZE] = [0; MASTER_PAYLOAD_MAX_SIZE]; @@ -465,7 +464,7 @@ mod remote_coremgmt { pub fn config_erase(io: &Io, aux_mutex: &Mutex, ddma_mutex: &Mutex, subkernel_mutex: &Mutex, - routing_table: &drtio_routing::RoutingTable, linkno: u8, + routing_table: &RoutingTable, linkno: u8, destination: u8, stream: &mut TcpStream, _restart_idle: &Urc>) -> Result<(), Error> { let reply = drtio::aux_transact(io, aux_mutex, ddma_mutex, subkernel_mutex, routing_table, linkno, &Packet::CoreMgmtConfigEraseRequest { @@ -492,7 +491,7 @@ mod remote_coremgmt { pub fn reboot(io: &Io, aux_mutex: &Mutex, ddma_mutex: &Mutex, subkernel_mutex: &Mutex, - routing_table: &drtio_routing::RoutingTable, linkno: u8, + routing_table: &RoutingTable, linkno: u8, destination: u8, stream: &mut TcpStream) -> Result<(), Error> { let reply = drtio::aux_transact(io, aux_mutex, ddma_mutex, subkernel_mutex, routing_table, linkno, &Packet::CoreMgmtRebootRequest { @@ -519,7 +518,7 @@ mod remote_coremgmt { pub fn debug_allocator(io: &Io, aux_mutex: &Mutex, ddma_mutex: &Mutex, subkernel_mutex: &Mutex, - routing_table: &drtio_routing::RoutingTable, linkno: u8, + routing_table: &RoutingTable, linkno: u8, destination: u8, _stream: &mut TcpStream) -> Result<(), Error> { let reply = drtio::aux_transact(io, aux_mutex, ddma_mutex, subkernel_mutex, routing_table, linkno, &Packet::CoreMgmtAllocatorDebugRequest { @@ -541,7 +540,7 @@ mod remote_coremgmt { pub fn flash(io: &Io, aux_mutex: &Mutex, ddma_mutex: &Mutex, subkernel_mutex: &Mutex, - routing_table: &drtio_routing::RoutingTable, linkno: u8, + routing_table: &RoutingTable, linkno: u8, destination: u8, stream: &mut TcpStream, image: &[u8]) -> Result<(), Error> { match drtio::partition_data(&image, |slice, status, len: usize| { @@ -604,7 +603,7 @@ macro_rules! process { fn worker(io: &Io, stream: &mut TcpStream, restart_idle: &Urc>, _aux_mutex: &Mutex, _ddma_mutex: &Mutex, _subkernel_mutex: &Mutex, - _routing_table: &drtio_routing::RoutingTable, stream: &mut TcpStream) -> Result<(), Error> { + _routing_table: &RoutingTable, stream: &mut TcpStream) -> Result<(), Error> { read_magic(stream)?; let _destination = stream.read_u8()?; Write::write_all(stream, "e".as_bytes())?; @@ -628,7 +627,7 @@ fn worker(io: &Io, stream: &mut TcpStream, restart_idle: &Urc>, } } -pub fn thread(io: Io, restart_idle: &Urc>, aux_mutex: &Mutex, ddma_mutex: &Mutex, subkernel_mutex: &Mutex, routing_table: &Urc>) { +pub fn thread(io: Io, restart_idle: &Urc>, aux_mutex: &Mutex, ddma_mutex: &Mutex, subkernel_mutex: &Mutex, routing_table: &Urc>) { let listener = TcpListener::new(&io, 8192); listener.listen(1380).expect("mgmt: cannot listen"); info!("management interface active"); From 2b73d5a4c6f54a22b4e8cb76cf229ad0c54f924b Mon Sep 17 00:00:00 2001 From: occheung Date: Mon, 9 Sep 2024 17:28:14 +0800 Subject: [PATCH 074/111] drtio-proto: avoid expecting drop link ack response --- artiq/firmware/libproto_artiq/drtioaux_proto.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/artiq/firmware/libproto_artiq/drtioaux_proto.rs b/artiq/firmware/libproto_artiq/drtioaux_proto.rs index 19a6502af..9a0f423dc 100644 --- a/artiq/firmware/libproto_artiq/drtioaux_proto.rs +++ b/artiq/firmware/libproto_artiq/drtioaux_proto.rs @@ -946,7 +946,7 @@ impl Packet { Packet::DmaAddTraceReply { .. } | Packet::DmaRemoveTraceReply { .. } | Packet::DmaPlaybackReply { .. } | Packet::SubkernelLoadRunReply { .. } | Packet::SubkernelMessageAck { .. } | Packet::DmaPlaybackStatus { .. } | - Packet::SubkernelFinished { .. } => false, + Packet::SubkernelFinished { .. } | Packet::CoreMgmtDropLinkAck { .. } => false, _ => true } } From 5c21649d10b5a9c9de11c398a5b3bc2af3aa8d16 Mon Sep 17 00:00:00 2001 From: occheung Date: Thu, 19 Sep 2024 09:46:17 +0800 Subject: [PATCH 075/111] frontend: make coremgmt flash zynq-compatible --- artiq/coredevice/comm_mgmt.py | 3 +- artiq/frontend/artiq_coremgmt.py | 68 ++++++++++++++++++-------------- 2 files changed, 40 insertions(+), 31 deletions(-) diff --git a/artiq/coredevice/comm_mgmt.py b/artiq/coredevice/comm_mgmt.py index b0ab94509..c9f466b59 100644 --- a/artiq/coredevice/comm_mgmt.py +++ b/artiq/coredevice/comm_mgmt.py @@ -208,7 +208,8 @@ class CommMgmt: for filename in bin_paths: with open(filename, "rb") as fi: bin_ = fi.read() - image_buf.write(struct.pack(self.endian + "I", len(bin_))) + if (len(bin_paths) > 1): + image_buf.write(struct.pack(self.endian + "I", len(bin_))) image_buf.write(bin_) crc = binascii.crc32(image_buf.getvalue()) diff --git a/artiq/frontend/artiq_coremgmt.py b/artiq/frontend/artiq_coremgmt.py index b51861a95..e9cada16f 100755 --- a/artiq/frontend/artiq_coremgmt.py +++ b/artiq/frontend/artiq_coremgmt.py @@ -94,6 +94,9 @@ def get_argparser(): t_flash = tools.add_parser("flash", help="flash the running system") + p_zynq = t_flash.add_argument("-z", "--zynq", default=False, + help="target zynq device", action="store_true") + p_directory = t_flash.add_argument("directory", metavar="DIRECTORY", type=str, help="directory that contains the binaries") @@ -160,40 +163,45 @@ def main(): mgmt.config_erase() if args.tool == "flash": - def artifact_path(this_binary_dir, *path_filename): - if args.srcbuild: - # source tree - use path elements to locate file - return os.path.join(this_binary_dir, *path_filename) - else: - # flat tree - all files in the same directory, discard path elements - *_, filename = path_filename - return os.path.join(this_binary_dir, filename) + if args.zynq: + boot = os.path.join(args.directory, "boot.bin") + bins = [ boot ] + else: + def artifact_path(this_binary_dir, *path_filename): + if args.srcbuild: + # source tree - use path elements to locate file + return os.path.join(this_binary_dir, *path_filename) + else: + # flat tree - all files in the same directory, discard path elements + *_, filename = path_filename + return os.path.join(this_binary_dir, filename) - def convert_gateware(bit_filename): - bin_handle, bin_filename = tempfile.mkstemp( - prefix="artiq_", suffix="_" + os.path.basename(bit_filename)) - with open(bit_filename, "rb") as bit_file, open(bin_handle, "wb") as bin_file: - bit2bin(bit_file, bin_file) - atexit.register(lambda: os.unlink(bin_filename)) - return bin_filename + def convert_gateware(bit_filename): + bin_handle, bin_filename = tempfile.mkstemp( + prefix="artiq_", suffix="_" + os.path.basename(bit_filename)) + with open(bit_filename, "rb") as bit_file, open(bin_handle, "wb") as bin_file: + bit2bin(bit_file, bin_file) + atexit.register(lambda: os.unlink(bin_filename)) + return bin_filename - gateware = convert_gateware( - artifact_path(args.directory, "gateware", "top.bit")) - bootloader = artifact_path(args.directory, "software", "bootloader", "bootloader.bin") + gateware = convert_gateware( + artifact_path(args.directory, "gateware", "top.bit")) + bootloader = artifact_path(args.directory, "software", "bootloader", "bootloader.bin") - firmwares = [] - for firmware in "satman", "runtime": - filename = artifact_path(args.directory, "software", firmware, firmware + ".fbi") - if os.path.exists(filename): - firmwares.append(filename) - if not firmwares: - raise FileNotFoundError("no firmware found") - if len(firmwares) > 1: - raise ValueError("more than one firmware file, please clean up your build directory. " - "Found firmware files: {}".format(" ".join(firmwares))) - firmware = firmwares[0] + firmwares = [] + for firmware in "satman", "runtime": + filename = artifact_path(args.directory, "software", firmware, firmware + ".fbi") + if os.path.exists(filename): + firmwares.append(filename) + if not firmwares: + raise FileNotFoundError("no firmware found") + if len(firmwares) > 1: + raise ValueError("more than one firmware file, please clean up your build directory. " + "Found firmware files: {}".format(" ".join(firmwares))) + firmware = firmwares[0] + + bins = [ gateware, bootloader, firmware ] - bins = [ gateware, bootloader, firmware ] mgmt.flash(bins) if args.tool == "reboot": From a2d341f4d62edfac402d8f090479613b2ed4dbba Mon Sep 17 00:00:00 2001 From: occheung Date: Thu, 19 Sep 2024 17:18:16 +0800 Subject: [PATCH 076/111] satman mgmt: get_slice_sat -> get_slice_satellite --- artiq/firmware/satman/mgmt.rs | 2 +- artiq/firmware/satman/routing.rs | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/artiq/firmware/satman/mgmt.rs b/artiq/firmware/satman/mgmt.rs index 2b44bc4bb..ad0526b0d 100644 --- a/artiq/firmware/satman/mgmt.rs +++ b/artiq/firmware/satman/mgmt.rs @@ -31,7 +31,7 @@ impl Manager { } pub fn get_config_value_slice(&mut self, data_slice: &mut [u8; SAT_PAYLOAD_MAX_SIZE]) -> SliceMeta { - self.last_value.get_slice_sat(data_slice) + self.last_value.get_slice_satellite(data_slice) } pub fn add_config_data(&mut self, data: &[u8], data_len: usize) { diff --git a/artiq/firmware/satman/routing.rs b/artiq/firmware/satman/routing.rs index f8e0e6d24..2861f9fe6 100644 --- a/artiq/firmware/satman/routing.rs +++ b/artiq/firmware/satman/routing.rs @@ -4,6 +4,7 @@ use board_artiq::{drtioaux, drtio_routing}; use board_misoc::csr; use core::cmp::min; use proto_artiq::drtioaux_proto::PayloadStatus; +use SAT_PAYLOAD_MAX_SIZE; use MASTER_PAYLOAD_MAX_SIZE; /* represents data that has to be sent with the aux protocol */ @@ -56,6 +57,7 @@ impl Sliceable { self.data.extend(data); } + get_slice_fn!(get_slice_satellite, SAT_PAYLOAD_MAX_SIZE); get_slice_fn!(get_slice_master, MASTER_PAYLOAD_MAX_SIZE); } From 28654501af376b940d9968649bf843df75edeaf2 Mon Sep 17 00:00:00 2001 From: occheung Date: Thu, 19 Sep 2024 17:18:57 +0800 Subject: [PATCH 077/111] runtime mgmt: minor fix --- artiq/firmware/runtime/mgmt.rs | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/artiq/firmware/runtime/mgmt.rs b/artiq/firmware/runtime/mgmt.rs index 436d1d22c..480443ba0 100644 --- a/artiq/firmware/runtime/mgmt.rs +++ b/artiq/firmware/runtime/mgmt.rs @@ -1,11 +1,8 @@ -use log::{self, LevelFilter}; -use core::cell::Cell; -use core::cell::RefCell; +use core::cell::{Cell, RefCell}; use board_artiq::drtio_routing::RoutingTable; use io::{ProtoRead, Write, Error as IoError}; use mgmt_proto::*; -use sched::{Io, TcpListener, TcpStream, Error as SchedError}; use sched::{Io, Mutex, TcpListener, TcpStream, Error as SchedError}; use urc::Urc; @@ -102,7 +99,7 @@ mod local_coremgmt { Ok(()) } - pub fn config_write(_io: &Io, stream: &mut TcpStream, key: &String, value: &Vec, restart_idle: &Urc>) -> Result<(), Error> { + pub fn config_write(io: &Io, stream: &mut TcpStream, key: &String, value: &Vec, restart_idle: &Urc>) -> Result<(), Error> { match config::write(key, value) { Ok(_) => { if key == "idle_kernel" { @@ -116,7 +113,7 @@ mod local_coremgmt { Ok(()) } - pub fn config_remove(_io: &Io, stream: &mut TcpStream, key: &String, restart_idle: &Urc>) -> Result<(), Error> { + pub fn config_remove(io: &Io, stream: &mut TcpStream, key: &String, restart_idle: &Urc>) -> Result<(), Error> { match config::remove(key) { Ok(()) => { if key == "idle_kernel" { @@ -130,7 +127,7 @@ mod local_coremgmt { Ok(()) } - pub fn config_erase(_io: &Io, stream: &mut TcpStream, restart_idle: &Urc>) -> Result<(), Error> { + pub fn config_erase(io: &Io, stream: &mut TcpStream, restart_idle: &Urc>) -> Result<(), Error> { match config::erase() { Ok(()) => { io.until(|| !restart_idle.get())?; @@ -603,7 +600,7 @@ macro_rules! process { fn worker(io: &Io, stream: &mut TcpStream, restart_idle: &Urc>, _aux_mutex: &Mutex, _ddma_mutex: &Mutex, _subkernel_mutex: &Mutex, - _routing_table: &RoutingTable, stream: &mut TcpStream) -> Result<(), Error> { + _routing_table: &RoutingTable) -> Result<(), Error> { read_magic(stream)?; let _destination = stream.read_u8()?; Write::write_all(stream, "e".as_bytes())?; From 6b6bcdb6d6dfacc662968cabced79a9efcd0f635 Mon Sep 17 00:00:00 2001 From: occheung Date: Thu, 19 Sep 2024 17:42:03 +0800 Subject: [PATCH 078/111] runtime mgmt: avoid passing incomplete message to corelog --- artiq/firmware/runtime/mgmt.rs | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/artiq/firmware/runtime/mgmt.rs b/artiq/firmware/runtime/mgmt.rs index 480443ba0..fd39be059 100644 --- a/artiq/firmware/runtime/mgmt.rs +++ b/artiq/firmware/runtime/mgmt.rs @@ -269,14 +269,20 @@ mod remote_coremgmt { ddma_mutex: &Mutex, subkernel_mutex: &Mutex, routing_table: &RoutingTable, linkno: u8, destination: u8, stream: &mut TcpStream) -> Result<(), Error> { + let mut buffer = Vec::new(); loop { let reply = drtio::aux_transact(io, aux_mutex, ddma_mutex, subkernel_mutex, routing_table, linkno, &Packet::CoreMgmtGetLogRequest { destination, clear: true } ); match reply { - Ok(Packet::CoreMgmtGetLogReply { last: _, length, data }) => { - stream.write_bytes(&data[..length as usize])?; + Ok(Packet::CoreMgmtGetLogReply { last, length, data }) => { + buffer.extend(&data[..length as usize]); + + if last { + stream.write_bytes(&buffer[..length as usize])?; + buffer.clear(); + } } Ok(packet) => { error!("received unexpected aux packet: {:?}", packet); From 9d3204d0193de6f8bb8091687285ccad56303537 Mon Sep 17 00:00:00 2001 From: occheung Date: Fri, 20 Sep 2024 11:46:33 +0800 Subject: [PATCH 079/111] runtim mgmt: fix pull log message relaying --- artiq/firmware/runtime/mgmt.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/artiq/firmware/runtime/mgmt.rs b/artiq/firmware/runtime/mgmt.rs index fd39be059..5527edb3b 100644 --- a/artiq/firmware/runtime/mgmt.rs +++ b/artiq/firmware/runtime/mgmt.rs @@ -280,7 +280,7 @@ mod remote_coremgmt { buffer.extend(&data[..length as usize]); if last { - stream.write_bytes(&buffer[..length as usize])?; + stream.write_bytes(&buffer)?; buffer.clear(); } } From 2e9622b9d6738c8edcdf0172638c8b0ebb81c8e3 Mon Sep 17 00:00:00 2001 From: occheung Date: Fri, 20 Sep 2024 11:47:05 +0800 Subject: [PATCH 080/111] update RELEASE_NOTES --- RELEASE_NOTES.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/RELEASE_NOTES.rst b/RELEASE_NOTES.rst index 7d1c160c8..3e4d3c4b4 100644 --- a/RELEASE_NOTES.rst +++ b/RELEASE_NOTES.rst @@ -22,6 +22,8 @@ ARTIQ-9 (Unreleased) * Idle kernels now restart when written with ``artiq_coremgmt`` and stop when erased/removed from config. * New support for the EBAZ4205 Zynq-SoC control card. * New core device driver for the AD9834 DDS, tested with the ZonRi Technology Co., Ltd. AD9834-Module. +* Support for coredevice reflashing through new the ``flash`` tool in ``artiq_coremgmt``. +* ``artiq_coremgmt`` now supports configuring satellites. ARTIQ-8 ------- From cc40313501eb182cb2ecc1d109feaa3458893606 Mon Sep 17 00:00:00 2001 From: occheung Date: Fri, 20 Sep 2024 17:38:14 +0800 Subject: [PATCH 081/111] flake8 --- artiq/coredevice/comm_mgmt.py | 3 +- artiq/frontend/artiq_coremgmt.py | 51 +++++++++++++++++++------------- 2 files changed, 33 insertions(+), 21 deletions(-) diff --git a/artiq/coredevice/comm_mgmt.py b/artiq/coredevice/comm_mgmt.py index c9f466b59..c77d0b9ec 100644 --- a/artiq/coredevice/comm_mgmt.py +++ b/artiq/coredevice/comm_mgmt.py @@ -209,7 +209,8 @@ class CommMgmt: with open(filename, "rb") as fi: bin_ = fi.read() if (len(bin_paths) > 1): - image_buf.write(struct.pack(self.endian + "I", len(bin_))) + image_buf.write( + struct.pack(self.endian + "I", len(bin_))) image_buf.write(bin_) crc = binascii.crc32(image_buf.getvalue()) diff --git a/artiq/frontend/artiq_coremgmt.py b/artiq/frontend/artiq_coremgmt.py index e9cada16f..c1b56d2a8 100755 --- a/artiq/frontend/artiq_coremgmt.py +++ b/artiq/frontend/artiq_coremgmt.py @@ -13,7 +13,6 @@ from artiq.master.databases import DeviceDB from artiq.coredevice.comm_kernel import CommKernel from artiq.coredevice.comm_mgmt import CommMgmt from artiq.frontend.bit2bin import bit2bin -from misoc.tools.mkmscimg import insert_crc def get_argparser(): @@ -95,14 +94,18 @@ def get_argparser(): help="flash the running system") p_zynq = t_flash.add_argument("-z", "--zynq", default=False, - help="target zynq device", action="store_true") + help="target zynq device", + action="store_true") - p_directory = t_flash.add_argument("directory", metavar="DIRECTORY", type=str, - help="directory that contains the binaries") + p_directory = t_flash.add_argument("directory", + metavar="DIRECTORY", type=str, + help="directory that contains the " + "binaries") p_srcbuild = t_flash.add_argument("--srcbuild", - help="board binaries directory is laid out as a source build tree", - default=False, action="store_true") + help="board binaries directory is laid " + "out as a source build tree", + default=False, action="store_true") # misc debug t_debug = tools.add_parser("debug", @@ -116,8 +119,9 @@ def get_argparser(): # manage target p_drtio_dest = parser.add_argument("-s", "--satellite", default=0, - metavar="DRTIO ID", type=int, - help="specify DRTIO destination that receives this command") + metavar="DRTIO ID", type=int, + help="specify DRTIO destination that " + "receives this command") return parser @@ -161,46 +165,53 @@ def main(): mgmt.config_remove(key) if args.action == "erase": mgmt.config_erase() - + if args.tool == "flash": if args.zynq: boot = os.path.join(args.directory, "boot.bin") - bins = [ boot ] + bins = [boot] else: def artifact_path(this_binary_dir, *path_filename): if args.srcbuild: # source tree - use path elements to locate file return os.path.join(this_binary_dir, *path_filename) else: - # flat tree - all files in the same directory, discard path elements + # flat tree + # all files in the same directory, discard path elements *_, filename = path_filename return os.path.join(this_binary_dir, filename) def convert_gateware(bit_filename): bin_handle, bin_filename = tempfile.mkstemp( - prefix="artiq_", suffix="_" + os.path.basename(bit_filename)) - with open(bit_filename, "rb") as bit_file, open(bin_handle, "wb") as bin_file: + prefix="artiq_", + suffix="_" + os.path.basename(bit_filename)) + with open(bit_filename, "rb") as bit_file, \ + open(bin_handle, "wb") as bin_file: bit2bin(bit_file, bin_file) atexit.register(lambda: os.unlink(bin_filename)) return bin_filename - gateware = convert_gateware( - artifact_path(args.directory, "gateware", "top.bit")) - bootloader = artifact_path(args.directory, "software", "bootloader", "bootloader.bin") + gateware = convert_gateware(artifact_path( + args.directory, "gateware", "top.bit")) + bootloader = artifact_path( + args.directory, "software", "bootloader", "bootloader.bin") firmwares = [] for firmware in "satman", "runtime": - filename = artifact_path(args.directory, "software", firmware, firmware + ".fbi") + filename = artifact_path( + args.directory, "software", firmware, firmware + ".fbi") if os.path.exists(filename): firmwares.append(filename) if not firmwares: raise FileNotFoundError("no firmware found") if len(firmwares) > 1: - raise ValueError("more than one firmware file, please clean up your build directory. " - "Found firmware files: {}".format(" ".join(firmwares))) + raise ValueError("more than one firmware file, " + "please clean up your build directory. " + "Found firmware files: {}".format( + " ".join(firmwares))) firmware = firmwares[0] - bins = [ gateware, bootloader, firmware ] + bins = [gateware, bootloader, firmware] mgmt.flash(bins) From ea61d9bd39ece543bc2fa570193964ae66d5763b Mon Sep 17 00:00:00 2001 From: occheung Date: Wed, 2 Oct 2024 10:56:48 +0800 Subject: [PATCH 082/111] satman: uart_logger -> buffer_logger --- artiq/firmware/Cargo.lock | 1 + artiq/firmware/satman/Cargo.toml | 1 + artiq/firmware/satman/main.rs | 37 +++++++++++++++++++++++++++++--- 3 files changed, 36 insertions(+), 3 deletions(-) diff --git a/artiq/firmware/Cargo.lock b/artiq/firmware/Cargo.lock index aa156989e..98d7d730f 100644 --- a/artiq/firmware/Cargo.lock +++ b/artiq/firmware/Cargo.lock @@ -560,6 +560,7 @@ dependencies = [ "eh", "io", "log", + "logger_artiq", "proto_artiq", "riscv", ] diff --git a/artiq/firmware/satman/Cargo.toml b/artiq/firmware/satman/Cargo.toml index 522ae7b32..dca0c91f9 100644 --- a/artiq/firmware/satman/Cargo.toml +++ b/artiq/firmware/satman/Cargo.toml @@ -20,6 +20,7 @@ crc = { version = "1.7", default-features = false } cslice = { version = "0.3" } board_misoc = { path = "../libboard_misoc", features = ["uart_console", "log"] } board_artiq = { path = "../libboard_artiq", features = ["alloc"] } +logger_artiq = { path = "../liblogger_artiq" } alloc_list = { path = "../liballoc_list" } riscv = { version = "0.6.0", features = ["inline-asm"] } proto_artiq = { path = "../libproto_artiq", features = ["log", "alloc"] } diff --git a/artiq/firmware/satman/main.rs b/artiq/firmware/satman/main.rs index 3909ce6d3..55cf7b65f 100644 --- a/artiq/firmware/satman/main.rs +++ b/artiq/firmware/satman/main.rs @@ -6,6 +6,7 @@ extern crate log; #[macro_use] extern crate board_misoc; extern crate board_artiq; +extern crate logger_artiq; extern crate riscv; extern crate alloc; extern crate proto_artiq; @@ -16,14 +17,14 @@ extern crate io; extern crate eh; use core::convert::TryFrom; -use board_misoc::{csr, ident, clock, config, uart_logger, i2c, pmp}; +use board_misoc::{csr, ident, clock, config, i2c, pmp}; #[cfg(has_si5324)] use board_artiq::si5324; #[cfg(has_si549)] use board_artiq::si549; #[cfg(soc_platform = "kasli")] use board_misoc::irq; -use board_misoc::spiflash; +use board_misoc::{boot, spiflash}; use board_artiq::{spi, drtioaux, drtio_routing}; #[cfg(soc_platform = "efc")] use board_artiq::ad9117; @@ -793,6 +794,27 @@ fn sysclk_setup() { } } +fn setup_log_levels() { + match config::read_str("log_level", |r| r.map(|s| s.parse())) { + Ok(Ok(log_level_filter)) => { + info!("log level set to {} by `log_level` config key", + log_level_filter); + log::set_max_level(log_level_filter); + } + _ => info!("log level set to INFO by default") + } + match config::read_str("uart_log_level", |r| r.map(|s| s.parse())) { + Ok(Ok(uart_log_level_filter)) => { + info!("UART log level set to {} by `uart_log_level` config key", + uart_log_level_filter); + logger_artiq::BufferLogger::with(|logger| + logger.set_uart_log_level(uart_log_level_filter)); + } + _ => info!("UART log level set to INFO by default") + } +} + +static mut LOG_BUFFER: [u8; 1<<17] = [0; 1<<17]; #[no_mangle] pub extern fn main() -> i32 { @@ -812,12 +834,21 @@ pub extern fn main() -> i32 { irq::enable(csr::WRPLL_INTERRUPT); clock::init(); - uart_logger::ConsoleLogger::register(); + unsafe { + logger_artiq::BufferLogger::new(&mut LOG_BUFFER[..]).register(|| + boot::start_user(startup as usize)); + } + 0 +} + +fn startup() { info!("ARTIQ satellite manager starting..."); info!("software ident {}", csr::CONFIG_IDENTIFIER_STR); info!("gateware ident {}", ident::read(&mut [0; 64])); + setup_log_levels(); + #[cfg(has_i2c)] i2c::init().expect("I2C initialization failed"); #[cfg(all(soc_platform = "kasli", hw_rev = "v2.0"))] From ffcf79b74e62b81d18ec9a7f10e22e398ff1d6d0 Mon Sep 17 00:00:00 2001 From: occheung Date: Wed, 2 Oct 2024 10:58:13 +0800 Subject: [PATCH 083/111] satman: support coremgmt logging --- artiq/firmware/satman/main.rs | 47 +++++++++++++++++++++++++++++------ artiq/firmware/satman/mgmt.rs | 42 +++++++++++++++++++++++++++++++ 2 files changed, 81 insertions(+), 8 deletions(-) diff --git a/artiq/firmware/satman/main.rs b/artiq/firmware/satman/main.rs index 55cf7b65f..ca0253e04 100644 --- a/artiq/firmware/satman/main.rs +++ b/artiq/firmware/satman/main.rs @@ -501,19 +501,50 @@ fn process_aux_packet(dmamgr: &mut DmaManager, analyzer: &mut Analyzer, kernelmg Ok(()) } - drtioaux::Packet::CoreMgmtGetLogRequest { destination: _destination, .. } | - drtioaux::Packet::CoreMgmtClearLogRequest { destination: _destination } | - drtioaux::Packet::CoreMgmtSetLogLevelRequest {destination: _destination, .. } => { + drtioaux::Packet::CoreMgmtGetLogRequest { destination: _destination, clear } => { forward!(router, _routing_table, _destination, *rank, *self_destination, _repeaters, &packet); - error!("RISC-V satellite devices do not support buffered logging"); - drtioaux::send(0, &drtioaux::Packet::CoreMgmtReply { succeeded: false }) + let mut data_slice = [0; SAT_PAYLOAD_MAX_SIZE]; + if let Ok(meta) = coremgr.log_get_slice(&mut data_slice, clear) { + drtioaux::send( + 0, + &drtioaux::Packet::CoreMgmtGetLogReply { + last: meta.status.is_last(), + length: meta.len as u16, + data: data_slice, + }, + ) + } else { + drtioaux::send(0, &drtioaux::Packet::CoreMgmtReply { succeeded: false }) + } } - drtioaux::Packet::CoreMgmtSetUartLogLevelRequest { destination: _destination, .. } => { + drtioaux::Packet::CoreMgmtClearLogRequest { destination: _destination } => { forward!(router, _routing_table, _destination, *rank, *self_destination, _repeaters, &packet); - error!("RISC-V satellite devices has fixed UART log level fixed at TRACE"); - drtioaux::send(0, &drtioaux::Packet::CoreMgmtReply { succeeded: false }) + drtioaux::send(0, &drtioaux::Packet::CoreMgmtReply { succeeded: mgmt::clear_log().is_ok() }) + } + drtioaux::Packet::CoreMgmtSetLogLevelRequest {destination: _destination, log_level } => { + forward!(router, _routing_table, _destination, *rank, *self_destination, _repeaters, &packet); + + if let Ok(level_filter) = mgmt::byte_to_level_filter(log_level) { + info!("changing log level to {}", level_filter); + log::set_max_level(level_filter); + drtioaux::send(0, &drtioaux::Packet::CoreMgmtReply { succeeded: true }) + } else { + drtioaux::send(0, &drtioaux::Packet::CoreMgmtReply { succeeded: false }) + } + } + drtioaux::Packet::CoreMgmtSetUartLogLevelRequest { destination: _destination, log_level } => { + forward!(router, _routing_table, _destination, *rank, *self_destination, _repeaters, &packet); + + if let Ok(level_filter) = mgmt::byte_to_level_filter(log_level) { + info!("changing UART log level to {}", level_filter); + logger_artiq::BufferLogger::with(|logger| + logger.set_uart_log_level(level_filter)); + drtioaux::send(0, &drtioaux::Packet::CoreMgmtReply { succeeded: true }) + } else { + drtioaux::send(0, &drtioaux::Packet::CoreMgmtReply { succeeded: false }) + } } drtioaux::Packet::CoreMgmtConfigReadRequest { destination: _destination, diff --git a/artiq/firmware/satman/mgmt.rs b/artiq/firmware/satman/mgmt.rs index ad0526b0d..9ad6f203a 100644 --- a/artiq/firmware/satman/mgmt.rs +++ b/artiq/firmware/satman/mgmt.rs @@ -5,14 +5,39 @@ use crc::crc32; use routing::{Sliceable, SliceMeta}; use board_artiq::drtioaux; use board_misoc::{mem, config, spiflash}; +use log::LevelFilter; +use logger_artiq::BufferLogger; use io::{Cursor, ProtoRead, ProtoWrite}; use proto_artiq::drtioaux_proto::SAT_PAYLOAD_MAX_SIZE; +pub fn clear_log() -> Result<(), ()> { + BufferLogger::with(|logger| { + let mut buffer = logger.buffer()?; + Ok(buffer.clear()) + }).map_err(|()| error!("error on clearing log buffer")) +} + +pub fn byte_to_level_filter(level_byte: u8) -> Result { + Ok(match level_byte { + 0 => LevelFilter::Off, + 1 => LevelFilter::Error, + 2 => LevelFilter::Warn, + 3 => LevelFilter::Info, + 4 => LevelFilter::Debug, + 5 => LevelFilter::Trace, + lv => { + error!("unknown log level: {}", lv); + return Err(()); + } + }) +} + pub struct Manager { config_payload: Cursor>, image_payload: Cursor>, last_value: Sliceable, + last_log: Sliceable, } impl Manager { @@ -21,6 +46,7 @@ impl Manager { config_payload: Cursor::new(Vec::new()), image_payload: Cursor::new(Vec::new()), last_value: Sliceable::new(0, Vec::new()), + last_log: Sliceable::new(0, Vec::new()), } } @@ -30,6 +56,22 @@ impl Manager { )).map_err(|_err| warn!("read error: no such key")) } + pub fn log_get_slice(&mut self, data_slice: &mut [u8; SAT_PAYLOAD_MAX_SIZE], consume: bool) -> Result { + // Populate buffer if depleted + if self.last_log.at_end() { + BufferLogger::with(|logger| { + let mut buffer = logger.buffer()?; + self.last_log = Sliceable::new(0, buffer.extract().as_bytes().to_vec()); + if consume { + buffer.clear(); + } + Ok(()) + }).map_err(|()| error!("error on getting log buffer"))?; + } + + Ok(self.last_log.get_slice_satellite(data_slice)) + } + pub fn get_config_value_slice(&mut self, data_slice: &mut [u8; SAT_PAYLOAD_MAX_SIZE]) -> SliceMeta { self.last_value.get_slice_satellite(data_slice) } From 9882e16216b03eeaf27ff97545aa5294d306e33a Mon Sep 17 00:00:00 2001 From: occheung Date: Wed, 2 Oct 2024 11:00:04 +0800 Subject: [PATCH 084/111] RELEASE_NOTES: fix typo --- RELEASE_NOTES.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/RELEASE_NOTES.rst b/RELEASE_NOTES.rst index 3e4d3c4b4..658322357 100644 --- a/RELEASE_NOTES.rst +++ b/RELEASE_NOTES.rst @@ -22,7 +22,7 @@ ARTIQ-9 (Unreleased) * Idle kernels now restart when written with ``artiq_coremgmt`` and stop when erased/removed from config. * New support for the EBAZ4205 Zynq-SoC control card. * New core device driver for the AD9834 DDS, tested with the ZonRi Technology Co., Ltd. AD9834-Module. -* Support for coredevice reflashing through new the ``flash`` tool in ``artiq_coremgmt``. +* Support for coredevice reflashing through the new ``flash`` tool in ``artiq_coremgmt``. * ``artiq_coremgmt`` now supports configuring satellites. ARTIQ-8 From 21944ff865fb2b27531c21e29bc044d7ebc7009a Mon Sep 17 00:00:00 2001 From: occheung Date: Wed, 2 Oct 2024 11:22:46 +0800 Subject: [PATCH 085/111] coremgmt flash: update crc mismatch message --- artiq/firmware/runtime/mgmt.rs | 2 +- artiq/firmware/satman/mgmt.rs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/artiq/firmware/runtime/mgmt.rs b/artiq/firmware/runtime/mgmt.rs index 5527edb3b..c411519df 100644 --- a/artiq/firmware/runtime/mgmt.rs +++ b/artiq/firmware/runtime/mgmt.rs @@ -181,7 +181,7 @@ mod local_coremgmt { reboot(_io, stream)?; } else { - error!("CRC failed in SDRAM (actual {:08x}, expected {:08x})", actual_crc, expected_crc); + error!("CRC failed, images have not been written to flash.\n(actual {:08x}, expected {:08x})", actual_crc, expected_crc); Reply::Error.write_to(stream)?; } Ok(()) diff --git a/artiq/firmware/satman/mgmt.rs b/artiq/firmware/satman/mgmt.rs index 9ad6f203a..338b1c26f 100644 --- a/artiq/firmware/satman/mgmt.rs +++ b/artiq/firmware/satman/mgmt.rs @@ -139,7 +139,7 @@ impl Manager { } } else { - panic!("CRC failed in SDRAM (actual {:08x}, expected {:08x})", actual_crc, expected_crc); + panic!("CRC failed, images have not been written to flash.\n(actual {:08x}, expected {:08x})", actual_crc, expected_crc); } } } From e8bd99048ed8df389d2c71ffbc65d3f3bb84b08e Mon Sep 17 00:00:00 2001 From: occheung Date: Wed, 2 Oct 2024 11:40:30 +0800 Subject: [PATCH 086/111] coremgmt frontend: fix arg name --- artiq/frontend/aqctl_corelog.py | 6 +++--- artiq/frontend/artiq_coremgmt.py | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/artiq/frontend/aqctl_corelog.py b/artiq/frontend/aqctl_corelog.py index d2dd33a4a..49f5b2565 100755 --- a/artiq/frontend/aqctl_corelog.py +++ b/artiq/frontend/aqctl_corelog.py @@ -25,9 +25,9 @@ def get_argparser(): help="Simulation - does not connect to device") parser.add_argument("core_addr", metavar="CORE_ADDR", help="hostname or IP address of the core device") - parser.add_argument("-s", "--satellite", default=0, - metavar="DRTIO_ID", type=int, - help="the logged DRTIO destination") + parser.add_argument("-s", "--drtio-dest", default=0, + metavar="DRTIO_DEST", type=int, + help="specifies the DRTIO destination") return parser diff --git a/artiq/frontend/artiq_coremgmt.py b/artiq/frontend/artiq_coremgmt.py index c1b56d2a8..e5ccd7215 100755 --- a/artiq/frontend/artiq_coremgmt.py +++ b/artiq/frontend/artiq_coremgmt.py @@ -118,8 +118,8 @@ def get_argparser(): help="show heap layout") # manage target - p_drtio_dest = parser.add_argument("-s", "--satellite", default=0, - metavar="DRTIO ID", type=int, + p_drtio_dest = parser.add_argument("-s", "--drtio-dest", default=0, + metavar="DRTIO_DEST", type=int, help="specify DRTIO destination that " "receives this command") From f5ff90809805b078c4d4887bf96c5f5d0f690641 Mon Sep 17 00:00:00 2001 From: occheung Date: Wed, 2 Oct 2024 15:51:02 +0800 Subject: [PATCH 087/111] artiq_flash/coremgmt flash -> fetch_bin --- artiq/frontend/artiq_coremgmt.py | 44 +++--------------------- artiq/frontend/artiq_flash.py | 36 +++---------------- artiq/frontend/fetch_bin.py | 59 ++++++++++++++++++++++++++++++++ 3 files changed, 68 insertions(+), 71 deletions(-) create mode 100644 artiq/frontend/fetch_bin.py diff --git a/artiq/frontend/artiq_coremgmt.py b/artiq/frontend/artiq_coremgmt.py index e5ccd7215..d2163a5bc 100755 --- a/artiq/frontend/artiq_coremgmt.py +++ b/artiq/frontend/artiq_coremgmt.py @@ -13,6 +13,7 @@ from artiq.master.databases import DeviceDB from artiq.coredevice.comm_kernel import CommKernel from artiq.coredevice.comm_mgmt import CommMgmt from artiq.frontend.bit2bin import bit2bin +from artiq.frontend.fetch_bin import fetch_bin def get_argparser(): @@ -171,46 +172,9 @@ def main(): boot = os.path.join(args.directory, "boot.bin") bins = [boot] else: - def artifact_path(this_binary_dir, *path_filename): - if args.srcbuild: - # source tree - use path elements to locate file - return os.path.join(this_binary_dir, *path_filename) - else: - # flat tree - # all files in the same directory, discard path elements - *_, filename = path_filename - return os.path.join(this_binary_dir, filename) - - def convert_gateware(bit_filename): - bin_handle, bin_filename = tempfile.mkstemp( - prefix="artiq_", - suffix="_" + os.path.basename(bit_filename)) - with open(bit_filename, "rb") as bit_file, \ - open(bin_handle, "wb") as bin_file: - bit2bin(bit_file, bin_file) - atexit.register(lambda: os.unlink(bin_filename)) - return bin_filename - - gateware = convert_gateware(artifact_path( - args.directory, "gateware", "top.bit")) - bootloader = artifact_path( - args.directory, "software", "bootloader", "bootloader.bin") - - firmwares = [] - for firmware in "satman", "runtime": - filename = artifact_path( - args.directory, "software", firmware, firmware + ".fbi") - if os.path.exists(filename): - firmwares.append(filename) - if not firmwares: - raise FileNotFoundError("no firmware found") - if len(firmwares) > 1: - raise ValueError("more than one firmware file, " - "please clean up your build directory. " - "Found firmware files: {}".format( - " ".join(firmwares))) - firmware = firmwares[0] - + gateware = fetch_bin(args.directory, "gateware", args.srcbuild) + bootloader = fetch_bin(args.directory, "bootloader", args.srcbuild) + firmware = fetch_bin(args.directory, ["runtime", "satman"], args.srcbuild) bins = [gateware, bootloader, firmware] mgmt.flash(bins) diff --git a/artiq/frontend/artiq_flash.py b/artiq/frontend/artiq_flash.py index b1914a4b5..b9563a339 100755 --- a/artiq/frontend/artiq_flash.py +++ b/artiq/frontend/artiq_flash.py @@ -15,6 +15,7 @@ from sipyco import common_args from artiq import __version__ as artiq_version from artiq.remoting import SSHClient, LocalClient from artiq.frontend.bit2bin import bit2bin +from artiq.frontend.fetch_bin import fetch_bin def get_argparser(): @@ -302,46 +303,19 @@ def main(): programmer = config["programmer"](client, preinit_script=args.preinit_command) - def artifact_path(this_binary_dir, *path_filename): - if args.srcbuild: - # source tree - use path elements to locate file - return os.path.join(this_binary_dir, *path_filename) - else: - # flat tree - all files in the same directory, discard path elements - *_, filename = path_filename - return os.path.join(this_binary_dir, filename) - - def convert_gateware(bit_filename): - bin_handle, bin_filename = tempfile.mkstemp( - prefix="artiq_", suffix="_" + os.path.basename(bit_filename)) - with open(bit_filename, "rb") as bit_file, open(bin_handle, "wb") as bin_file: - bit2bin(bit_file, bin_file) - atexit.register(lambda: os.unlink(bin_filename)) - return bin_filename - for action in args.action: if action == "gateware": - gateware_bin = convert_gateware( - artifact_path(binary_dir, "gateware", "top.bit")) + gateware_bin = fetch_bin(binary_dir, "gateware", args.srcbuild) programmer.write_binary(*config["gateware"], gateware_bin) elif action == "bootloader": - bootloader_bin = artifact_path(binary_dir, "software", "bootloader", "bootloader.bin") + bootloader_bin = fetch_bin(binary_dir, "bootloader", args.srcbuild) programmer.write_binary(*config["bootloader"], bootloader_bin) elif action == "storage": storage_img = args.storage programmer.write_binary(*config["storage"], storage_img) elif action == "firmware": - firmware_fbis = [] - for firmware in "satman", "runtime": - filename = artifact_path(binary_dir, "software", firmware, firmware + ".fbi") - if os.path.exists(filename): - firmware_fbis.append(filename) - if not firmware_fbis: - raise FileNotFoundError("no firmware found") - if len(firmware_fbis) > 1: - raise ValueError("more than one firmware file, please clean up your build directory. " - "Found firmware files: {}".format(" ".join(firmware_fbis))) - programmer.write_binary(*config["firmware"], firmware_fbis[0]) + firmware_fbi = fetch_bin(binary_dir, ["satman", "runtime"], args.srcbuild) + programmer.write_binary(*config["firmware"], firmware_fbi) elif action == "load": gateware_bit = artifact_path(binary_dir, "gateware", "top.bit") programmer.load(gateware_bit, 0) diff --git a/artiq/frontend/fetch_bin.py b/artiq/frontend/fetch_bin.py new file mode 100644 index 000000000..890c3413c --- /dev/null +++ b/artiq/frontend/fetch_bin.py @@ -0,0 +1,59 @@ +import atexit +import os +import tempfile + +from artiq.frontend.bit2bin import bit2bin + + +def fetch_bin(binary_dir, component, srcbuild=False): + def artifact_path(this_binary_dir, *path_filename): + if srcbuild: + # source tree - use path elements to locate file + return os.path.join(this_binary_dir, *path_filename) + else: + # flat tree - all files in the same directory, discard path elements + *_, filename = path_filename + return os.path.join(this_binary_dir, filename) + + def convert_gateware(bit_filename): + bin_handle, bin_filename = tempfile.mkstemp( + prefix="artiq_", suffix="_" + os.path.basename(bit_filename)) + with open(bit_filename, "rb") as bit_file, open(bin_handle, "wb") as bin_file: + bit2bin(bit_file, bin_file) + atexit.register(lambda: os.unlink(bin_filename)) + return bin_filename + + if type(component) == list: + bins = [] + for option in component: + try: + bins.append(fetch_bin(binary_dir, option, srcbuild)) + except FileNotFoundError: + pass + + if bins is None: + raise FileNotFoundError("multiple components not found: {}".format( + " ".join(component))) + + if len(bins) > 1: + raise ValueError("more than one file, " + "please clean up your build directory. " + "Found files: {}".format( + " ".join(bins))) + + return bins[0] + + path = artifact_path(binary_dir, *{ + "gateware": ["gateware", "top.bit"], + "bootloader": ["software", "bootloader", "bootloader.bin"], + "runtime": ["software", "runtime", "runtime.fbi"], + "satman": ["software", "satman", "satman.fbi"], + }[component]) + + if not os.path.exists(path): + raise FileNotFoundError("{} not found".format(component)) + + if component == "gateware": + path = convert_gateware(path) + + return path From a643da2c5edef5e83aef000a5371410f4c3e9fe9 Mon Sep 17 00:00:00 2001 From: occheung Date: Wed, 2 Oct 2024 15:51:43 +0800 Subject: [PATCH 088/111] fix frontend args reference --- artiq/frontend/aqctl_corelog.py | 2 +- artiq/frontend/artiq_coremgmt.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/artiq/frontend/aqctl_corelog.py b/artiq/frontend/aqctl_corelog.py index 49f5b2565..cd954f590 100755 --- a/artiq/frontend/aqctl_corelog.py +++ b/artiq/frontend/aqctl_corelog.py @@ -100,7 +100,7 @@ def main(): signal_handler.setup() try: get_logs_task = asyncio.ensure_future( - get_logs_sim(args.core_addr) if args.simulation else get_logs(args.core_addr, args.satellite), + get_logs_sim(args.core_addr) if args.simulation else get_logs(args.core_addr, args.drtio_dest), loop=loop) try: server = Server({"corelog": PingTarget()}, None, True) diff --git a/artiq/frontend/artiq_coremgmt.py b/artiq/frontend/artiq_coremgmt.py index d2163a5bc..903a5615f 100755 --- a/artiq/frontend/artiq_coremgmt.py +++ b/artiq/frontend/artiq_coremgmt.py @@ -136,7 +136,7 @@ def main(): core_addr = ddb.get("core", resolve_alias=True)["arguments"]["host"] else: core_addr = args.device - mgmt = CommMgmt(core_addr, drtio_dest=args.satellite) + mgmt = CommMgmt(core_addr, drtio_dest=args.drtio_dest) if args.tool == "log": if args.action == "set_level": From 1583debfe7a144b48c1c3eabff88d92efad52252 Mon Sep 17 00:00:00 2001 From: occheung Date: Thu, 24 Oct 2024 12:35:10 +0800 Subject: [PATCH 089/111] feature gate txenable with has_drtio_eem --- artiq/firmware/satman/main.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/artiq/firmware/satman/main.rs b/artiq/firmware/satman/main.rs index ca0253e04..b5dfff323 100644 --- a/artiq/firmware/satman/main.rs +++ b/artiq/firmware/satman/main.rs @@ -641,7 +641,7 @@ fn process_aux_packet(dmamgr: &mut DmaManager, analyzer: &mut Analyzer, kernelmg drtioaux::Packet::CoreMgmtDropLinkAck { destination: _destination } => { forward!(router, _routing_table, _destination, *rank, *self_destination, _repeaters, &packet); - #[cfg(not(soc_platform = "efc"))] + #[cfg(not(has_drtio_eem))] unsafe { csr::gt_drtio::txenable_write(0); } From f2c13a504114c712a78af80f9f8a8e176e744f8e Mon Sep 17 00:00:00 2001 From: occheung Date: Thu, 24 Oct 2024 12:36:43 +0800 Subject: [PATCH 090/111] drtio_proto: add allocate step for flashing This avoids reallocation when transfering binaries. Reallocation during flash handshakes could cause DRTIO timeouts. --- .../firmware/libproto_artiq/drtioaux_proto.rs | 38 ++++++++++++------- artiq/firmware/runtime/mgmt.rs | 22 ++++++++++- artiq/firmware/satman/main.rs | 8 +++- artiq/firmware/satman/mgmt.rs | 4 ++ 4 files changed, 56 insertions(+), 16 deletions(-) diff --git a/artiq/firmware/libproto_artiq/drtioaux_proto.rs b/artiq/firmware/libproto_artiq/drtioaux_proto.rs index 9a0f423dc..4891bedf1 100644 --- a/artiq/firmware/libproto_artiq/drtioaux_proto.rs +++ b/artiq/firmware/libproto_artiq/drtioaux_proto.rs @@ -139,7 +139,8 @@ pub enum Packet { CoreMgmtConfigEraseRequest { destination: u8 }, CoreMgmtRebootRequest { destination: u8 }, CoreMgmtAllocatorDebugRequest { destination: u8 }, - CoreMgmtFlashRequest { destination: u8, last: bool, length: u16, data: [u8; MASTER_PAYLOAD_MAX_SIZE] }, + CoreMgmtFlashRequest { destination: u8, payload_length: u32 }, + CoreMgmtFlashAddDataRequest { destination: u8, last: bool, length: u16, data: [u8; MASTER_PAYLOAD_MAX_SIZE] }, CoreMgmtDropLinkAck { destination: u8 }, CoreMgmtDropLink, CoreMgmtGetLogReply { last: bool, length: u16, data: [u8; SAT_PAYLOAD_MAX_SIZE] }, @@ -485,24 +486,28 @@ impl Packet { 0xda => Packet::CoreMgmtAllocatorDebugRequest { destination: reader.read_u8()?, }, - 0xdb => { + 0xdb => Packet::CoreMgmtFlashRequest { + destination: reader.read_u8()?, + payload_length: reader.read_u32()?, + }, + 0xdc => { let destination = reader.read_u8()?; let last = reader.read_bool()?; let length = reader.read_u16()?; let mut data: [u8; MASTER_PAYLOAD_MAX_SIZE] = [0; MASTER_PAYLOAD_MAX_SIZE]; reader.read_exact(&mut data[0..length as usize])?; - Packet::CoreMgmtFlashRequest { + Packet::CoreMgmtFlashAddDataRequest { destination: destination, last: last, length: length, data: data, } }, - 0xdc => Packet::CoreMgmtDropLinkAck { + 0xdd => Packet::CoreMgmtDropLinkAck { destination: reader.read_u8()?, }, - 0xdd => Packet::CoreMgmtDropLink, - 0xde => { + 0xde => Packet::CoreMgmtDropLink, + 0xdf => { let last = reader.read_bool()?; let length = reader.read_u16()?; let mut data: [u8; SAT_PAYLOAD_MAX_SIZE] = [0; SAT_PAYLOAD_MAX_SIZE]; @@ -513,7 +518,7 @@ impl Packet { data: data, } }, - 0xdf => { + 0xe0 => { let last = reader.read_bool()?; let length = reader.read_u16()?; let mut value: [u8; SAT_PAYLOAD_MAX_SIZE] = [0; SAT_PAYLOAD_MAX_SIZE]; @@ -524,7 +529,7 @@ impl Packet { value: value, } }, - 0xe0 => Packet::CoreMgmtReply { + 0xe1 => Packet::CoreMgmtReply { succeeded: reader.read_bool()?, }, @@ -884,33 +889,38 @@ impl Packet { writer.write_u8(0xda)?; writer.write_u8(destination)?; }, - Packet::CoreMgmtFlashRequest { destination, last, length, data } => { + Packet::CoreMgmtFlashRequest { destination, payload_length } => { writer.write_u8(0xdb)?; writer.write_u8(destination)?; + writer.write_u32(payload_length)?; + }, + Packet::CoreMgmtFlashAddDataRequest { destination, last, length, data } => { + writer.write_u8(0xdc)?; + writer.write_u8(destination)?; writer.write_bool(last)?; writer.write_u16(length)?; writer.write_all(&data[..length as usize])?; }, Packet::CoreMgmtDropLinkAck { destination } => { - writer.write_u8(0xdc)?; + writer.write_u8(0xdd)?; writer.write_u8(destination)?; }, Packet::CoreMgmtDropLink => - writer.write_u8(0xdd)?, + writer.write_u8(0xde)?, Packet::CoreMgmtGetLogReply { last, length, data } => { - writer.write_u8(0xde)?; + writer.write_u8(0xdf)?; writer.write_bool(last)?; writer.write_u16(length)?; writer.write_all(&data[0..length as usize])?; }, Packet::CoreMgmtConfigReadReply { last, length, value } => { - writer.write_u8(0xdf)?; + writer.write_u8(0xe0)?; writer.write_bool(last)?; writer.write_u16(length)?; writer.write_all(&value[0..length as usize])?; }, Packet::CoreMgmtReply { succeeded } => { - writer.write_u8(0xe0)?; + writer.write_u8(0xe1)?; writer.write_bool(succeeded)?; }, } diff --git a/artiq/firmware/runtime/mgmt.rs b/artiq/firmware/runtime/mgmt.rs index c411519df..d0c6d8d95 100644 --- a/artiq/firmware/runtime/mgmt.rs +++ b/artiq/firmware/runtime/mgmt.rs @@ -546,9 +546,29 @@ mod remote_coremgmt { routing_table: &RoutingTable, linkno: u8, destination: u8, stream: &mut TcpStream, image: &[u8]) -> Result<(), Error> { + let alloc_reply = drtio::aux_transact(io, aux_mutex, ddma_mutex, subkernel_mutex, routing_table, linkno, + &Packet::CoreMgmtFlashRequest { + destination: destination, + payload_length: image.len() as u32, + }); + + match alloc_reply { + Ok(Packet::CoreMgmtReply { succeeded: true }) => Ok(()), + Ok(packet) => { + error!("received unexpected aux packet: {:?}", packet); + Reply::Error.write_to(stream)?; + Err(drtio::Error::UnexpectedReply) + } + Err(e) => { + error!("aux packet error ({})", e); + Reply::Error.write_to(stream)?; + Err(e) + } + }?; + match drtio::partition_data(&image, |slice, status, len: usize| { let reply = drtio::aux_transact(io, aux_mutex, ddma_mutex, subkernel_mutex, routing_table, linkno, - &Packet::CoreMgmtFlashRequest { + &Packet::CoreMgmtFlashAddDataRequest { destination: destination, length: len as u16, last: status.is_last(), data: *slice}); match reply { Ok(Packet::CoreMgmtReply { succeeded: true }) => Ok(()), diff --git a/artiq/firmware/satman/main.rs b/artiq/firmware/satman/main.rs index b5dfff323..c289b6b50 100644 --- a/artiq/firmware/satman/main.rs +++ b/artiq/firmware/satman/main.rs @@ -628,7 +628,13 @@ fn process_aux_packet(dmamgr: &mut DmaManager, analyzer: &mut Analyzer, kernelmg warn!("restarting"); unsafe { spiflash::reload(); } } - drtioaux::Packet::CoreMgmtFlashRequest { destination: _destination, last, length, data } => { + drtioaux::Packet::CoreMgmtFlashRequest { destination: _destination, payload_length } => { + forward!(router, _routing_table, _destination, *rank, *self_destination, _repeaters, &packet); + + coremgr.allocate_image_buffer(payload_length as usize); + drtioaux::send(0, &drtioaux::Packet::CoreMgmtReply { succeeded: true }) + } + drtioaux::Packet::CoreMgmtFlashAddDataRequest { destination: _destination, last, length, data } => { forward!(router, _routing_table, _destination, *rank, *self_destination, _repeaters, &packet); coremgr.add_image_data(&data, length as usize); diff --git a/artiq/firmware/satman/mgmt.rs b/artiq/firmware/satman/mgmt.rs index 338b1c26f..d7058b837 100644 --- a/artiq/firmware/satman/mgmt.rs +++ b/artiq/firmware/satman/mgmt.rs @@ -106,6 +106,10 @@ impl Manager { drtioaux::send(0, &drtioaux::Packet::CoreMgmtReply { succeeded }) } + pub fn allocate_image_buffer(&mut self, image_size: usize) { + self.image_payload = Cursor::new(Vec::with_capacity(image_size)); + } + pub fn add_image_data(&mut self, data: &[u8], data_len: usize) { self.image_payload.write_all(&data[..data_len]).unwrap(); } From de349e4c39fb05c99846290abf90cfe1c4ef41f5 Mon Sep 17 00:00:00 2001 From: occheung Date: Thu, 24 Oct 2024 12:40:25 +0800 Subject: [PATCH 091/111] bit2bin, fetch_bin -> flash_tools --- artiq/frontend/artiq_coremgmt.py | 3 +- artiq/frontend/artiq_flash.py | 7 +- artiq/frontend/bit2bin.py | 69 ------------------- artiq/frontend/fetch_bin.py | 59 ---------------- artiq/frontend/flash_tools.py | 114 +++++++++++++++++++++++++++++++ 5 files changed, 118 insertions(+), 134 deletions(-) delete mode 100755 artiq/frontend/bit2bin.py delete mode 100644 artiq/frontend/fetch_bin.py create mode 100644 artiq/frontend/flash_tools.py diff --git a/artiq/frontend/artiq_coremgmt.py b/artiq/frontend/artiq_coremgmt.py index 903a5615f..448349f85 100755 --- a/artiq/frontend/artiq_coremgmt.py +++ b/artiq/frontend/artiq_coremgmt.py @@ -12,8 +12,7 @@ from artiq import __version__ as artiq_version from artiq.master.databases import DeviceDB from artiq.coredevice.comm_kernel import CommKernel from artiq.coredevice.comm_mgmt import CommMgmt -from artiq.frontend.bit2bin import bit2bin -from artiq.frontend.fetch_bin import fetch_bin +from artiq.frontend.flash_tools import bit2bin, fetch_bin def get_argparser(): diff --git a/artiq/frontend/artiq_flash.py b/artiq/frontend/artiq_flash.py index b9563a339..5a5296979 100755 --- a/artiq/frontend/artiq_flash.py +++ b/artiq/frontend/artiq_flash.py @@ -14,8 +14,7 @@ from sipyco import common_args from artiq import __version__ as artiq_version from artiq.remoting import SSHClient, LocalClient -from artiq.frontend.bit2bin import bit2bin -from artiq.frontend.fetch_bin import fetch_bin +from artiq.frontend.flash_tools import artifact_path, bit2bin, fetch_bin def get_argparser(): @@ -305,10 +304,10 @@ def main(): for action in args.action: if action == "gateware": - gateware_bin = fetch_bin(binary_dir, "gateware", args.srcbuild) + gateware_bin = fetch_bin(binary_dir, ["gateware"], args.srcbuild) programmer.write_binary(*config["gateware"], gateware_bin) elif action == "bootloader": - bootloader_bin = fetch_bin(binary_dir, "bootloader", args.srcbuild) + bootloader_bin = fetch_bin(binary_dir, ["bootloader"], args.srcbuild) programmer.write_binary(*config["bootloader"], bootloader_bin) elif action == "storage": storage_img = args.storage diff --git a/artiq/frontend/bit2bin.py b/artiq/frontend/bit2bin.py deleted file mode 100755 index df3927415..000000000 --- a/artiq/frontend/bit2bin.py +++ /dev/null @@ -1,69 +0,0 @@ -#!/usr/bin/env python3 -# Copyright 2014-2017 Robert Jordens -# after -# https://github.com/mfischer/fpgadev-zynq/blob/master/top/python/bit_to_zynq_bin.py - -import struct - - -def flip32(data): - sl = struct.Struct("I") - b = memoryview(data) - d = bytearray(len(data)) - for offset in range(0, len(data), sl.size): - sb.pack_into(d, offset, *sl.unpack_from(b, offset)) - return d - - -def bit2bin(bit, bin, flip=False): - l, = struct.unpack(">H", bit.read(2)) - if l != 9: - raise ValueError("Missing <0009> header, not a bit file") - _ = bit.read(l) # unknown data - l, = struct.unpack(">H", bit.read(2)) - if l != 1: - raise ValueError("Missing <0001> header, not a bit file") - - while True: - key = bit.read(1).decode() - if not key: - break - if key in "abcd": - d = bit.read(*struct.unpack(">H", bit.read(2))) - assert d.endswith(b"\x00") - d = d[:-1].decode() - name = { - "a": "Design", - "b": "Part name", - "c": "Date", - "d": "Time" - }[key] - print("{}: {}".format(name, d)) - elif key == "e": - l, = struct.unpack(">I", bit.read(4)) - print("Bitstream payload length: {:#x}".format(l)) - d = bit.read(l) - if flip: - d = flip32(d) - bin.write(d) - else: - d = bit.read(*struct.unpack(">H", bit.read(2))) - print("Unexpected key: {}: {}".format(key, d)) - - -if __name__ == "__main__": - import argparse - parser = argparse.ArgumentParser( - description="Convert FPGA bit files to raw bin format " - "suitable for flashing") - parser.add_argument("-f", "--flip", dest="flip", action="store_true", - default=False, help="Flip 32-bit endianess (needed for Zynq)") - parser.add_argument("bitfile", metavar="BITFILE", - help="Input bit file name") - parser.add_argument("binfile", metavar="BINFILE", - help="Output bin file name") - args = parser.parse_args() - - with open(args.bitfile, "rb") as f, open(args.binfile, "wb") as g: - bit2bin(f, g, args.flip) diff --git a/artiq/frontend/fetch_bin.py b/artiq/frontend/fetch_bin.py deleted file mode 100644 index 890c3413c..000000000 --- a/artiq/frontend/fetch_bin.py +++ /dev/null @@ -1,59 +0,0 @@ -import atexit -import os -import tempfile - -from artiq.frontend.bit2bin import bit2bin - - -def fetch_bin(binary_dir, component, srcbuild=False): - def artifact_path(this_binary_dir, *path_filename): - if srcbuild: - # source tree - use path elements to locate file - return os.path.join(this_binary_dir, *path_filename) - else: - # flat tree - all files in the same directory, discard path elements - *_, filename = path_filename - return os.path.join(this_binary_dir, filename) - - def convert_gateware(bit_filename): - bin_handle, bin_filename = tempfile.mkstemp( - prefix="artiq_", suffix="_" + os.path.basename(bit_filename)) - with open(bit_filename, "rb") as bit_file, open(bin_handle, "wb") as bin_file: - bit2bin(bit_file, bin_file) - atexit.register(lambda: os.unlink(bin_filename)) - return bin_filename - - if type(component) == list: - bins = [] - for option in component: - try: - bins.append(fetch_bin(binary_dir, option, srcbuild)) - except FileNotFoundError: - pass - - if bins is None: - raise FileNotFoundError("multiple components not found: {}".format( - " ".join(component))) - - if len(bins) > 1: - raise ValueError("more than one file, " - "please clean up your build directory. " - "Found files: {}".format( - " ".join(bins))) - - return bins[0] - - path = artifact_path(binary_dir, *{ - "gateware": ["gateware", "top.bit"], - "bootloader": ["software", "bootloader", "bootloader.bin"], - "runtime": ["software", "runtime", "runtime.fbi"], - "satman": ["software", "satman", "satman.fbi"], - }[component]) - - if not os.path.exists(path): - raise FileNotFoundError("{} not found".format(component)) - - if component == "gateware": - path = convert_gateware(path) - - return path diff --git a/artiq/frontend/flash_tools.py b/artiq/frontend/flash_tools.py new file mode 100644 index 000000000..d486454d3 --- /dev/null +++ b/artiq/frontend/flash_tools.py @@ -0,0 +1,114 @@ +import atexit +import os +import tempfile + + +def artifact_path(this_binary_dir, *path_filename, srcbuild=False): + if srcbuild: + # source tree - use path elements to locate file + return os.path.join(this_binary_dir, *path_filename) + else: + # flat tree - all files in the same directory, discard path elements + *_, filename = path_filename + return os.path.join(this_binary_dir, filename) + + +def fetch_bin(binary_dir, components, srcbuild=False): + def convert_gateware(bit_filename): + bin_handle, bin_filename = tempfile.mkstemp( + prefix="artiq_", suffix="_" + os.path.basename(bit_filename)) + with open(bit_filename, "rb") as bit_file, open(bin_handle, "wb") as bin_file: + bit2bin(bit_file, bin_file) + atexit.register(lambda: os.unlink(bin_filename)) + return bin_filename + + if len(components) > 1: + bins = [] + for option in components: + try: + bins.append(fetch_bin(binary_dir, [option], srcbuild)) + except FileNotFoundError: + pass + + if bins is None: + raise FileNotFoundError("multiple components not found: {}".format( + " ".join(components))) + + if len(bins) > 1: + raise ValueError("more than one file, " + "please clean up your build directory. " + "Found files: {}".format( + " ".join(bins))) + + return bins[0] + + else: + component = components[0] + path = artifact_path(binary_dir, *{ + "gateware": ["gateware", "top.bit"], + "boot": ["boot.bin"], + "bootloader": ["software", "bootloader", "bootloader.bin"], + "runtime": ["software", "runtime", "runtime.fbi"], + "satman": ["software", "satman", "satman.fbi"], + }[component], srcbuild=srcbuild) + + if not os.path.exists(path): + raise FileNotFoundError("{} not found".format(component)) + + if component == "gateware": + path = convert_gateware(path) + + return path + + +# Copyright 2014-2017 Robert Jordens +# after +# https://github.com/mfischer/fpgadev-zynq/blob/master/top/python/bit_to_zynq_bin.py + +import struct + + +def flip32(data): + sl = struct.Struct("I") + b = memoryview(data) + d = bytearray(len(data)) + for offset in range(0, len(data), sl.size): + sb.pack_into(d, offset, *sl.unpack_from(b, offset)) + return d + + +def bit2bin(bit, bin, flip=False): + l, = struct.unpack(">H", bit.read(2)) + if l != 9: + raise ValueError("Missing <0009> header, not a bit file") + _ = bit.read(l) # unknown data + l, = struct.unpack(">H", bit.read(2)) + if l != 1: + raise ValueError("Missing <0001> header, not a bit file") + + while True: + key = bit.read(1).decode() + if not key: + break + if key in "abcd": + d = bit.read(*struct.unpack(">H", bit.read(2))) + assert d.endswith(b"\x00") + d = d[:-1].decode() + name = { + "a": "Design", + "b": "Part name", + "c": "Date", + "d": "Time" + }[key] + print("{}: {}".format(name, d)) + elif key == "e": + l, = struct.unpack(">I", bit.read(4)) + print("Bitstream payload length: {:#x}".format(l)) + d = bit.read(l) + if flip: + d = flip32(d) + bin.write(d) + else: + d = bit.read(*struct.unpack(">H", bit.read(2))) + print("Unexpected key: {}: {}".format(key, d)) From e36916b931937a4ccaee51ce5baf583126fb0a6d Mon Sep 17 00:00:00 2001 From: occheung Date: Thu, 24 Oct 2024 12:42:11 +0800 Subject: [PATCH 092/111] coremgmt flashing: detect risc-v or zynq targets --- artiq/frontend/artiq_coremgmt.py | 41 ++++++++++++++++++++++---------- 1 file changed, 29 insertions(+), 12 deletions(-) diff --git a/artiq/frontend/artiq_coremgmt.py b/artiq/frontend/artiq_coremgmt.py index 448349f85..c73dcfd70 100755 --- a/artiq/frontend/artiq_coremgmt.py +++ b/artiq/frontend/artiq_coremgmt.py @@ -93,10 +93,6 @@ def get_argparser(): t_flash = tools.add_parser("flash", help="flash the running system") - p_zynq = t_flash.add_argument("-z", "--zynq", default=False, - help="target zynq device", - action="store_true") - p_directory = t_flash.add_argument("directory", metavar="DIRECTORY", type=str, help="directory that contains the " @@ -167,15 +163,36 @@ def main(): mgmt.config_erase() if args.tool == "flash": - if args.zynq: - boot = os.path.join(args.directory, "boot.bin") - bins = [boot] - else: - gateware = fetch_bin(args.directory, "gateware", args.srcbuild) - bootloader = fetch_bin(args.directory, "bootloader", args.srcbuild) - firmware = fetch_bin(args.directory, ["runtime", "satman"], args.srcbuild) - bins = [gateware, bootloader, firmware] + retrieved_bins = [] + bin_dict = { + "zynq":[ + ["boot"] + ], + "riscv": [ + ["gateware"], + ["bootloader"], + ["runtime", "satman"], + ], + } + for bin_list in bin_dict.values(): + try: + bins = [] + for bin_name in bin_list: + bins.append(fetch_bin( + args.directory, bin_name, args.srcbuild)) + retrieved_bins.append(bins) + except FileNotFoundError: + pass + + if retrieved_bins is None: + raise FileNotFoundError("both risc-v and zynq binaries not found") + + if len(retrieved_bins) > 1: + raise ValueError("both risc-v and zynq binaries were found, " + "please clean up your build directory. ") + + bins = retrieved_bins[0] mgmt.flash(bins) if args.tool == "reboot": From 6251e734599c62b8532b55648ae9d9fb7e675b9c Mon Sep 17 00:00:00 2001 From: occheung Date: Thu, 24 Oct 2024 12:43:09 +0800 Subject: [PATCH 093/111] board_misoc: enable efc rebooting --- artiq/firmware/libboard_misoc/spiflash.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/artiq/firmware/libboard_misoc/spiflash.rs b/artiq/firmware/libboard_misoc/spiflash.rs index 7d66977f7..e686d17e6 100644 --- a/artiq/firmware/libboard_misoc/spiflash.rs +++ b/artiq/firmware/libboard_misoc/spiflash.rs @@ -124,7 +124,7 @@ pub unsafe fn flash_binary(origin: usize, payload: &[u8]) { write(origin, payload); } -#[cfg(any(soc_platform = "kasli", soc_platform = "kc705"))] +#[cfg(any(soc_platform = "kasli", soc_platform = "kc705", soc_platform = "efc"))] pub unsafe fn reload () -> ! { csr::icap::iprog_write(1); loop {} From 90764b04f9af1c0a20099bbdecdbf74c891ab40e Mon Sep 17 00:00:00 2001 From: occheung Date: Mon, 18 Nov 2024 16:04:44 +0800 Subject: [PATCH 094/111] both ... not -> neither ... nor --- artiq/frontend/artiq_coremgmt.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/artiq/frontend/artiq_coremgmt.py b/artiq/frontend/artiq_coremgmt.py index c73dcfd70..a137a402a 100755 --- a/artiq/frontend/artiq_coremgmt.py +++ b/artiq/frontend/artiq_coremgmt.py @@ -186,7 +186,7 @@ def main(): pass if retrieved_bins is None: - raise FileNotFoundError("both risc-v and zynq binaries not found") + raise FileNotFoundError("neither risc-v nor zynq binaries were found") if len(retrieved_bins) > 1: raise ValueError("both risc-v and zynq binaries were found, " From cb836cd4a0106afef666c05cc12c0b1126681912 Mon Sep 17 00:00:00 2001 From: Sebastien Bourdeauducq Date: Wed, 20 Nov 2024 08:20:26 +0800 Subject: [PATCH 095/111] fmcdio_vhdci_eem: remove --- RELEASE_NOTES.rst | 1 + artiq/coredevice/fmcdio_vhdci_eem.py | 51 ---------------------------- 2 files changed, 1 insertion(+), 51 deletions(-) delete mode 100644 artiq/coredevice/fmcdio_vhdci_eem.py diff --git a/RELEASE_NOTES.rst b/RELEASE_NOTES.rst index 7d1c160c8..3d9dcd923 100644 --- a/RELEASE_NOTES.rst +++ b/RELEASE_NOTES.rst @@ -22,6 +22,7 @@ ARTIQ-9 (Unreleased) * Idle kernels now restart when written with ``artiq_coremgmt`` and stop when erased/removed from config. * New support for the EBAZ4205 Zynq-SoC control card. * New core device driver for the AD9834 DDS, tested with the ZonRi Technology Co., Ltd. AD9834-Module. +* artiq.coredevice.fmcdio_vhdci_eem has been removed. ARTIQ-8 ------- diff --git a/artiq/coredevice/fmcdio_vhdci_eem.py b/artiq/coredevice/fmcdio_vhdci_eem.py deleted file mode 100644 index bddb55812..000000000 --- a/artiq/coredevice/fmcdio_vhdci_eem.py +++ /dev/null @@ -1,51 +0,0 @@ -# Definitions for using the "FMC DIO 32ch LVDS a" card with the VHDCI-EEM breakout v1.1 - -eem_fmc_connections = { - 0: [0, 8, 2, 3, 4, 5, 6, 7], - 1: [1, 9, 10, 11, 12, 13, 14, 15], - 2: [17, 16, 24, 19, 20, 21, 22, 23], - 3: [18, 25, 26, 27, 28, 29, 30, 31], -} - - -def shiftreg_bits(eem, out_pins): - """ - Returns the bits that have to be set in the FMC card direction - shift register for the given EEM. - - Takes a set of pin numbers (0-7) at the EEM. Return values - of this function for different EEMs should be ORed together. - """ - r = 0 - for i in range(8): - if i not in out_pins: - lvds_line = eem_fmc_connections[eem][i] - # lines are swapped in pairs to ease PCB routing - # at the shift register - shift = lvds_line ^ 1 - r |= 1 << shift - return r - - -dio_bank0_out_pins = set(range(4)) -dio_bank1_out_pins = set(range(4, 8)) -urukul_out_pins = { - 0, # clk - 1, # mosi - 3, 4, 5, # cs_n - 6, # io_update - 7, # dds_reset -} -urukul_aux_out_pins = { - 4, # sw0 - 5, # sw1 - 6, # sw2 - 7, # sw3 -} -zotino_out_pins = { - 0, # clk - 1, # mosi - 3, 4, # cs_n - 5, # ldac_n - 7, # clr_n -} From df1a389007f141f8652052abe301a853cd7fa1ee Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Bourdeauducq?= Date: Wed, 20 Nov 2024 08:23:51 +0800 Subject: [PATCH 096/111] flash_tools: cleanup import --- artiq/frontend/flash_tools.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/artiq/frontend/flash_tools.py b/artiq/frontend/flash_tools.py index d486454d3..432d03f37 100644 --- a/artiq/frontend/flash_tools.py +++ b/artiq/frontend/flash_tools.py @@ -1,6 +1,7 @@ import atexit import os import tempfile +import struct def artifact_path(this_binary_dir, *path_filename, srcbuild=False): @@ -65,9 +66,6 @@ def fetch_bin(binary_dir, components, srcbuild=False): # after # https://github.com/mfischer/fpgadev-zynq/blob/master/top/python/bit_to_zynq_bin.py -import struct - - def flip32(data): sl = struct.Struct("I") From e1aa8a5a8c99abcd5dd8efb34b9f58c8d0f7fd3f Mon Sep 17 00:00:00 2001 From: Jonathan Coates Date: Tue, 19 Nov 2024 15:46:12 +0000 Subject: [PATCH 097/111] Remove final_branch from ARTIQ IR transform As of da4ff44377512b4b2fdd61c6e0d437153662dd4a this is always None, and so can be skipped. Signed-off-by: Jonathan Coates --- .../compiler/transforms/artiq_ir_generator.py | 18 ------------------ 1 file changed, 18 deletions(-) diff --git a/artiq/compiler/transforms/artiq_ir_generator.py b/artiq/compiler/transforms/artiq_ir_generator.py index a47088d0d..a7177210b 100644 --- a/artiq/compiler/transforms/artiq_ir_generator.py +++ b/artiq/compiler/transforms/artiq_ir_generator.py @@ -65,10 +65,6 @@ class ARTIQIRGenerator(algorithm.Visitor): :ivar catch_clauses: (list of (:class:`ir.BasicBlock`, :class:`types.Type` or None)) a list of catch clauses that should be appended to inner try block landingpad - :ivar final_branch: (function (target: :class:`ir.BasicBlock`, block: :class:`ir.BasicBlock) - or None) - the function that appends to ``block`` a jump through the ``finally`` statement - to ``target`` There is, additionally, some global state that is used to translate the results of analyses on AST level to IR level: @@ -114,7 +110,6 @@ class ARTIQIRGenerator(algorithm.Visitor): self.return_target = None self.unwind_target = None self.catch_clauses = [] - self.final_branch = None self.function_map = dict() self.variable_map = dict() self.method_map = defaultdict(lambda: []) @@ -635,11 +630,6 @@ class ARTIQIRGenerator(algorithm.Visitor): self.append(ir.Branch(self.continue_target)) def raise_exn(self, exn=None, loc=None): - if self.final_branch is not None: - raise_proxy = self.add_block("try.raise") - self.final_branch(raise_proxy, self.current_block) - self.current_block = raise_proxy - if exn is not None: # if we need to raise the exception in a final body, we have to # lazy-evaluate the exception object to make sure that we generate @@ -1114,13 +1104,11 @@ class ARTIQIRGenerator(algorithm.Visitor): entry = self.add_block("entry") old_block, self.current_block = self.current_block, entry - old_final_branch, self.final_branch = self.final_branch, None old_unwind, self.unwind_target = self.unwind_target, None self.raise_exn(lambda: exn_gen(*args[1:]), loc=loc) finally: self.current_function = old_func self.current_block = old_block - self.final_branch = old_final_branch self.unwind_target = old_unwind # cond: bool Value, condition @@ -1539,7 +1527,6 @@ class ARTIQIRGenerator(algorithm.Visitor): entry = self.add_block("entry") old_block, self.current_block = self.current_block, entry - old_final_branch, self.final_branch = self.final_branch, None old_unwind, self.unwind_target = self.unwind_target, None shape = self.append(ir.GetAttr(arg, "shape")) @@ -1565,7 +1552,6 @@ class ARTIQIRGenerator(algorithm.Visitor): self.current_loc = old_loc self.current_function = old_func self.current_block = old_block - self.final_branch = old_final_branch self.unwind_target = old_unwind def _get_array_unaryop(self, name, make_op, result_type, arg_type): @@ -1666,7 +1652,6 @@ class ARTIQIRGenerator(algorithm.Visitor): entry = self.add_block("entry") old_block, self.current_block = self.current_block, entry - old_final_branch, self.final_branch = self.final_branch, None old_unwind, self.unwind_target = self.unwind_target, None body_gen(result, lhs, rhs) @@ -1677,7 +1662,6 @@ class ARTIQIRGenerator(algorithm.Visitor): self.current_loc = old_loc self.current_function = old_func self.current_block = old_block - self.final_branch = old_final_branch self.unwind_target = old_unwind def _make_array_elementwise_binop(self, name, result_type, lhs_type, @@ -2820,7 +2804,6 @@ class ARTIQIRGenerator(algorithm.Visitor): entry = self.add_block("entry") old_block, self.current_block = self.current_block, entry - old_final_branch, self.final_branch = self.final_branch, None old_unwind, self.unwind_target = self.unwind_target, None exn = self.alloc_exn(builtins.TException("AssertionError"), @@ -2833,7 +2816,6 @@ class ARTIQIRGenerator(algorithm.Visitor): finally: self.current_function = old_func self.current_block = old_block - self.final_branch = old_final_branch self.unwind_target = old_unwind self.raise_assert_func = func From 270a417a28b516d36983779a1adb6d33a3c55a4a Mon Sep 17 00:00:00 2001 From: Jonathan Coates Date: Tue, 19 Nov 2024 17:24:53 +0000 Subject: [PATCH 098/111] Correctly handle try/catch try/finally blocks If we have a try/catch block nested inside a try/finally, then the finally block would not be executed in the event of the exception. This is because the landing pad of the inner try is marked as having no cleanup clause. We now set has_cleanup to False only if there is no outer try block. Signed-off-by: Jonathan Coates --- .../compiler/transforms/artiq_ir_generator.py | 4 +--- .../test/lit/exceptions/finally_catch_try.py | 19 ++++++++++++++++++ artiq/test/lit/exceptions/finally_try.py | 20 +++++++++++++++++++ 3 files changed, 40 insertions(+), 3 deletions(-) create mode 100644 artiq/test/lit/exceptions/finally_catch_try.py create mode 100644 artiq/test/lit/exceptions/finally_try.py diff --git a/artiq/compiler/transforms/artiq_ir_generator.py b/artiq/compiler/transforms/artiq_ir_generator.py index a7177210b..d3d348b12 100644 --- a/artiq/compiler/transforms/artiq_ir_generator.py +++ b/artiq/compiler/transforms/artiq_ir_generator.py @@ -703,7 +703,7 @@ class ARTIQIRGenerator(algorithm.Visitor): return_action.append(ir.Return(value)) final_branch(return_action, return_proxy) else: - landingpad.has_cleanup = False + landingpad.has_cleanup = self.unwind_target is not None # we should propagate the clauses to nested try catch blocks # so nested try catch will jump to our clause if the inner one does not @@ -767,7 +767,6 @@ class ARTIQIRGenerator(algorithm.Visitor): self.continue_target = old_continue self.return_target = old_return - if any(node.finalbody): # create new unwind target for cleanup final_dispatcher = self.add_block("try.final.dispatch") final_landingpad = ir.LandingPad(cleanup) @@ -776,7 +775,6 @@ class ARTIQIRGenerator(algorithm.Visitor): # make sure that exception clauses are unwinded to the finally block old_unwind, self.unwind_target = self.unwind_target, final_dispatcher - if any(node.finalbody): # if we have a while:try/finally continue must execute finally # before continuing the while redirect = final_branch diff --git a/artiq/test/lit/exceptions/finally_catch_try.py b/artiq/test/lit/exceptions/finally_catch_try.py new file mode 100644 index 000000000..c1b500a78 --- /dev/null +++ b/artiq/test/lit/exceptions/finally_catch_try.py @@ -0,0 +1,19 @@ +# RUN: %python -m artiq.compiler.testbench.jit %s >%t +# RUN: OutputCheck %s --file-to-check=%t +# REQUIRES: exceptions + +def doit(): + try: + try: + raise RuntimeError("Error") + except ValueError: + print("ValueError") + except RuntimeError: + print("Caught") + finally: + print("Cleanup") + +doit() + +# CHECK-L: Caught +# CHECK-NEXT-L: Cleanup diff --git a/artiq/test/lit/exceptions/finally_try.py b/artiq/test/lit/exceptions/finally_try.py new file mode 100644 index 000000000..377ca1c06 --- /dev/null +++ b/artiq/test/lit/exceptions/finally_try.py @@ -0,0 +1,20 @@ +# RUN: %python -m artiq.compiler.testbench.jit %s >%t +# RUN: OutputCheck %s --file-to-check=%t +# REQUIRES: exceptions + +def doit(): + try: + try: + raise RuntimeError("Error") + except ValueError: + print("ValueError") + finally: + print("Cleanup") + +try: + doit() +except RuntimeError: + print("Caught") + +# CHECK-L: Cleanup +# CHECK-NEXT-L: Caught From 61c311fc54cd77be64412dc3c6da84b35725caaf Mon Sep 17 00:00:00 2001 From: architeuthidae Date: Sun, 17 Nov 2024 19:16:36 +0100 Subject: [PATCH 099/111] doc: Add FAQ on port conflicts --- doc/manual/faq.rst | 33 +++++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/doc/manual/faq.rst b/doc/manual/faq.rst index 5dd925678..e21832301 100644 --- a/doc/manual/faq.rst +++ b/doc/manual/faq.rst @@ -125,6 +125,39 @@ This and other similar messages almost always indicate that your device database to let the controller manager start the necessary controllers automatically. +fix ``address already in use`` when running ARTIQ commands? +----------------------------------------------------------- + +A message like ``OSError: [Errno 98] error while attempting to bind on address ('127.0.0.1', 1067): [errno 98] address already in use`` indicates that the IP address and port number combination you're trying to use is already occupied by some other process. Often this simply means that the ARTIQ process you're trying to start is in fact already running. Note for example that trying to start a controller which is already being run by a controller manager will generally fail for this reason. + +.. note:: + ARTIQ management system communications, whether distributed or local, run over TCP/IP, using TCP port numbers to identify their destinations. Generally speaking, client processes like the dashboard don't require fixed ports of their own, since they can simply reach out to the master when they want to establish a connection. Running multiple dashboards will never cause a port conflict. On the other hand, server processes like the ARTIQ master have to be 'listening' at a fixed, open port in order to be able to receive incoming connections. For more details, look into `ports in computer networking `_. + + Most management system processes belong to the second category, and are bound to one or several fixed communication ports while they're running. See also :doc:`default_network_ports`. + +You can use the command ``netstat`` to list the ports currently in use on your system. To check the status of a specific port on Linux, try either of: :: + + $ netstat -anp --inet | grep "" + $ lsof -i: + +On Windows, use :: + + $ netstat -ano -p ip | find "" + +In all cases, if there are no results, the port isn't in use and should be free for new processes. + +.. tip:: + While it is possible to run, for example, two identical ARTIQ controllers on the same machine, they can't be bound to the same port numbers at the same time. If you're intentionally running multiple copies of the same ARTIQ processes, use the command-line ``--port`` options to set alternate ports for at least one of the two. See :doc:`main_frontend_tools` and :doc:`utilities` for exact flags to use. Controllers should have similar flags available and will also require updated :ref:`device database entries `. Note that alternate ports must be consistent to be useful, e.g., a master and dashboard must have the same ``--port-notify`` set in order to communicate with each other! + +Otherwise, either the running process must be stopped, or you'll have to set different port numbers for the process you're trying to start. In some cases it might happen that a process is no longer accessible or has become unresponsive but is still occupying its ports. The easiest way to free the ports is to kill the process manually. On Linux, you can use the ``kill`` command with ``lsof``: :: + + $ kill $(lsof -t -i:) + +On Windows, use ``netstat`` again to identify the process ID, and then feed it into ``taskkill``, e.g.: :: + + $ netstat -ano | find "" + $ taskkill /F /PID + diagnose and fix sequence errors? --------------------------------- From 9b5b4c07eaa4994c659b73d8ec15693424fdc4c0 Mon Sep 17 00:00:00 2001 From: architeuthidae Date: Thu, 21 Nov 2024 18:09:10 +0100 Subject: [PATCH 100/111] doc: note use of scheduler attributes --- doc/manual/management_system.rst | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/doc/manual/management_system.rst b/doc/manual/management_system.rst index 69a575cbf..02b0fabd5 100644 --- a/doc/manual/management_system.rst +++ b/doc/manual/management_system.rst @@ -156,6 +156,11 @@ Accessing the :meth:`pause` and :meth:`~artiq.master.scheduler.Scheduler.check_p :meth:`~artiq.master.scheduler.Scheduler.check_pause` can be called (via RPC) from a kernel, but :meth:`pause` cannot be. +Scheduler attributes +^^^^^^^^^^^^^^^^^^^^ + +The ``scheduler`` virtual device also exposes information about an experiment's scheduling status through the attributes ``rid``, ``pipeline_name``, ``priority``, and ``expid``. This allows e.g. access to an experiment's current RID as ``self.scheduler.rid``. + Internal details ---------------- From f4e3b3a0a5042ff0249c61349064d5e8059c018f Mon Sep 17 00:00:00 2001 From: architeuthidae Date: Thu, 21 Nov 2024 18:15:44 +0100 Subject: [PATCH 101/111] doc: cli argument submission format --- artiq/frontend/artiq_client.py | 2 +- artiq/frontend/artiq_run.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/artiq/frontend/artiq_client.py b/artiq/frontend/artiq_client.py index b0a2345e6..06b1974fb 100755 --- a/artiq/frontend/artiq_client.py +++ b/artiq/frontend/artiq_client.py @@ -75,7 +75,7 @@ def get_argparser(): parser_add.add_argument("file", metavar="FILE", help="file containing the experiment to run") parser_add.add_argument("arguments", metavar="ARGUMENTS", nargs="*", - help="run arguments") + help="run arguments, use format KEY=VALUE") parser_delete = subparsers.add_parser("delete", help="delete an experiment " diff --git a/artiq/frontend/artiq_run.py b/artiq/frontend/artiq_run.py index ea6a50c00..cf6d5e45c 100755 --- a/artiq/frontend/artiq_run.py +++ b/artiq/frontend/artiq_run.py @@ -161,7 +161,7 @@ def get_argparser(with_file=True): parser.add_argument("file", metavar="FILE", help="file containing the experiment to run") parser.add_argument("arguments", metavar="ARGUMENTS", nargs="*", - help="run arguments") + help="run arguments, use format KEY=VALUE") return parser From d5020f205e74ee8caacbcce7ec82514114882913 Mon Sep 17 00:00:00 2001 From: architeuthidae Date: Thu, 21 Nov 2024 17:57:49 +0100 Subject: [PATCH 102/111] doc: config over DRTIO changes --- doc/manual/building_developing.rst | 31 +++++++++++++++--------------- doc/manual/configuring.rst | 16 +++++++++++++++ doc/manual/faq.rst | 7 ------- doc/manual/flashing.rst | 26 +++++++++++-------------- 4 files changed, 42 insertions(+), 38 deletions(-) diff --git a/doc/manual/building_developing.rst b/doc/manual/building_developing.rst index db9ed1a0c..b14e5642e 100644 --- a/doc/manual/building_developing.rst +++ b/doc/manual/building_developing.rst @@ -140,7 +140,7 @@ The parallel command does exist for ARTIQ-Zynq: :: but if you are building ARTIQ-Zynq without intention to change the source, it is not actually necessary to enter the development environment at all; Nix is capable of accessing the official flake remotely for the build itself, eliminating the requirement for any particular environment. -This is equally possible for original ARTIQ, but not as useful, as the development environment (specifically the ``#boards`` shell) is still the easiest way to access the necessary tools for flashing the board. On the other hand, with Zynq, it is normally recommended to boot from SD card, which requires no further special tools. As long as you have a functioning Nix installation with flakes enabled, you can progress directly to the building instructions below. +This is equally possible for original ARTIQ, but not as useful, as the development environment (specifically the ``#boards`` shell) is still the easiest way to access the necessary tools for flashing the board. On the other hand, Zynq boards can also be flashed by writing to the SD card directly, which requires no further special tools. As long as you have a functioning Nix/Vivado installation with flakes enabled, you can progress directly to the building instructions below. .. _building: @@ -160,8 +160,9 @@ With KC705, use: :: $ python -m artiq.gateware.targets.kc705 -V -This will create a directory ``artiq_kasli`` or ``artiq_kc705`` containing the binaries in a subdirectory named after your description file or variant. Flash the board as described in :ref:`writing-flash`, adding the option ``--srcbuild``, e.g., assuming your board is already connected by JTAG USB: :: +This will create a directory ``artiq_kasli`` or ``artiq_kc705`` containing the binaries in a subdirectory named after your description file or variant. Flash the board as described in :ref:`writing-flash`, adding the option ``--srcbuild``, e.g., assuming your board is connected by network or JTAG USB respectively: :: + $ artiq_coremgmt flash --srcbuild artiq_/ $ artiq_flash --srcbuild [-t kc705] -d artiq_/ .. note:: @@ -190,20 +191,24 @@ or you can use the more direct version: :: $ nix build --print-build-logs git+https://git.m-labs.hk/m-labs/artiq-zynq\?ref=release-[number]#--sd -(which is possible for ZC706 or EBAZ4205 because there is no need to be able to specify a system description file in the arguments.) +(which is possible for ZC706 and EBAZ4205 because there is no need to be able to specify a system description file in the arguments.) .. note:: - To see supported ZC706 variants (for EBAZ4205 variants, replace ``zc706`` with ``ebaz4205``), you can run the following at the root of the repository: :: + To see supported variants for ZC705 or EBA4205, you can run the following at the root of the repository: :: - $ src/gateware/zc706.py --help + $ src/gateware/.py --help Look for the option ``-V VARIANT, --variant VARIANT``. If you have not cloned the repository or are not in the development environment, try: :: - $ nix flake show git+https://git.m-labs.hk/m-labs/artiq-zynq\?ref=release-[number] | grep "package 'zc706.*sd" + $ nix flake show git+https://git.m-labs.hk/m-labs/artiq-zynq\?ref=release-[number] | grep "package '.*sd" to see the list of suitable build targets directly. -Any of these commands should produce a directory ``result`` which contains a file ``boot.bin``. As described in :ref:`writing-flash`, if your core device is currently accessible over the network, it can be flashed with :mod:`~artiq.frontend.artiq_coremgmt`. If it is not connected to the network: +Any of these commands should produce a directory ``result`` which contains a file ``boot.bin``. If your core device is accessible by network, flash with: :: + + $ artiq_coremgmt flash result + +Otherwise: 1. Power off the board, extract the SD card and load ``boot.bin`` onto it manually. 2. Insert the SD card back into the board. @@ -244,17 +249,11 @@ For Kasli-SoC: $ gateware/kasli_soc.py -g ../build/gateware $ make TARGET=kasli_soc GWARGS="path/to/description.json" -For ZC706: +For ZC706 or EBAZ4205: :: - $ gateware/zc706.py -g ../build/gateware -V - $ make TARGET=zc706 GWARGS="-V " - -For EBAZ4205: - :: - - $ gateware/ebaz4205.py -g ../build/gateware -V - $ make TARGET=ebaz4205 GWARGS="-V " + $ gateware/.py -g ../build/gateware -V + $ make TARGET= GWARGS="-V " where ``fw-type`` is ``runtime`` for standalone or DRTIO master builds and ``satman`` for DRTIO satellites. Both the gateware and the firmware will generate into the ``../build`` destination directory. At this stage, if supported, you can :ref:`boot from JTAG `; either of the ``*_run.sh`` scripts will expect the gateware and firmware files at their default locations, and the ``szl.elf`` bootloader is retrieved automatically. diff --git a/doc/manual/configuring.rst b/doc/manual/configuring.rst index 998344f24..5da7c4faa 100644 --- a/doc/manual/configuring.rst +++ b/doc/manual/configuring.rst @@ -6,6 +6,9 @@ Networking and configuration Setting up core device networking --------------------------------- +.. note:: + Satellite core devices (in a DRTIO setting, see :doc:`using_drtio_subkernels`) do not support independent networking and this section does not apply to them. Follow the instructions on this page for your master core device, and proceed to :ref:`configuring-satellite` once DRTIO communications are established. + For Kasli, insert a SFP/RJ45 transceiver (normally included with purchases from M-Labs and QUARTIQ) into the SFP0 port and connect it to an Ethernet port in your network. If the port is 10Mbps or 100Mbps and not 1000Mbps, make sure that the SFP/RJ45 transceiver supports the lower rate. Many SFP/RJ45 transceivers only support the 1000Mbps rate. If you do not have a SFP/RJ45 transceiver that supports 10Mbps and 100Mbps rates, you may instead use a gigabit Ethernet switch in the middle to perform rate conversion. You can also insert other types of SFP transceivers into Kasli if you wish to use it directly in e.g. an optical fiber Ethernet network. Kasli-SoC already directly features RJ45 10/100/1000 Ethernet. @@ -130,3 +133,16 @@ Load the DRTIO routing table ^^^^^^^^^^^^^^^^^^^^^^^^^^^^ If you are using DRTIO and the default routing table (for a star topology) is not suitable to your needs, you will first need to prepare and load a different routing table. See :ref:`Using DRTIO `. + +.. _configuring-satellite: + +Configuring DRTIO satellites +---------------------------- + +Once DRTIO communications are online, any satellite devices can be accessed as normal using :mod:`~artiq.frontend.artiq_coremgmt`, e.g.: :: + + $ artiq_coremgmt -s log + +The destination number corresponds to the number assigned to that satellite both in the device database and, earlier, in the system configuration file. See the notes in :ref:`drtio-routing` if you are not sure what destination number to use. + +It is also possible to set configuration values, reflash, or reboot the device. Notably, :ref:`event spreading ` is a per-device setting considered particularly useful on satellites. Most other configuration settings, e.g. networking, clocking, will not be used in practice in a satellite context -- satellites do not support direct network connections and are always bound to the master's clock. \ No newline at end of file diff --git a/doc/manual/faq.rst b/doc/manual/faq.rst index e21832301..6903c8e65 100644 --- a/doc/manual/faq.rst +++ b/doc/manual/faq.rst @@ -75,13 +75,6 @@ Either reflash your core device with a newer version of ARTIQ (see :doc:`flashin Minor version mismatches are common, even in stable ARTIQ versions, but should not cause any issues. The ARTIQ release system ensures breaking changes are strictly limited to new release versions, or to the beta branch (which explicitly makes no promises of stability.) Updates that *are* applied to the stable version are usually bug fixes, documentation improvements, or other quality-of-life changes. As long as gateware and software are using the same stable release version of ARTIQ, even if there is a minor mismatch, no warning will be displayed. -change configuration settings of satellite devices? ---------------------------------------------------- - -Currently, it is not possible to reach satellites through ``artiq_coremgmt config``, although this is being worked on. On Kasli, use :class:`~artiq.frontend.artiq_mkfs` and :class:`~artiq.frontend.artiq_flash`; on Kasli-SoC, preload the SD card with a ``config.txt``, formatted as a list of ``key=value`` pairs, one per line. - -Don't worry about individually flashing idle or startup kernels. If your idle or startup kernel contains subkernels, it will automatically compile as a ``.tar``, which you only need to flash to the master. - fix unreliable DRTIO master-satellite links? -------------------------------------------- diff --git a/doc/manual/flashing.rst b/doc/manual/flashing.rst index bf14d3296..c5584da9a 100644 --- a/doc/manual/flashing.rst +++ b/doc/manual/flashing.rst @@ -78,29 +78,25 @@ On Windows Writing the flash ----------------- -First ensure the board is connected to your computer. In the case of Kasli, the JTAG adapter is integrated into the Kasli board; for flashing (and debugging) you can simply connect your computer to the micro-USB connector on the Kasli front panel. For :ref:`Zynq devices `, which use :mod:`~artiq.frontend.artiq_coremgmt` to flash over network, an Ethernet connection and an IP address, supplied either with the ``-D`` option or in your :ref:`device database `, are sufficient. +If your device is already accessible over the network, all you need is an Ethernet connection and a correct IP address (supplied either with the ``-D`` option or in :ref:`your device database `). :: -For Kasli-SoC, ZC706 or EBAZ4205: - :: + $ artiq_coremgmt [-D IP_address] flash + $ artiq_coremgmt [-D IP_address] reboot - $ artiq_coremgmt [-D IP_address] config write -f boot /boot.bin - $ artiq_coremgmt reboot +If the device is not reachable due to corrupted firmware or networking problems, binaries can be loaded manually. On Kasli or KC705, connect the board directly to your computer by JTAG USB and use :mod:`~artiq.frontend.artiq_flash`, as follows: :: - If the device is not reachable due to corrupted firmware or networking problems, extract the SD card and copy ``boot.bin`` onto it manually. + $ artiq_flash [-t kc705] -d -For Kasli: - :: +Note the micro-USB in the Kasli front panel. On KC705, the SW13 switches need to be set to 00001. - $ artiq_flash -d +For Zynq devices (Kasli-SoC, ZC706 or EBAZ4205), extract the SD card and copy ``boot.bin`` onto it manually. -For KC705: - :: +Writing to satellite devices +^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - $ artiq_flash -t kc705 -d +Satellite devices can at any time be flashed directly through the SD card or :mod:`~artiq.frontend.artiq_flash`, as applicable. Satellite devices do not support individual networking and do not have IP addresses. If your DRTIO system is up and running and the routing table is in place, on the other hand, they can be flashed through the master's network connection: :: - The SW13 switches need to be set to 00001. - -Flashing over network is also possible for Kasli and KC705, assuming IP networking has already been set up. In this case, the ``-H HOSTNAME`` option is used; see the entry for :mod:`~artiq.frontend.artiq_flash` in the :ref:`Utilities ` reference. + $ artiq_coremgmt [-D IP_address] -s flash .. _connecting-uart: From 07ba7a075fabec93d8a169f2e3b040d76e8a4548 Mon Sep 17 00:00:00 2001 From: Sebastien Bourdeauducq Date: Fri, 22 Nov 2024 14:23:31 +0800 Subject: [PATCH 103/111] RELEASE_NOTES: formatting --- RELEASE_NOTES.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/RELEASE_NOTES.rst b/RELEASE_NOTES.rst index 373910864..0b035930c 100644 --- a/RELEASE_NOTES.rst +++ b/RELEASE_NOTES.rst @@ -24,7 +24,7 @@ ARTIQ-9 (Unreleased) * New core device driver for the AD9834 DDS, tested with the ZonRi Technology Co., Ltd. AD9834-Module. * Support for coredevice reflashing through the new ``flash`` tool in ``artiq_coremgmt``. * ``artiq_coremgmt`` now supports configuring satellites. -* artiq.coredevice.fmcdio_vhdci_eem has been removed. +* ``artiq.coredevice.fmcdio_vhdci_eem`` has been removed. ARTIQ-8 ------- From fd664e82d1738870e9886933c88217644f2c0efd Mon Sep 17 00:00:00 2001 From: architeuthidae Date: Fri, 22 Nov 2024 11:16:12 +0100 Subject: [PATCH 104/111] doc: Windows netstat fix --- doc/manual/faq.rst | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/doc/manual/faq.rst b/doc/manual/faq.rst index 6903c8e65..398fcdc2a 100644 --- a/doc/manual/faq.rst +++ b/doc/manual/faq.rst @@ -133,9 +133,11 @@ You can use the command ``netstat`` to list the ports currently in use on your s $ netstat -anp --inet | grep "" $ lsof -i: -On Windows, use :: +On Windows, you can list ports with: :: - $ netstat -ano -p ip | find "" + $ netstat -ano -p TCP + +Use your preferred method to search through the output; suitable commands will vary by environment (e.g. ``grep`` in an MSYS2 shell, ``Select-String`` in PowerShell, ``find`` in the Windows command line, etc.) In all cases, if there are no results, the port isn't in use and should be free for new processes. @@ -148,7 +150,7 @@ Otherwise, either the running process must be stopped, or you'll have to set dif On Windows, use ``netstat`` again to identify the process ID, and then feed it into ``taskkill``, e.g.: :: - $ netstat -ano | find "" + $ netstat -ano -p TCP $ taskkill /F /PID diagnose and fix sequence errors? From 145a9732139208413b0dba1a5f76aabc92e5b8eb Mon Sep 17 00:00:00 2001 From: Florian Agbuya Date: Mon, 25 Nov 2024 11:38:58 +0800 Subject: [PATCH 105/111] flake: factor out Qt paths into exported variable Signed-off-by: Florian Agbuya --- flake.nix | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/flake.nix b/flake.nix index a7e461e66..676699906 100644 --- a/flake.nix +++ b/flake.nix @@ -49,6 +49,11 @@ artiqVersion = (builtins.toString artiqVersionMajor) + "." + (builtins.toString artiqVersionMinor) + "+" + artiqVersionId + ".beta"; artiqRev = self.sourceInfo.rev or "unknown"; + qtPaths = { + QT_PLUGIN_PATH = "${pkgs.qt6.qtbase}/${pkgs.qt6.qtbase.dev.qtPluginPrefix}:${pkgs.qt6.qtsvg}/${pkgs.qt6.qtbase.dev.qtPluginPrefix}"; + QML2_IMPORT_PATH = "${pkgs.qt6.qtbase}/${pkgs.qt6.qtbase.dev.qtQmlPrefix}"; + }; + rust = pkgs.rust-bin.nightly."2021-09-01".default.override { extensions = [ "rust-src" ]; targets = [ ]; @@ -414,7 +419,7 @@ }; }; - inherit makeArtiqBoardPackage openocd-bscanspi-f; + inherit qtPaths makeArtiqBoardPackage openocd-bscanspi-f; defaultPackage.x86_64-linux = pkgs.python3.withPackages(ps: [ packages.x86_64-linux.artiq ]); @@ -446,8 +451,8 @@ ]; shellHook = '' export LIBARTIQ_SUPPORT=`libartiq-support` - export QT_PLUGIN_PATH=${pkgs.qt6.qtbase}/${pkgs.qt6.qtbase.dev.qtPluginPrefix}:${pkgs.qt6.qtsvg}/${pkgs.qt6.qtbase.dev.qtPluginPrefix} - export QML2_IMPORT_PATH=${pkgs.qt6.qtbase}/${pkgs.qt6.qtbase.dev.qtQmlPrefix} + export QT_PLUGIN_PATH=${qtPaths.QT_PLUGIN_PATH} + export QML2_IMPORT_PATH=${qtPaths.QML2_IMPORT_PATH} export PYTHONPATH=`git rev-parse --show-toplevel`:$PYTHONPATH ''; }; From de8f8af3dd5f9b858343b38df8088b4ccbe98c81 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Bourdeauducq?= Date: Tue, 26 Nov 2024 14:57:20 +0800 Subject: [PATCH 106/111] drtio: add InjectionRequest to expects_response --- artiq/firmware/libproto_artiq/drtioaux_proto.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/artiq/firmware/libproto_artiq/drtioaux_proto.rs b/artiq/firmware/libproto_artiq/drtioaux_proto.rs index 4891bedf1..2883895a2 100644 --- a/artiq/firmware/libproto_artiq/drtioaux_proto.rs +++ b/artiq/firmware/libproto_artiq/drtioaux_proto.rs @@ -956,7 +956,8 @@ impl Packet { Packet::DmaAddTraceReply { .. } | Packet::DmaRemoveTraceReply { .. } | Packet::DmaPlaybackReply { .. } | Packet::SubkernelLoadRunReply { .. } | Packet::SubkernelMessageAck { .. } | Packet::DmaPlaybackStatus { .. } | - Packet::SubkernelFinished { .. } | Packet::CoreMgmtDropLinkAck { .. } => false, + Packet::SubkernelFinished { .. } | Packet::CoreMgmtDropLinkAck { .. } | + Packet::InjectionRequest { .. } => false, _ => true } } From 592f0a770842a5e11da157099c35b5fa7af1c9f7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Bourdeauducq?= Date: Thu, 28 Nov 2024 18:55:19 +0800 Subject: [PATCH 107/111] afws_client: sync --- artiq/frontend/afws_client.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/artiq/frontend/afws_client.py b/artiq/frontend/afws_client.py index 50c3de832..01e7c3b20 100755 --- a/artiq/frontend/afws_client.py +++ b/artiq/frontend/afws_client.py @@ -79,13 +79,16 @@ class Client: self.writer.write((" ".join(command) + "\n").encode()) async def read_line(self): - return (await self.reader.readline()).decode("ascii") + line = (await self.reader.readline()).decode("ascii") + if not line and self.reader.at_eof(): + raise ConnectionError("connection was closed unexpectedly") + return line async def read_reply(self): - return (await self.reader.readline()).decode("ascii").split() + return (await self.read_line()).split() async def read_json(self): - return json.loads((await self.reader.readline()).decode("ascii")) + return json.loads((await self.read_line())) async def login(self, username, password): await self.send_command("LOGIN", username, password) From 6097a32f4ac34cfb23979590007c4e45a7f3cd11 Mon Sep 17 00:00:00 2001 From: Charlie Root Date: Sun, 1 Dec 2024 01:26:44 +0100 Subject: [PATCH 108/111] flake: move defaultPackage. to packages..default change deprecated defaultPackage.x86_64-linux to packages.x86_64-linux.default, remove unneeded ps --- flake.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flake.nix b/flake.nix index 676699906..b541fe3a0 100644 --- a/flake.nix +++ b/flake.nix @@ -421,7 +421,7 @@ inherit qtPaths makeArtiqBoardPackage openocd-bscanspi-f; - defaultPackage.x86_64-linux = pkgs.python3.withPackages(ps: [ packages.x86_64-linux.artiq ]); + packages.x86_64-linux.default = pkgs.python3.withPackages(_: [ packages.x86_64-linux.artiq ]); # Main development shell with everything you need to develop ARTIQ on Linux. # The current copy of the ARTIQ sources is added to PYTHONPATH so changes can be tested instantly. From 3db8d2310cb45989a7fb4408508013ea1c03f7b9 Mon Sep 17 00:00:00 2001 From: Sebastien Bourdeauducq Date: Wed, 11 Dec 2024 13:29:46 +0800 Subject: [PATCH 109/111] flake: update dependencies --- flake.lock | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/flake.lock b/flake.lock index 2cd1b97ab..9a4eaee4e 100644 --- a/flake.lock +++ b/flake.lock @@ -44,11 +44,11 @@ }, "nixpkgs": { "locked": { - "lastModified": 1731319897, - "narHash": "sha256-PbABj4tnbWFMfBp6OcUK5iGy1QY+/Z96ZcLpooIbuEI=", + "lastModified": 1733759999, + "narHash": "sha256-463SNPWmz46iLzJKRzO3Q2b0Aurff3U1n0nYItxq7jU=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "dc460ec76cbff0e66e269457d7b728432263166c", + "rev": "a73246e2eef4c6ed172979932bc80e1404ba2d56", "type": "github" }, "original": { @@ -97,11 +97,11 @@ ] }, "locked": { - "lastModified": 1728371104, - "narHash": "sha256-PPnAyDedUQ7Og/Cby9x5OT9wMkNGTP8GS53V6N/dk4w=", + "lastModified": 1733319649, + "narHash": "sha256-ATJV2UV9FXEiTF6/1BvZ2HmB0goF5TZ2ytgRBwD/BGg=", "owner": "m-labs", "repo": "sipyco", - "rev": "094a6cd63ffa980ef63698920170e50dc9ba77fd", + "rev": "27312727bdb8a182bd6e222e4cbdd3f39ae41d4e", "type": "github" }, "original": { From 2508ff4812107d77876719731c5f2d53800ffafc Mon Sep 17 00:00:00 2001 From: Charlie Root Date: Sun, 1 Dec 2024 00:58:34 +0100 Subject: [PATCH 110/111] flake: clean up devShells Move both devShells into devshells.<...>, switch from buildInputs to packages --- flake.nix | 104 +++++++++++++++++++++++++++++++----------------------- 1 file changed, 59 insertions(+), 45 deletions(-) diff --git a/flake.nix b/flake.nix index b541fe3a0..4cf61fb97 100644 --- a/flake.nix +++ b/flake.nix @@ -423,52 +423,66 @@ packages.x86_64-linux.default = pkgs.python3.withPackages(_: [ packages.x86_64-linux.artiq ]); - # Main development shell with everything you need to develop ARTIQ on Linux. - # The current copy of the ARTIQ sources is added to PYTHONPATH so changes can be tested instantly. - # Additionally, executable wrappers that import the current ARTIQ sources for the ARTIQ frontends - # are added to PATH. - devShells.x86_64-linux.default = pkgs.mkShell { - name = "artiq-dev-shell"; - buildInputs = [ - (pkgs.python3.withPackages(ps: with packages.x86_64-linux; [ migen misoc ps.paramiko microscope ps.packaging ] ++ artiq.propagatedBuildInputs )) - rust - pkgs.llvmPackages_15.clang-unwrapped - pkgs.llvm_15 - pkgs.lld_15 - pkgs.git - artiq-frontend-dev-wrappers - # To manually run compiler tests: - pkgs.lit - pkgs.outputcheck - libartiq-support - # use the vivado-env command to enter a FHS shell that lets you run the Vivado installer - packages.x86_64-linux.vivadoEnv - packages.x86_64-linux.vivado - packages.x86_64-linux.openocd-bscanspi - pkgs.python3Packages.sphinx pkgs.python3Packages.sphinx_rtd_theme pkgs.pdf2svg - pkgs.python3Packages.sphinx-argparse pkgs.python3Packages.sphinxcontrib-wavedrom latex-artiq-manual - pkgs.python3Packages.sphinxcontrib-tikz - ]; - shellHook = '' - export LIBARTIQ_SUPPORT=`libartiq-support` - export QT_PLUGIN_PATH=${qtPaths.QT_PLUGIN_PATH} - export QML2_IMPORT_PATH=${qtPaths.QML2_IMPORT_PATH} - export PYTHONPATH=`git rev-parse --show-toplevel`:$PYTHONPATH - ''; - }; + devShells.x86_64-linux = { + # Main development shell with everything you need to develop ARTIQ on Linux. + # The current copy of the ARTIQ sources is added to PYTHONPATH so changes can be tested instantly. + # Additionally, executable wrappers that import the current ARTIQ sources for the ARTIQ frontends + # are added to PATH. + default = pkgs.mkShell { + name = "artiq-dev-shell"; + packages = with pkgs; [ + git + lit + lld_15 + llvm_15 + llvmPackages_15.clang-unwrapped + outputcheck + pdf2svg - # Lighter development shell optimized for building firmware and flashing boards. - devShells.x86_64-linux.boards = pkgs.mkShell { - name = "artiq-boards-shell"; - buildInputs = [ - (pkgs.python3.withPackages(ps: with packages.x86_64-linux; [ migen misoc artiq ps.packaging ps.paramiko ])) - rust - pkgs.llvmPackages_15.clang-unwrapped - pkgs.llvm_15 - pkgs.lld_15 - packages.x86_64-linux.vivado - packages.x86_64-linux.openocd-bscanspi - ]; + python3Packages.sphinx + python3Packages.sphinx-argparse + python3Packages.sphinxcontrib-tikz + python3Packages.sphinxcontrib-wavedrom + python3Packages.sphinx_rtd_theme + + (python3.withPackages(ps: [ migen misoc microscope ps.packaging ps.paramiko ] ++ artiq.propagatedBuildInputs )) + ] ++ + [ + latex-artiq-manual + rust + artiq-frontend-dev-wrappers + + # To manually run compiler tests: + libartiq-support + + # use the vivado-env command to enter a FHS shell that lets you run the Vivado installer + packages.x86_64-linux.vivadoEnv + packages.x86_64-linux.vivado + packages.x86_64-linux.openocd-bscanspi + ]; + shellHook = '' + export LIBARTIQ_SUPPORT=`libartiq-support` + export QT_PLUGIN_PATH=${qtPaths.QT_PLUGIN_PATH} + export QML2_IMPORT_PATH=${qtPaths.QML2_IMPORT_PATH} + export PYTHONPATH=`git rev-parse --show-toplevel`:$PYTHONPATH + ''; + }; + # Lighter development shell optimized for building firmware and flashing boards. + boards = pkgs.mkShell { + name = "artiq-boards-shell"; + packages = [ + rust + + pkgs.llvmPackages_15.clang-unwrapped + pkgs.llvm_15 + pkgs.lld_15 + + packages.x86_64-linux.vivado + packages.x86_64-linux.openocd-bscanspi + + (pkgs.python3.withPackages(ps: [ migen misoc artiq ps.packaging ps.paramiko ])) + ]; + }; }; packages.aarch64-linux = { From 67dde30625909b8e670dbb78a19371dd91383688 Mon Sep 17 00:00:00 2001 From: Charlie Root Date: Wed, 11 Dec 2024 16:11:59 +0100 Subject: [PATCH 111/111] flake: clean up qt paths --- flake.nix | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/flake.nix b/flake.nix index 4cf61fb97..c6062ba11 100644 --- a/flake.nix +++ b/flake.nix @@ -49,9 +49,12 @@ artiqVersion = (builtins.toString artiqVersionMajor) + "." + (builtins.toString artiqVersionMinor) + "+" + artiqVersionId + ".beta"; artiqRev = self.sourceInfo.rev or "unknown"; - qtPaths = { - QT_PLUGIN_PATH = "${pkgs.qt6.qtbase}/${pkgs.qt6.qtbase.dev.qtPluginPrefix}:${pkgs.qt6.qtsvg}/${pkgs.qt6.qtbase.dev.qtPluginPrefix}"; - QML2_IMPORT_PATH = "${pkgs.qt6.qtbase}/${pkgs.qt6.qtbase.dev.qtQmlPrefix}"; + qtPaths = let + inherit (pkgs.qt6) qtbase qtsvg; + inherit (qtbase.dev) qtPluginPrefix qtQmlPrefix; + in { + QT_PLUGIN_PATH = "${qtbase}/${qtPluginPrefix}:${qtsvg}/${qtPluginPrefix}"; + QML2_IMPORT_PATH = "${qtbase}/${qtQmlPrefix}"; }; rust = pkgs.rust-bin.nightly."2021-09-01".default.override {