Merge branch 'master' into nac3

RELEASE_NOTES.rst

@@ -32,9 +32,6 @@ Highlights:
 * Previously detected RTIO async errors are reported to the host after each kernel terminates and a
   warning is logged. The warning is additional to the one already printed in the core device log upon
   detection of the error.
-* HDF5 options can now be passed when creating datasets with ``set_dataset``. This allows
-  in particular to use transparent compression filters as follows:
-  ``set_dataset(name, value, hdf5_options={"compression": "gzip"})``.
 * Removed worker DB warning for writing a dataset that is also in the archive
 
 Breaking changes:
@@ -51,10 +48,6 @@ Breaking changes:
   calling `ADF5356.init()`.
 * DRTIO: Changed message alignment from 32-bits to 64-bits.
 * The deprecated ``set_dataset(..., save=...)`` is no longer supported.
-* The internal dataset representation was changed to support tracking HDF5 options like e.g.
-  a compression method. This requires changes to code reading the dataset persistence file
-  (``dataset_db.pyon``) and to custom applets.
-
 
 ARTIQ-6
 -------
@@ -129,7 +122,6 @@ Breaking changes:
 * Experiment classes with underscore-prefixed names are now ignored when ``artiq_client``
   determines which experiment to submit (consistent with ``artiq_run``).
 
-
 ARTIQ-5
 -------
 

artiq/applets/big_number.py

@@ -13,7 +13,7 @@ class NumberWidget(QtWidgets.QLCDNumber):
 
     def data_changed(self, data, mods):
         try:
-            n = float(data[self.dataset_name]["value"])
+            n = float(data[self.dataset_name][1])
         except (KeyError, ValueError, TypeError):
             n = "---"
         self.display(n)

artiq/applets/image.py

@@ -13,7 +13,7 @@ class Image(pyqtgraph.ImageView):
 
     def data_changed(self, data, mods):
         try:
-            img = data[self.args.img]["value"]
+            img = data[self.args.img][1]
         except KeyError:
             return
         self.setImage(img)

artiq/applets/plot_hist.py

@@ -17,11 +17,11 @@ class HistogramPlot(pyqtgraph.PlotWidget):
 
     def data_changed(self, data, mods, title):
         try:
-            y = data[self.args.y]["value"]
+            y = data[self.args.y][1]
             if self.args.x is None:
                 x = None
             else:
-                x = data[self.args.x]["value"]
+                x = data[self.args.x][1]
         except KeyError:
             return
         if x is None:

artiq/applets/plot_xy.py

@@ -6,7 +6,6 @@ from PyQt5.QtCore import QTimer
 import pyqtgraph
 
 from artiq.applets.simple import TitleApplet
-from artiq.master.databases import make_dataset as empty_dataset
 
 
 class XYPlot(pyqtgraph.PlotWidget):
@@ -22,14 +21,14 @@ class XYPlot(pyqtgraph.PlotWidget):
 
     def data_changed(self, data, mods, title):
         try:
-            y = data[self.args.y]["value"]
+            y = data[self.args.y][1]
         except KeyError:
             return
-        x = data.get(self.args.x, empty_dataset())["value"]
+        x = data.get(self.args.x, (False, None))[1]
         if x is None:
             x = np.arange(len(y))
-        error = data.get(self.args.error, empty_dataset())["value"]
+        error = data.get(self.args.error, (False, None))[1]
-        fit = data.get(self.args.fit, empty_dataset())["value"]
+        fit = data.get(self.args.fit, (False, None))[1]
 
         if not len(y) or len(y) != len(x):
             self.mismatch['X values'] = True

artiq/applets/plot_xy_hist.py

@@ -126,9 +126,9 @@ class XYHistPlot(QtWidgets.QSplitter):
 
     def data_changed(self, data, mods):
         try:
-            xs = data[self.args.xs]["value"]
+            xs = data[self.args.xs][1]
-            histogram_bins = data[self.args.histogram_bins]["value"]
+            histogram_bins = data[self.args.histogram_bins][1]
-            histograms_counts = data[self.args.histograms_counts]["value"]
+            histograms_counts = data[self.args.histograms_counts][1]
         except KeyError:
             return
         if len(xs) != histograms_counts.shape[0]:

artiq/applets/simple.py

@@ -10,7 +10,6 @@ from sipyco.sync_struct import Subscriber, process_mod
 from sipyco import pyon
 from sipyco.pipe_ipc import AsyncioChildComm
 
-from artiq.master.databases import make_dataset as empty_dataset
 
 logger = logging.getLogger(__name__)
 
@@ -252,7 +251,7 @@ class TitleApplet(SimpleApplet):
 
     def emit_data_changed(self, data, mod_buffer):
         if self.args.title is not None:
-            title_values = {k.replace(".", "/"): data.get(k, empty_dataset())["value"]
+            title_values = {k.replace(".", "/"): data.get(k, (False, None))[1]
                             for k in self.dataset_title}
             try:
                 title = self.args.title.format(**title_values)

artiq/browser/datasets.py

@@ -104,8 +104,8 @@ class DatasetsDock(QtWidgets.QDockWidget):
             idx = self.table_model_filter.mapToSource(idx[0])
             key = self.table_model.index_to_key(idx)
             if key is not None:
-                dataset = self.table_model.backing_store[key]
+                persist, value = self.table_model.backing_store[key]
-                asyncio.ensure_future(self._upload_dataset(key, dataset["value"]))
+                asyncio.ensure_future(self._upload_dataset(key, value))
 
     def save_state(self):
         return bytes(self.table.header().saveState())

artiq/coredevice/comm_kernel.py

@@ -571,29 +571,33 @@ class CommKernel:
 
             self._write_header(Request.RPCException)
 
+            # Note: instead of sending strings, we send object ID
+            # This is to avoid the need of allocatio on the device side
+            # This is a special case: this only applies to exceptions
             if hasattr(exn, "artiq_core_exception"):
                 exn = exn.artiq_core_exception
-                self._write_string(exn.name)
+                self._write_int32(embedding_map.store_str(exn.name))
-                self._write_string(self._truncate_message(exn.message))
+                self._write_int32(embedding_map.store_str(self._truncate_message(exn.message)))
                 for index in range(3):
                     self._write_int64(exn.param[index])
 
                 filename, line, column, function = exn.traceback[-1]
-                self._write_string(filename)
+                self._write_int32(embedding_map.store_str(filename))
                 self._write_int32(line)
                 self._write_int32(column)
-                self._write_string(function)
+                self._write_int32(embedding_map.store_str(function))
             else:
                 exn_type = type(exn)
                 if exn_type in (ZeroDivisionError, ValueError, IndexError, RuntimeError) or \
                         hasattr(exn, "artiq_builtin"):
-                    self._write_string("0:{}".format(exn_type.__name__))
+                    name = "0:{}".format(exn_type.__name__)
                 else:
                     exn_id = embedding_map.store_object(exn_type)
-                    self._write_string("{}:{}.{}".format(exn_id,
+                    name = "{}:{}.{}".format(exn_id,
-                                                         exn_type.__module__,
+                                             exn_type.__module__,
-                                                         exn_type.__qualname__))
+                                             exn_type.__qualname__)
-                self._write_string(self._truncate_message(str(exn)))
+                self._write_int32(embedding_map.store_str(name))
+                self._write_int32(embedding_map.store_str(self._truncate_message(str(exn))))
                 for index in range(3):
                     self._write_int64(0)
 
@@ -604,10 +608,10 @@ class CommKernel:
                     ((filename, line, function, _), ) = tb
                 else:
                     assert False
-                self._write_string(filename)
+                self._write_int32(embedding_map.store_str(filename))
                 self._write_int32(line)
                 self._write_int32(-1)  # column not known
-                self._write_string(function)
+                self._write_int32(embedding_map.store_str(function))
             self._flush()
         else:
             logger.debug("rpc service: %d %r %r = %r",
@@ -619,28 +623,59 @@ class CommKernel:
             self._flush()
 
     def _serve_exception(self, embedding_map, symbolizer, demangler):
-        name = self._read_string()
+        exception_count = self._read_int32()
-        message = self._read_string()
+        nested_exceptions = []
-        params = [self._read_int64() for _ in range(3)]
 
-        filename = self._read_string()
+        def read_exception_string():
-        line = self._read_int32()
+            # note: if length == -1, the following int32 is the object key
-        column = self._read_int32()
+            length = self._read_int32()
-        function = self._read_string()
+            if length == -1:
+                return embedding_map.retrieve_str(self._read_int32())
+            else:
+                return self._read(length).decode("utf-8")
+
+        for _ in range(exception_count):
+            name = embedding_map.retrieve_str(self._read_int32())
+            message = read_exception_string()
+            params = [self._read_int64() for _ in range(3)]
+
+            filename = read_exception_string()
+            line = self._read_int32()
+            column = self._read_int32()
+            function = read_exception_string()
+            nested_exceptions.append([name, message, params,
+                                      filename, line, column, function])
+
+        demangled_names = demangler([ex[6] for ex in nested_exceptions])
+        for i in range(exception_count):
+            nested_exceptions[i][6] = demangled_names[i]
+
+        exception_info = []
+        for _ in range(exception_count):
+            sp = self._read_int32()
+            initial_backtrace = self._read_int32()
+            current_backtrace = self._read_int32()
+            exception_info.append((sp, initial_backtrace, current_backtrace))
+
+        backtrace = []
+        stack_pointers = []
+        for _ in range(self._read_int32()):
+            backtrace.append(self._read_int32())
+            stack_pointers.append(self._read_int32())
 
-        backtrace = [self._read_int32() for _ in range(self._read_int32())]
         self._process_async_error()
 
-        traceback = list(reversed(symbolizer(backtrace))) + \
+        traceback = list(symbolizer(backtrace))
-            [(filename, line, column, *demangler([function]), None)]
+        core_exn = exceptions.CoreException(nested_exceptions, exception_info,
-        core_exn = exceptions.CoreException(name, message, params, traceback)
+                                            traceback, stack_pointers)
 
         if core_exn.id == 0:
             python_exn_type = getattr(exceptions, core_exn.name.split('.')[-1])
         else:
             python_exn_type = embedding_map.retrieve_object(core_exn.id)
 
-        python_exn = python_exn_type(message.format(*params))
+        python_exn = python_exn_type(
+            nested_exceptions[-1][1].format(*nested_exceptions[0][2]))
         python_exn.artiq_core_exception = core_exn
         raise python_exn
 

artiq/coredevice/exceptions.py

@@ -16,50 +16,94 @@ AssertionError = builtins.AssertionError
 
 class CoreException:
     """Information about an exception raised or passed through the core device."""
+    def __init__(self, exceptions, exception_info, traceback, stack_pointers):
+        self.exceptions = exceptions
+        self.exception_info = exception_info
+        self.traceback = list(traceback)
+        self.stack_pointers = stack_pointers
 
-    def __init__(self, name, message, params, traceback):
+        first_exception = exceptions[0]
+        name = first_exception[0]
         if ':' in name:
             exn_id, self.name = name.split(':', 2)
             self.id = int(exn_id)
         else:
             self.id, self.name = 0, name
-        self.message, self.params = message, params
+        self.message = first_exception[1]
-        self.traceback = list(traceback)
+        self.params = first_exception[2]
+
+    def append_backtrace(self, record, inlined=False):
+        filename, line, column, function, address = record
+        stub_globals = {"__name__": filename, "__loader__": source_loader}
+        source_line = linecache.getline(filename, line, stub_globals)
+        indentation = re.search(r"^\s*", source_line).end()
+
+        if address is None:
+            formatted_address = ""
+        elif inlined:
+            formatted_address = " (inlined)"
+        else:
+            formatted_address = " (RA=+0x{:x})".format(address)
+
+        filename = filename.replace(artiq_dir, "<artiq>")
+        lines = []
+        if column == -1:
+            lines.append("    {}".format(source_line.strip() if source_line else "<unknown>"))
+            lines.append("  File \"{file}\", line {line}, in {function}{address}".
+                         format(file=filename, line=line, function=function,
+                                address=formatted_address))
+        else:
+            lines.append("    {}^".format(" " * (column - indentation)))
+            lines.append("    {}".format(source_line.strip() if source_line else "<unknown>"))
+            lines.append("  File \"{file}\", line {line}, column {column},"
+                         " in {function}{address}".
+                         format(file=filename, line=line, column=column + 1,
+                                function=function, address=formatted_address))
+        return lines
+
+    def single_traceback(self, exception_index):
+        # note that we insert in reversed order
+        lines = []
+        last_sp = 0
+        start_backtrace_index = self.exception_info[exception_index][1]
+        zipped = list(zip(self.traceback[start_backtrace_index:],
+                          self.stack_pointers[start_backtrace_index:]))
+        exception = self.exceptions[exception_index]
+        name = exception[0]
+        message = exception[1]
+        params = exception[2]
+        if ':' in name:
+            exn_id, name = name.split(':', 2)
+            exn_id = int(exn_id)
+        else:
+            exn_id = 0
+        lines.append("{}({}): {}".format(name, exn_id, message.format(*params)))
+        zipped.append(((exception[3], exception[4], exception[5], exception[6],
+                       None, []), None))
+
+        for ((filename, line, column, function, address, inlined), sp) in zipped:
+            # backtrace of nested exceptions may be discontinuous
+            # but the stack pointer must increase monotonically
+            if sp is not None and sp <= last_sp:
+                continue
+            last_sp = sp
+
+            for record in reversed(inlined):
+                lines += self.append_backtrace(record, True)
+            lines += self.append_backtrace((filename, line, column, function,
+                                            address))
+
+        lines.append("Traceback (most recent call first):")
+
+        return "\n".join(reversed(lines))
 
     def __str__(self):
-        lines = []
+        tracebacks = [self.single_traceback(i) for i in range(len(self.exceptions))]
-        lines.append("Core Device Traceback (most recent call last):")
+        traceback_str = ('\n\nDuring handling of the above exception, ' +
-        last_address = 0
+                        'another exception occurred:\n\n').join(tracebacks)
-        for (filename, line, column, function, address) in self.traceback:
+        return 'Core Device Traceback:\n' +\
-            stub_globals = {"__name__": filename, "__loader__": source_loader}
+                traceback_str +\
-            source_line = linecache.getline(filename, line, stub_globals)
+                '\n\nEnd of Code Device Traceback\n'
-            indentation = re.search(r"^\s*", source_line).end()
-
-            if address is None:
-                formatted_address = ""
-            elif address == last_address:
-                formatted_address = " (inlined)"
-            else:
-                formatted_address = " (RA=+0x{:x})".format(address)
-            last_address = address
-
-            filename = filename.replace(artiq_dir, "<artiq>")
-            if column == -1:
-                lines.append("  File \"{file}\", line {line}, in {function}{address}".
-                             format(file=filename, line=line, function=function,
-                                    address=formatted_address))
-                lines.append("    {}".format(source_line.strip() if source_line else "<unknown>"))
-            else:
-                lines.append("  File \"{file}\", line {line}, column {column},"
-                             " in {function}{address}".
-                             format(file=filename, line=line, column=column + 1,
-                                    function=function, address=formatted_address))
-                lines.append("    {}".format(source_line.strip() if source_line else "<unknown>"))
-                lines.append("    {}^".format(" " * (column - indentation)))
-
-        lines.append("{}({}): {}".format(self.name, self.id,
-                                         self.message.format(*self.params)))
-        return "\n".join(lines)
 
 
 class InternalError(Exception):

artiq/dashboard/datasets.py

@@ -146,16 +146,16 @@ class Creator(QtWidgets.QDialog):
 
 
 class Model(DictSyncTreeSepModel):
-    def __init__(self, init):
+    def __init__(self,  init):
-        DictSyncTreeSepModel.__init__(
+        DictSyncTreeSepModel.__init__(self, ".",
-            self, ".", ["Dataset", "Persistent", "Value"], init
+                                      ["Dataset", "Persistent", "Value"],
-        )
+                                      init)
 
     def convert(self, k, v, column):
         if column == 1:
-            return "Y" if v["persist"] else "N"
+            return "Y" if v[0] else "N"
         elif column == 2:
-            return short_format(v["value"])
+            return short_format(v[1])
         else:
             raise ValueError
 
@@ -223,8 +223,8 @@ class DatasetsDock(QtWidgets.QDockWidget):
             idx = self.table_model_filter.mapToSource(idx[0])
             key = self.table_model.index_to_key(idx)
             if key is not None:
-                dataset = self.table_model.backing_store[key]
+                persist, value = self.table_model.backing_store[key]
-                t = type(dataset["value"])
+                t = type(value)
                 if np.issubdtype(t, np.number):
                     dialog_cls = NumberEditor
                 elif np.issubdtype(t, np.bool_):
@@ -235,7 +235,7 @@ class DatasetsDock(QtWidgets.QDockWidget):
                     logger.error("Cannot edit dataset %s: "
                                  "type %s is not supported", key, t)
                     return
-                dialog_cls(self, self.dataset_ctl, key, dataset["value"]).open()
+                dialog_cls(self, self.dataset_ctl, key, value).open()
 
     def delete_clicked(self):
         idx = self.table.selectedIndexes()

artiq/firmware/Cargo.lock

@@ -253,6 +253,7 @@ dependencies = [
  "byteorder",
  "cslice",
  "dyld",
+ "eh",
  "failure",
  "failure_derive",
  "io",

artiq/firmware/ksupport/api.rs

@@ -117,7 +117,8 @@ static mut API: &'static [(&'static str, *const ())] = &[
     api!(_Unwind_Resume = ::unwind::_Unwind_Resume),
     api!(__artiq_personality = ::eh_artiq::personality),
     api!(__artiq_raise = ::eh_artiq::raise),
-    api!(__artiq_reraise = ::eh_artiq::reraise),
+    api!(__artiq_resume = ::eh_artiq::resume),
+    api!(__artiq_end_catch = ::eh_artiq::end_catch),
 
     /* proxified syscalls */
     api!(core_log),

artiq/firmware/ksupport/eh_artiq.rs

@@ -10,12 +10,15 @@
 // except according to those terms.
 #![allow(non_camel_case_types)]
 
-use core::{ptr, mem};
+use core::mem;
-use cslice::CSlice;
+use cslice::AsCSlice;
 use unwind as uw;
 use libc::{c_int, c_void};
 
-use eh::dwarf::{self, EHAction, EHContext};
+use eh::{self, dwarf::{self, EHAction, EHContext}};
+
+pub type Exception<'a> = eh::eh_artiq::Exception<'a>;
+pub type StackPointerBacktrace = eh::eh_artiq::StackPointerBacktrace;
 
 type _Unwind_Stop_Fn = extern "C" fn(version: c_int,
                                      actions: uw::_Unwind_Action,
@@ -30,40 +33,74 @@ extern {
                             stop_parameter: *mut c_void) -> uw::_Unwind_Reason_Code;
 }
 
-#[repr(C)]
+pub static mut PAYLOAD_ADDRESS: usize = 0;
-#[derive(Clone, Copy)]
-pub struct Exception<'a> {
-    pub name:     CSlice<'a, u8>,
-    pub file:     CSlice<'a, u8>,
-    pub line:     u32,
-    pub column:   u32,
-    pub function: CSlice<'a, u8>,
-    pub message:  CSlice<'a, u8>,
-    pub param:    [i64; 3]
-}
 
 const EXCEPTION_CLASS: uw::_Unwind_Exception_Class = 0x4d_4c_42_53_41_52_54_51; /* 'MLBSARTQ' */
 
+const MAX_INFLIGHT_EXCEPTIONS: usize = 10;
 const MAX_BACKTRACE_SIZE: usize = 128;
 
-#[repr(C)]
+struct ExceptionBuffer {
-struct ExceptionInfo {
+    // we need n _Unwind_Exception, because each will have their own private data
-    uw_exception:   uw::_Unwind_Exception,
+    uw_exceptions: [uw::_Unwind_Exception; MAX_INFLIGHT_EXCEPTIONS],
-    exception:      Option<Exception<'static>>,
+    exceptions: [Option<Exception<'static>>; MAX_INFLIGHT_EXCEPTIONS + 1],
-    handled:        bool,
+    exception_stack: [isize; MAX_INFLIGHT_EXCEPTIONS + 1],
-    backtrace:      [usize; MAX_BACKTRACE_SIZE],
+    // nested exceptions will share the backtrace buffer, treated as a tree
-    backtrace_size: usize
+    // backtrace contains a tuple of IP and SP
+    backtrace: [(usize, usize); MAX_BACKTRACE_SIZE],
+    backtrace_size: usize,
+    // stack pointers are stored to reconstruct backtrace for each exception
+    stack_pointers: [StackPointerBacktrace; MAX_INFLIGHT_EXCEPTIONS + 1],
+    // current allocated nested exceptions
+    exception_count: usize,
+}
+
+static mut EXCEPTION_BUFFER: ExceptionBuffer = ExceptionBuffer {
+    uw_exceptions: [uw::_Unwind_Exception {
+        exception_class:   EXCEPTION_CLASS,
+        exception_cleanup: cleanup,
+        private:           [0; uw::unwinder_private_data_size],
+    }; MAX_INFLIGHT_EXCEPTIONS],
+    exceptions: [None; MAX_INFLIGHT_EXCEPTIONS + 1],
+    exception_stack: [-1; MAX_INFLIGHT_EXCEPTIONS + 1],
+    backtrace: [(0, 0); MAX_BACKTRACE_SIZE],
+    backtrace_size: 0,
+    stack_pointers: [StackPointerBacktrace {
+        stack_pointer: 0,
+        initial_backtrace_size: 0,
+        current_backtrace_size: 0
+    }; MAX_INFLIGHT_EXCEPTIONS + 1],
+    exception_count: 0
+};
+
+pub unsafe extern fn reset_exception_buffer(payload_addr: usize) {
+    EXCEPTION_BUFFER.uw_exceptions = [uw::_Unwind_Exception {
+        exception_class:   EXCEPTION_CLASS,
+        exception_cleanup: cleanup,
+        private:           [0; uw::unwinder_private_data_size],
+    }; MAX_INFLIGHT_EXCEPTIONS];
+    EXCEPTION_BUFFER.exceptions = [None; MAX_INFLIGHT_EXCEPTIONS + 1];
+    EXCEPTION_BUFFER.exception_stack = [-1; MAX_INFLIGHT_EXCEPTIONS + 1];
+    EXCEPTION_BUFFER.backtrace_size = 0;
+    EXCEPTION_BUFFER.exception_count = 0;
+    PAYLOAD_ADDRESS = payload_addr;
 }
 
 #[cfg(target_arch = "x86_64")]
 const UNWIND_DATA_REG: (i32, i32) = (0, 1); // RAX, RDX
+#[cfg(target_arch = "x86_64")]
+// actually this is not the SP, but frame pointer
+// but it serves its purpose, and getting SP will somehow cause segfault...
+const UNW_FP_REG: c_int = 12;
 
 #[cfg(any(target_arch = "riscv32"))]
 const UNWIND_DATA_REG: (i32, i32) = (10, 11); // X10, X11
+#[cfg(any(target_arch = "riscv32"))]
+const UNW_FP_REG: c_int = 2;
 
 #[export_name="__artiq_personality"]
 pub extern fn personality(version: c_int,
-                          actions: uw::_Unwind_Action,
+                          _actions: uw::_Unwind_Action,
                           uw_exception_class: uw::_Unwind_Exception_Class,
                           uw_exception: *mut uw::_Unwind_Exception,
                           context: *mut uw::_Unwind_Context)
@@ -85,133 +122,236 @@ pub extern fn personality(version: c_int,
             get_data_start: &|| uw::_Unwind_GetDataRelBase(context),
         };
 
-        let exception_info = &mut *(uw_exception as *mut ExceptionInfo);
+        let index = EXCEPTION_BUFFER.exception_stack[EXCEPTION_BUFFER.exception_count - 1];
-        let exception = &exception_info.exception.unwrap();
+        assert!(index != -1);
+        let exception = EXCEPTION_BUFFER.exceptions[index as usize].as_ref().unwrap();
+        let id = exception.id;
 
-        let name_ptr = exception.name.as_ptr();
+        let eh_action = match dwarf::find_eh_action(lsda, &eh_context, id) {
-        let len = exception.name.len();
-        let eh_action = match dwarf::find_eh_action(lsda, &eh_context, name_ptr, len) {
             Ok(action) => action,
             Err(_) => return uw::_URC_FATAL_PHASE1_ERROR,
         };
 
-        if actions as u32 & uw::_UA_SEARCH_PHASE as u32 != 0 {
+        match eh_action {
-            match eh_action {
+            EHAction::None => return uw::_URC_CONTINUE_UNWIND,
-                EHAction::None |
+            EHAction::Cleanup(lpad) |
-                EHAction::Cleanup(_) => return uw::_URC_CONTINUE_UNWIND,
+            EHAction::Catch(lpad) => {
-                EHAction::Catch(_) => return uw::_URC_HANDLER_FOUND,
+                // Pass a pair of the unwinder exception and ARTIQ exception
-                EHAction::Terminate => return uw::_URC_FATAL_PHASE1_ERROR,
+                // (which immediately follows).
-            }
+                uw::_Unwind_SetGR(context, UNWIND_DATA_REG.0,
-        } else {
+                                  uw_exception as uw::_Unwind_Word);
-            match eh_action {
+                uw::_Unwind_SetGR(context, UNWIND_DATA_REG.1,
-                EHAction::None => return uw::_URC_CONTINUE_UNWIND,
+                                  exception as *const _ as uw::_Unwind_Word);
-                EHAction::Cleanup(lpad) |
+                uw::_Unwind_SetIP(context, lpad);
-                EHAction::Catch(lpad) => {
+                return uw::_URC_INSTALL_CONTEXT;
-                    if actions as u32 & uw::_UA_HANDLER_FRAME as u32 != 0 {
-                        exception_info.handled = true
-                    }
-
-                    // Pass a pair of the unwinder exception and ARTIQ exception
-                    // (which immediately follows).
-                    uw::_Unwind_SetGR(context, UNWIND_DATA_REG.0,
-                                      uw_exception as uw::_Unwind_Word);
-                    uw::_Unwind_SetGR(context, UNWIND_DATA_REG.1,
-                                      exception as *const _ as uw::_Unwind_Word);
-                    uw::_Unwind_SetIP(context, lpad);
-                    return uw::_URC_INSTALL_CONTEXT;
-                }
-                EHAction::Terminate => return uw::_URC_FATAL_PHASE2_ERROR,
             }
+            EHAction::Terminate => return uw::_URC_FATAL_PHASE2_ERROR,
         }
     }
 }
 
+#[export_name="__artiq_raise"]
+#[unwind(allowed)]
+pub unsafe extern fn raise(exception: *const Exception) -> ! {
+    let count = EXCEPTION_BUFFER.exception_count;
+    let stack = &mut EXCEPTION_BUFFER.exception_stack;
+    let diff = exception as isize - EXCEPTION_BUFFER.exceptions.as_ptr() as isize;
+    if 0 <= diff && diff <= (mem::size_of::<Option<Exception>>() * MAX_INFLIGHT_EXCEPTIONS) as isize {
+        let index = diff / (mem::size_of::<Option<Exception>>() as isize);
+        let mut found = false;
+        for i in 0..=MAX_INFLIGHT_EXCEPTIONS + 1 {
+            if found {
+                if stack[i] == -1 {
+                    stack[i - 1] = index;
+                    assert!(i == count);
+                    break;
+                } else {
+                    stack[i - 1] = stack[i];
+                }
+            } else {
+                if stack[i] == index {
+                    found = true;
+                }
+            }
+        }
+        assert!(found);
+        let _result = _Unwind_ForcedUnwind(&mut EXCEPTION_BUFFER.uw_exceptions[stack[count - 1] as usize],
+                                           stop_fn, core::ptr::null_mut());
+    } else {
+        if count < MAX_INFLIGHT_EXCEPTIONS {
+            let exception = &*exception;
+            for (i, slot) in EXCEPTION_BUFFER.exceptions.iter_mut().enumerate() {
+                // we should always be able to find a slot
+                if slot.is_none() {
+                    *slot = Some(
+                        *mem::transmute::<*const Exception, *const Exception<'static>>
+                        (exception));
+                    EXCEPTION_BUFFER.exception_stack[count] = i as isize;
+                    EXCEPTION_BUFFER.uw_exceptions[i].private =
+                        [0; uw::unwinder_private_data_size];
+                    EXCEPTION_BUFFER.stack_pointers[i] = StackPointerBacktrace {
+                        stack_pointer: 0,
+                        initial_backtrace_size: EXCEPTION_BUFFER.backtrace_size,
+                        current_backtrace_size: 0,
+                    };
+                    EXCEPTION_BUFFER.exception_count += 1;
+                    let _result = _Unwind_ForcedUnwind(&mut EXCEPTION_BUFFER.uw_exceptions[i],
+                                                       stop_fn, core::ptr::null_mut());
+                }
+            }
+        } else {
+            // TODO: better reporting?
+            let exception = Exception {
+                id:       get_exception_id("RuntimeError"),
+                file:     file!().as_c_slice(),
+                line:     line!(),
+                column:   column!(),
+                // https://github.com/rust-lang/rfcs/pull/1719
+                function: "__artiq_raise".as_c_slice(),
+                message:  "too many nested exceptions".as_c_slice(),
+                param:    [0, 0, 0]
+            };
+            EXCEPTION_BUFFER.exceptions[MAX_INFLIGHT_EXCEPTIONS] = Some(mem::transmute(exception));
+            EXCEPTION_BUFFER.stack_pointers[MAX_INFLIGHT_EXCEPTIONS] = Default::default();
+            EXCEPTION_BUFFER.exception_count += 1;
+            uncaught_exception()
+        }
+    }
+    unreachable!();
+}
+
+
+#[export_name="__artiq_resume"]
+#[unwind(allowed)]
+pub unsafe extern fn resume() -> ! {
+    assert!(EXCEPTION_BUFFER.exception_count != 0);
+    let i = EXCEPTION_BUFFER.exception_stack[EXCEPTION_BUFFER.exception_count - 1];
+    assert!(i != -1);
+    let _result = _Unwind_ForcedUnwind(&mut EXCEPTION_BUFFER.uw_exceptions[i as usize],
+                                       stop_fn, core::ptr::null_mut());
+    unreachable!()
+}
+
+#[export_name="__artiq_end_catch"]
+#[unwind(allowed)]
+pub unsafe extern fn end_catch() {
+    let mut count = EXCEPTION_BUFFER.exception_count;
+    assert!(count != 0);
+    // we remove all exceptions with SP <= current exception SP
+    // i.e. the outer exception escapes the finally block
+    let index = EXCEPTION_BUFFER.exception_stack[count - 1] as usize;
+    EXCEPTION_BUFFER.exception_stack[count - 1] = -1;
+    EXCEPTION_BUFFER.exceptions[index] = None;
+    let outer_sp = EXCEPTION_BUFFER.stack_pointers
+        [index].stack_pointer;
+    count -= 1;
+    for i in (0..count).rev() {
+        let index = EXCEPTION_BUFFER.exception_stack[i];
+        assert!(index != -1);
+        let index = index as usize;
+        let sp = EXCEPTION_BUFFER.stack_pointers[index].stack_pointer;
+        if sp >= outer_sp {
+            break;
+        }
+        EXCEPTION_BUFFER.exceptions[index] = None;
+        EXCEPTION_BUFFER.exception_stack[i] = -1;
+        count -= 1;
+    }
+    EXCEPTION_BUFFER.exception_count = count;
+    EXCEPTION_BUFFER.backtrace_size = if count > 0 {
+        let index = EXCEPTION_BUFFER.exception_stack[count - 1];
+        assert!(index != -1);
+        EXCEPTION_BUFFER.stack_pointers[index as usize].current_backtrace_size
+    } else {
+        0
+    };
+}
+
 extern fn cleanup(_unwind_code: uw::_Unwind_Reason_Code,
-                  uw_exception: *mut uw::_Unwind_Exception) {
+                  _uw_exception: *mut uw::_Unwind_Exception) {
-    unsafe {
+    unimplemented!()
-        let exception_info = &mut *(uw_exception as *mut ExceptionInfo);
+}
 
-        exception_info.exception = None;
+fn uncaught_exception() -> ! {
+    unsafe {
+        // dump way to reorder the stack
+        for i in 0..EXCEPTION_BUFFER.exception_count {
+            if EXCEPTION_BUFFER.exception_stack[i] != i as isize {
+                // find the correct index
+                let index = EXCEPTION_BUFFER.exception_stack
+                    .iter()
+                    .position(|v| *v == i as isize).unwrap();
+                let a = EXCEPTION_BUFFER.exception_stack[index];
+                let b = EXCEPTION_BUFFER.exception_stack[i];
+                assert!(a != -1 && b != -1);
+                core::mem::swap(&mut EXCEPTION_BUFFER.exception_stack[index],
+                                &mut EXCEPTION_BUFFER.exception_stack[i]);
+                core::mem::swap(&mut EXCEPTION_BUFFER.exceptions[a as usize],
+                                &mut EXCEPTION_BUFFER.exceptions[b as usize]);
+                core::mem::swap(&mut EXCEPTION_BUFFER.stack_pointers[a as usize],
+                                &mut EXCEPTION_BUFFER.stack_pointers[b as usize]);
+            }
+        }
+    }
+    unsafe {
+        ::terminate(
+            EXCEPTION_BUFFER.exceptions[..EXCEPTION_BUFFER.exception_count].as_ref(),
+            EXCEPTION_BUFFER.stack_pointers[..EXCEPTION_BUFFER.exception_count].as_ref(),
+            EXCEPTION_BUFFER.backtrace[..EXCEPTION_BUFFER.backtrace_size].as_mut())
     }
 }
 
-extern fn uncaught_exception(_version: c_int,
+
-                             actions: uw::_Unwind_Action,
+// stop function which would be executed when we unwind each frame
-                             _uw_exception_class: uw::_Unwind_Exception_Class,
+extern fn stop_fn(_version: c_int,
-                             uw_exception: *mut uw::_Unwind_Exception,
+                  actions: uw::_Unwind_Action,
-                             context: *mut uw::_Unwind_Context,
+                  _uw_exception_class: uw::_Unwind_Exception_Class,
-                             _stop_parameter: *mut c_void)
+                  _uw_exception: *mut uw::_Unwind_Exception,
-                            -> uw::_Unwind_Reason_Code {
+                  context: *mut uw::_Unwind_Context,
+                  _stop_parameter: *mut c_void) -> uw::_Unwind_Reason_Code {
     unsafe {
-        let exception_info = &mut *(uw_exception as *mut ExceptionInfo);
+        let backtrace_size = EXCEPTION_BUFFER.backtrace_size;
-
+        if backtrace_size < MAX_BACKTRACE_SIZE {
-        if exception_info.backtrace_size < exception_info.backtrace.len() {
             let ip = uw::_Unwind_GetIP(context);
-            exception_info.backtrace[exception_info.backtrace_size] = ip;
+            let fp = uw::_Unwind_GetGR(context, UNW_FP_REG);
-            exception_info.backtrace_size += 1;
+            if PAYLOAD_ADDRESS == 0 || ip > PAYLOAD_ADDRESS {
+                let ip = ip - PAYLOAD_ADDRESS;
+                EXCEPTION_BUFFER.backtrace[backtrace_size] = (ip, fp);
+                EXCEPTION_BUFFER.backtrace_size += 1;
+                let last_index = EXCEPTION_BUFFER.exception_stack[EXCEPTION_BUFFER.exception_count - 1];
+                assert!(last_index != -1);
+                let sp_info = &mut EXCEPTION_BUFFER.stack_pointers[last_index as usize];
+                sp_info.stack_pointer = fp;
+                sp_info.current_backtrace_size = backtrace_size + 1;
+            }
         }
-
         if actions as u32 & uw::_UA_END_OF_STACK as u32 != 0 {
-            ::terminate(&exception_info.exception.unwrap(),
+            uncaught_exception()
-                        exception_info.backtrace[..exception_info.backtrace_size].as_mut())
         } else {
             uw::_URC_NO_REASON
         }
     }
 }
 
-// We can unfortunately not use mem::zeroed in a static, so Option<> is used as a workaround.
+static EXCEPTION_ID_LOOKUP: [(&str, u32); 10] = [
-// See https://github.com/rust-lang/rust/issues/39498.
+    ("RuntimeError", 0),
-static mut INFLIGHT: ExceptionInfo = ExceptionInfo {
+    ("RTIOUnderflow", 1),
-    uw_exception: uw::_Unwind_Exception {
+    ("RTIOOverflow", 2),
-        exception_class:   EXCEPTION_CLASS,
+    ("RTIODestinationUnreachable", 3),
-        exception_cleanup: cleanup,
+    ("DMAError", 4),
-        private:           [0; uw::unwinder_private_data_size],
+    ("I2CError", 5),
-    },
+    ("CacheError", 6),
-    exception:      None,
+    ("SPIError", 7),
-    handled:        true,
+    ("ZeroDivisionError", 8),
-    backtrace:      [0; MAX_BACKTRACE_SIZE],
+    ("IndexError", 9)
-    backtrace_size: 0
+];
-};
 
-#[export_name="__artiq_raise"]
+pub fn get_exception_id(name: &str) -> u32 {
-#[unwind(allowed)]
+    for (n, id) in EXCEPTION_ID_LOOKUP.iter() {
-pub unsafe extern fn raise(exception: *const Exception) -> ! {
+        if *n == name {
-    // Zing! The Exception<'a> to Exception<'static> transmute is not really sound in case
+            return *id
-    // the exception is ever captured. Fortunately, they currently aren't, and we save
-    // on the hassle of having to allocate exceptions somewhere except on stack.
-    INFLIGHT.exception = Some(mem::transmute::<Exception, Exception<'static>>(*exception));
-    INFLIGHT.handled   = false;
-
-    let result = uw::_Unwind_RaiseException(&mut INFLIGHT.uw_exception);
-    assert!(result == uw::_URC_END_OF_STACK);
-
-    INFLIGHT.backtrace_size = 0;
-    let _result = _Unwind_ForcedUnwind(&mut INFLIGHT.uw_exception,
-                                       uncaught_exception, ptr::null_mut());
-    unreachable!()
-}
-
-#[export_name="__artiq_reraise"]
-#[unwind(allowed)]
-pub unsafe extern fn reraise() -> ! {
-    use cslice::AsCSlice;
-
-    if INFLIGHT.handled {
-        match INFLIGHT.exception {
-            Some(ref exception) => raise(exception),
-            None => raise(&Exception {
-                name:     "0:artiq.coredevice.exceptions.RuntimeError".as_c_slice(),
-                file:     file!().as_c_slice(),
-                line:     line!(),
-                column:   column!(),
-                // https://github.com/rust-lang/rfcs/pull/1719
-                function: "__artiq_reraise".as_c_slice(),
-                message:  "No active exception to reraise".as_c_slice(),
-                param:    [0, 0, 0]
-            })
         }
-    } else {
-        uw::_Unwind_Resume(&mut INFLIGHT.uw_exception)
     }
+    unimplemented!("unallocated internal exception id")
 }
+

artiq/firmware/ksupport/lib.rs

@@ -1,5 +1,5 @@
 #![feature(lang_items, llvm_asm, panic_unwind, libc, unwind_attributes,
-           panic_info_message, nll)]
+           panic_info_message, nll, const_in_array_repeat_expressions)]
 #![no_std]
 
 extern crate libc;
@@ -80,8 +80,9 @@ macro_rules! println {
 macro_rules! raise {
     ($name:expr, $message:expr, $param0:expr, $param1:expr, $param2:expr) => ({
         use cslice::AsCSlice;
+        let name_id = $crate::eh_artiq::get_exception_id($name);
         let exn = $crate::eh_artiq::Exception {
-            name:     concat!("0:artiq.coredevice.exceptions.", $name).as_c_slice(),
+            id:       name_id,
             file:     file!().as_c_slice(),
             line:     line!(),
             column:   column!(),
@@ -164,12 +165,12 @@ extern fn rpc_recv(slot: *mut ()) -> usize {
             &Err(ref exception) =>
             unsafe {
                 eh_artiq::raise(&eh_artiq::Exception {
-                    name:     exception.name.as_bytes().as_c_slice(),
+                    id:       exception.id,
-                    file:     exception.file.as_bytes().as_c_slice(),
+                    file:     exception.file,
                     line:     exception.line,
                     column:   exception.column,
-                    function: exception.function.as_bytes().as_c_slice(),
+                    function: exception.function,
-                    message:  exception.message.as_bytes().as_c_slice(),
+                    message:  exception.message,
                     param:    exception.param
                 })
             }
@@ -177,27 +178,13 @@ extern fn rpc_recv(slot: *mut ()) -> usize {
     })
 }
 
-fn terminate(exception: &eh_artiq::Exception, backtrace: &mut [usize]) -> ! {
+fn terminate(exceptions: &'static [Option<eh_artiq::Exception<'static>>],
-    let mut cursor = 0;
+             stack_pointers: &'static [eh_artiq::StackPointerBacktrace],
-    for index in 0..backtrace.len() {
+             backtrace: &mut [(usize, usize)]) -> ! {
-        if backtrace[index] > kernel_proto::KERNELCPU_PAYLOAD_ADDRESS {
-            backtrace[cursor] = backtrace[index] - kernel_proto::KERNELCPU_PAYLOAD_ADDRESS;
-            cursor += 1;
-        }
-    }
-    let backtrace = &mut backtrace.as_mut()[0..cursor];
-
     send(&RunException {
-        exception: kernel_proto::Exception {
+        exceptions,
-            name:     str::from_utf8(exception.name.as_ref()).unwrap(),
+        stack_pointers,
-            file:     str::from_utf8(exception.file.as_ref()).unwrap(),
+        backtrace
-            line:     exception.line,
-            column:   exception.column,
-            function: str::from_utf8(exception.function.as_ref()).unwrap(),
-            message:  str::from_utf8(exception.message.as_ref()).unwrap(),
-            param:    exception.param,
-        },
-        backtrace: backtrace
     });
     loop {}
 }
@@ -472,6 +459,7 @@ unsafe fn attribute_writeback(typeinfo: *const ()) {
 
 #[no_mangle]
 pub unsafe fn main() {
+    eh_artiq::reset_exception_buffer(KERNELCPU_PAYLOAD_ADDRESS);
     let image = slice::from_raw_parts_mut(kernel_proto::KERNELCPU_PAYLOAD_ADDRESS as *mut u8,
                                           kernel_proto::KERNELCPU_LAST_ADDRESS -
                                           kernel_proto::KERNELCPU_PAYLOAD_ADDRESS);

artiq/firmware/libeh/dwarf.rs

@@ -202,8 +202,7 @@ unsafe fn get_ttype_entry(
 pub unsafe fn find_eh_action(
     lsda: *const u8,
     context: &EHContext<'_>,
-    name: *const u8,
+    id: u32,
-    len: usize,
 ) -> Result<EHAction, ()> {
     if lsda.is_null() {
         return Ok(EHAction::None);
@@ -275,19 +274,9 @@ pub unsafe fn find_eh_action(
                                 return Ok(EHAction::Catch(lpad));
                             }
                             // this seems to be target dependent
-                            let clause_ptr = *(catch_type as *const CSlice<u8>);
+                            let clause_id = *(catch_type as *const u32);
-                            let clause_name_ptr = (clause_ptr).as_ptr();
+                            if clause_id == id {
-                            let clause_name_len = (clause_ptr).len();
+                                return Ok(EHAction::Catch(lpad));
-                            if clause_name_len == len {
-                                if (clause_name_ptr == core::ptr::null() ||
-                                    clause_name_ptr == name ||
-                                    // somehow their name pointers might differ, but the content is the
-                                    // same
-                                    core::slice::from_raw_parts(clause_name_ptr, clause_name_len) ==
-                                    core::slice::from_raw_parts(name, len))
-                                {
-                                    return Ok(EHAction::Catch(lpad));
-                                }
                             }
                         } else if ar_filter < 0 {
                             // FIXME: how to handle this?

artiq/firmware/libeh/eh_artiq.rs

@@ -0,0 +1,47 @@
+// ARTIQ Exception struct declaration
+use cslice::CSlice;
+
+// Note: CSlice within an exception may not be actual cslice, they may be strings that exist only
+// in the host. If the length == usize:MAX, the pointer is actually a string key in the host.
+#[repr(C)]
+#[derive(Copy, Clone)]
+pub struct Exception<'a> {
+    pub id:       u32,
+    pub file:     CSlice<'a, u8>,
+    pub line:     u32,
+    pub column:   u32,
+    pub function: CSlice<'a, u8>,
+    pub message:  CSlice<'a, u8>,
+    pub param:    [i64; 3]
+}
+
+fn str_err(_: core::str::Utf8Error) -> core::fmt::Error {
+    core::fmt::Error
+}
+
+fn exception_str<'a>(s: &'a CSlice<'a, u8>) -> Result<&'a str, core::str::Utf8Error> {
+    if s.len() == usize::MAX {
+        Ok("<host string>")
+    } else {
+        core::str::from_utf8(s.as_ref())
+    }
+}
+
+impl<'a> core::fmt::Debug for Exception<'a> {
+    fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result {
+        write!(f, "Exception {} from {} in {}:{}:{}, message: {}",
+            self.id,
+            exception_str(&self.function).map_err(str_err)?,
+            exception_str(&self.file).map_err(str_err)?,
+            self.line, self.column,
+            exception_str(&self.message).map_err(str_err)?)
+    }
+}
+
+#[derive(Copy, Clone, Debug, Default)]
+pub struct StackPointerBacktrace {
+    pub stack_pointer: usize,
+    pub initial_backtrace_size: usize,
+    pub current_backtrace_size: usize,
+}
+

artiq/firmware/libeh/eh_rust.rs

@@ -87,5 +87,5 @@ unsafe fn find_eh_action(context: *mut uw::_Unwind_Context) -> Result<EHAction,
         get_text_start: &|| uw::_Unwind_GetTextRelBase(context),
         get_data_start: &|| uw::_Unwind_GetDataRelBase(context),
     };
-    dwarf::find_eh_action(lsda, &eh_context, core::ptr::null(), 0)
+    dwarf::find_eh_action(lsda, &eh_context, 0)
 }

artiq/firmware/libeh/lib.rs

@@ -7,3 +7,4 @@ extern crate libc;
 
 pub mod dwarf;
 pub mod eh_rust;
+pub mod eh_artiq;

artiq/firmware/libproto_artiq/Cargo.toml

@@ -15,6 +15,7 @@ cslice = { version = "0.3" }
 log = { version = "0.4", default-features = false, optional = true }
 io = { path = "../libio", features = ["byteorder"] }
 dyld = { path = "../libdyld" }
+eh = { path = "../libeh" }
 
 [features]
 alloc = ["io/alloc"]

artiq/firmware/libproto_artiq/kernel_proto.rs

@@ -6,17 +6,6 @@ pub const KERNELCPU_PAYLOAD_ADDRESS: usize = 0x45060000;
 pub const KERNELCPU_LAST_ADDRESS:    usize = 0x4fffffff;
 pub const KSUPPORT_HEADER_SIZE:      usize = 0x80;
 
-#[derive(Debug, Clone)]
-pub struct Exception<'a> {
-    pub name:     &'a str,
-    pub file:     &'a str,
-    pub line:     u32,
-    pub column:   u32,
-    pub function: &'a str,
-    pub message:  &'a str,
-    pub param:    [i64; 3]
-}
-
 #[derive(Debug)]
 pub enum Message<'a> {
     LoadRequest(&'a [u8]),
@@ -47,8 +36,9 @@ pub enum Message<'a> {
 
     RunFinished,
     RunException {
-        exception: Exception<'a>,
+        exceptions: &'a [Option<eh::eh_artiq::Exception<'a>>],
-        backtrace: &'a [usize]
+        stack_pointers: &'a [eh::eh_artiq::StackPointerBacktrace],
+        backtrace: &'a [(usize, usize)]
     },
     RunAborted,
 
@@ -59,7 +49,7 @@ pub enum Message<'a> {
         data: *const *const ()
     },
     RpcRecvRequest(*mut ()),
-    RpcRecvReply(Result<usize, Exception<'a>>),
+    RpcRecvReply(Result<usize, eh::eh_artiq::Exception<'a>>),
     RpcFlush,
 
     CacheGetRequest { key: &'a str },

artiq/firmware/libproto_artiq/lib.rs

@@ -13,6 +13,7 @@ extern crate log;
 extern crate byteorder;
 extern crate io;
 extern crate dyld;
+extern crate eh;
 
 // Internal protocols.
 pub mod kernel_proto;

artiq/firmware/libproto_artiq/session_proto.rs

@@ -1,5 +1,7 @@
 use core::str::Utf8Error;
-use alloc::{vec::Vec, string::String};
+use alloc::vec::Vec;
+use eh::eh_artiq::{Exception, StackPointerBacktrace};
+use cslice::CSlice;
 
 use io::{Read, ProtoRead, Write, ProtoWrite, Error as IoError, ReadStringError};
 
@@ -70,13 +72,13 @@ pub enum Request {
 
     RpcReply { tag: Vec<u8> },
     RpcException {
-        name:     String,
+        id:       u32,
-        message:  String,
+        message:  u32,
         param:    [i64; 3],
-        file:     String,
+        file:     u32,
         line:     u32,
         column:   u32,
-        function: String,
+        function: u32,
     },
 }
 
@@ -95,14 +97,9 @@ pub enum Reply<'a> {
     },
     KernelStartupFailed,
     KernelException {
-        name:      &'a str,
+        exceptions: &'a [Option<Exception<'a>>],
-        message:   &'a str,
+        stack_pointers: &'a [StackPointerBacktrace],
-        param:     [i64; 3],
+        backtrace: &'a [(usize, usize)],
-        file:      &'a str,
-        line:      u32,
-        column:    u32,
-        function:  &'a str,
-        backtrace: &'a [usize],
         async_errors: u8
     },
 
@@ -126,15 +123,15 @@ impl Request {
                 tag: reader.read_bytes()?
             },
             8  => Request::RpcException {
-                name:     reader.read_string()?,
+                id:       reader.read_u32()?,
-                message:  reader.read_string()?,
+                message:  reader.read_u32()?,
                 param:    [reader.read_u64()? as i64,
                            reader.read_u64()? as i64,
                            reader.read_u64()? as i64],
-                file:     reader.read_string()?,
+                file:     reader.read_u32()?,
                 line:     reader.read_u32()?,
                 column:   reader.read_u32()?,
-                function: reader.read_string()?
+                function: reader.read_u32()?
             },
 
             ty  => return Err(Error::UnknownPacket(ty))
@@ -142,6 +139,18 @@ impl Request {
     }
 }
 
+fn write_exception_string<'a, W>(writer: &mut W, s: &CSlice<'a, u8>) -> Result<(), IoError<W::WriteError>>
+    where W: Write + ?Sized
+{
+    if s.len() == usize::MAX {
+        writer.write_u32(u32::MAX)?;
+        writer.write_u32(s.as_ptr() as u32)?;
+    } else {
+        writer.write_string(core::str::from_utf8(s.as_ref()).unwrap())?;
+    }
+    Ok(())
+}
+
 impl<'a> Reply<'a> {
     pub fn write_to<W>(&self, writer: &mut W) -> Result<(), IoError<W::WriteError>>
         where W: Write + ?Sized
@@ -171,22 +180,36 @@ impl<'a> Reply<'a> {
                 writer.write_u8(8)?;
             },
             Reply::KernelException {
-                name, message, param, file, line, column, function, backtrace,
+                exceptions,
+                stack_pointers,
+                backtrace,
                 async_errors
             } => {
                 writer.write_u8(9)?;
-                writer.write_string(name)?;
+                writer.write_u32(exceptions.len() as u32)?;
-                writer.write_string(message)?;
+                for exception in exceptions.iter() {
-                writer.write_u64(param[0] as u64)?;
+                    let exception = exception.as_ref().unwrap();
-                writer.write_u64(param[1] as u64)?;
+                    writer.write_u32(exception.id as u32)?;
-                writer.write_u64(param[2] as u64)?;
+                    write_exception_string(writer, &exception.message)?;
-                writer.write_string(file)?;
+                    writer.write_u64(exception.param[0] as u64)?;
-                writer.write_u32(line)?;
+                    writer.write_u64(exception.param[1] as u64)?;
-                writer.write_u32(column)?;
+                    writer.write_u64(exception.param[2] as u64)?;
-                writer.write_string(function)?;
+                    write_exception_string(writer, &exception.file)?;
+                    writer.write_u32(exception.line)?;
+                    writer.write_u32(exception.column)?;
+                    write_exception_string(writer, &exception.function)?;
+                }
+
+                for sp in stack_pointers.iter() {
+                    writer.write_u32(sp.stack_pointer as u32)?;
+                    writer.write_u32(sp.initial_backtrace_size as u32)?;
+                    writer.write_u32(sp.current_backtrace_size as u32)?;
+                }
+
                 writer.write_u32(backtrace.len() as u32)?;
-                for &addr in backtrace {
+                for &(addr, sp) in backtrace {
-                    writer.write_u32(addr as u32)?
+                    writer.write_u32(addr as u32)?;
+                    writer.write_u32(sp as u32)?;
                 }
                 writer.write_u8(async_errors)?;
             },

artiq/firmware/runtime/session.rs

@@ -1,6 +1,7 @@
 use core::{mem, str, cell::{Cell, RefCell}, fmt::Write as FmtWrite};
 use alloc::{vec::Vec, string::String};
 use byteorder::{ByteOrder, NativeEndian};
+use cslice::CSlice;
 
 use io::{Read, Write, Error as IoError};
 use board_misoc::{ident, cache, config};
@@ -291,7 +292,7 @@ fn process_host_message(io: &Io,
         }
 
         host::Request::RpcException {
-            name, message, param, file, line, column, function
+            id, message, param, file, line, column, function
         } => {
             if session.kernel_state != KernelState::RpcWait {
                 unexpected!("unsolicited RPC reply")
@@ -305,16 +306,18 @@ fn process_host_message(io: &Io,
                 }
             })?;
 
-            let exn = kern::Exception {
+            unsafe {
-                name:     name.as_ref(),
+                let exn = eh::eh_artiq::Exception {
-                message:  message.as_ref(),
+                    id:       id,
-                param:    param,
+                    message:  CSlice::new(message as *const u8, usize::MAX),
-                file:     file.as_ref(),
+                    param:    param,
-                line:     line,
+                    file:     CSlice::new(file as *const u8, usize::MAX),
-                column:   column,
+                    line:     line,
-                function: function.as_ref()
+                    column:   column,
-            };
+                    function: CSlice::new(function as *const u8, usize::MAX),
-            kern_send(io, &kern::RpcRecvReply(Err(exn)))?;
+                };
+                kern_send(io, &kern::RpcRecvReply(Err(exn)))?;
+            }
 
             session.kernel_state = KernelState::Running
         }
@@ -438,7 +441,8 @@ fn process_kern_message(io: &Io, aux_mutex: &Mutex,
                 }
             }
             &kern::RunException {
-                exception: kern::Exception { name, message, param, file, line, column, function },
+                exceptions,
+                stack_pointers,
                 backtrace
             } => {
                 unsafe { kernel::stop() }
@@ -448,19 +452,15 @@ fn process_kern_message(io: &Io, aux_mutex: &Mutex,
                 match stream {
                     None => {
                         error!("exception in flash kernel");
-                        error!("{}: {} {:?}", name, message, param);
+                        for exception in exceptions {
-                        error!("at {}:{}:{} in {}", file, line, column, function);
+                            error!("{:?}", exception.unwrap());
+                        }
                         return Ok(true)
                     },
                     Some(ref mut stream) => {
                         host_write(stream, host::Reply::KernelException {
-                            name:      name,
+                            exceptions: exceptions,
-                            message:   message,
+                            stack_pointers: stack_pointers,
-                            param:     param,
-                            file:      file,
-                            line:      line,
-                            column:    column,
-                            function:  function,
                             backtrace: backtrace,
                             async_errors: unsafe { get_async_errors() }
                         }).map_err(|e| e.into())

artiq/language/environment.py

@@ -330,8 +330,7 @@ class HasEnvironment:
 
     @rpc(flags={"async"})
     def set_dataset(self, key, value,
-                    broadcast=False, persist=False, archive=True,
+                    broadcast=False, persist=False, archive=True):
-                    hdf5_options=None):
         """Sets the contents and handling modes of a dataset.
 
         Datasets must be scalars (``bool``, ``int``, ``float`` or NumPy scalar)
@@ -343,16 +342,8 @@ class HasEnvironment:
             broadcast.
         :param archive: the data is saved into the local storage of the current
             run (archived as a HDF5 file).
-        :param hdf5_options: dict of keyword arguments to pass to
-            :meth:`h5py.Group.create_dataset`. For example, pass ``{"compression": "gzip"}``
-            to enable transparent zlib compression of this dataset in the HDF5 archive.
-            See the `h5py documentation <https://docs.h5py.org/en/stable/high/group.html#h5py.Group.create_dataset>`_
-            for a list of valid options.
         """
-        
+        self.__dataset_mgr.set(key, value, broadcast, persist, archive)
-        self.__dataset_mgr.set(
-            key, value, broadcast, persist, archive, hdf5_options
-        )
 
     @rpc(flags={"async"})
     def mutate_dataset(self, key, index, value):

artiq/master/databases.py

@@ -35,15 +35,6 @@ class DeviceDB:
         return desc
 
 
-def make_dataset(*, persist=False, value=None, hdf5_options=None):
-    "PYON-serializable representation of a dataset in the DatasetDB"
-    return {
-        "persist": persist,
-        "value": value,
-        "hdf5_options": hdf5_options or {},
-    }
-
-
 class DatasetDB(TaskObject):
     def __init__(self, persist_file, autosave_period=30):
         self.persist_file = persist_file
@@ -53,23 +44,10 @@ class DatasetDB(TaskObject):
             file_data = pyon.load_file(self.persist_file)
         except FileNotFoundError:
             file_data = dict()
-        self.data = Notifier(
+        self.data = Notifier({k: (True, v) for k, v in file_data.items()})
-            {
-                k: make_dataset(
-                    persist=True,
-                    value=v["value"],
-                    hdf5_options=v["hdf5_options"]
-                )
-                for k, v in file_data.items()
-            }
-        )
 
     def save(self):
-        data = {
+        data = {k: v[1] for k, v in self.data.raw_view.items() if v[0]}
-            k: d
-            for k, d in self.data.raw_view.items()
-            if d["persist"]
-        }
         pyon.store_file(self.persist_file, data)
 
     async def _do(self):
@@ -81,23 +59,20 @@ class DatasetDB(TaskObject):
             self.save()
 
     def get(self, key):
-        return self.data.raw_view[key]
+        return self.data.raw_view[key][1]
 
     def update(self, mod):
         process_mod(self.data, mod)
 
     # convenience functions (update() can be used instead)
-    def set(self, key, value, persist=None, hdf5_options=None):
+    def set(self, key, value, persist=None):
         if persist is None:
             if key in self.data.raw_view:
-                persist = self.data.raw_view[key]["persist"]
+                persist = self.data.raw_view[key][0]
             else:
                 persist = False
-        self.data[key] = make_dataset(
+        self.data[key] = (persist, value)
-            persist=persist,
-            value=value,
-            hdf5_options=hdf5_options,
-        )
 
     def delete(self, key):
         del self.data[key]
+    #

artiq/master/worker_db.py

@@ -8,12 +8,9 @@ from operator import setitem
 import importlib
 import logging
 
-import numpy as np
-
 from sipyco.sync_struct import Notifier
 from sipyco.pc_rpc import AutoTarget, Client, BestEffortClient
 
-from artiq.master.databases import make_dataset
 
 logger = logging.getLogger(__name__)
 
@@ -118,26 +115,17 @@ class DatasetManager:
         self.ddb = ddb
         self._broadcaster.publish = ddb.update
 
-    def set(self, key, value, broadcast=False, persist=False, archive=True,
+    def set(self, key, value, broadcast=False, persist=False, archive=True):
-            hdf5_options=None):
         if persist:
             broadcast = True
 
         if broadcast:
-            self._broadcaster[key] = make_dataset(
+            self._broadcaster[key] = persist, value
-                persist=persist,
-                value=value,
-                hdf5_options=hdf5_options,
-            )
         elif key in self._broadcaster.raw_view:
             del self._broadcaster[key]
 
         if archive:
-            self.local[key] = make_dataset(
+            self.local[key] = value
-                persist=persist,
-                value=value,
-                hdf5_options=hdf5_options,
-            )
         elif key in self.local:
             del self.local[key]
 
@@ -145,11 +133,11 @@ class DatasetManager:
         target = self.local.get(key, None)
         if key in self._broadcaster.raw_view:
             if target is not None:
-                assert target["value"] is self._broadcaster.raw_view[key]["value"]
+                assert target is self._broadcaster.raw_view[key][1]
-            return self._broadcaster[key]["value"]
+            return self._broadcaster[key][1]
         if target is None:
             raise KeyError("Cannot mutate nonexistent dataset '{}'".format(key))
-        return target["value"]
+        return target
 
     def mutate(self, key, index, value):
         target = self._get_mutation_target(key)
@@ -165,15 +153,15 @@ class DatasetManager:
 
     def get(self, key, archive=False):
         if key in self.local:
-            return self.local[key]["value"]
+            return self.local[key]
         
-        dataset = self.ddb.get(key)
+        data = self.ddb.get(key)
         if archive:
             if key in self.archive:
                 logger.warning("Dataset '%s' is already in archive, "
                                "overwriting", key, stack_info=True)
-            self.archive[key] = dataset
+            self.archive[key] = data
-        return dataset["value"]
+        return data
 
     def write_hdf5(self, f):
         datasets_group = f.create_group("datasets")
@@ -189,7 +177,7 @@ def _write(group, k, v):
     # Add context to exception message when the user writes a dataset that is
     # not representable in HDF5.
     try:
-        group.create_dataset(k, data=v["value"], **v["hdf5_options"])
+        group[k] = v
     except TypeError as e:
         raise TypeError("Error writing dataset '{}' of type '{}': {}".format(
-            k, type(v["value"]), e))
+            k, type(v), e))

artiq/test/coredevice/test_portability.py

@@ -125,6 +125,57 @@ class _Pulses(EnvExperiment):
 class _MyException(Exception):
     pass
 
+class _NestedFinally(EnvExperiment):
+    def build(self, trace):
+        self.setattr_device("core")
+        self.trace = trace
+
+    def _trace(self, i):
+        self.trace.append(i)
+
+    @kernel
+    def run(self):
+        try:
+            try:
+                raise ValueError
+            finally:
+                try:
+                    raise IndexError()
+                except ValueError:
+                    self._trace(0)
+        except:
+            self._trace(1)
+        finally:
+            self._trace(2)
+
+class _NestedExceptions(EnvExperiment):
+    def build(self, trace):
+        self.setattr_device("core")
+        self.trace = trace
+
+    def _trace(self, i):
+        self.trace.append(i)
+
+    @kernel
+    def run(self):
+        try:
+            try:
+                raise ValueError
+            except _MyException:
+                self._trace(0)
+                raise
+            finally:
+                try:
+                    raise IndexError()
+                except ValueError:
+                    self._trace(1)
+                    raise
+        except IndexError:
+            self._trace(2)
+        except:
+            self._trace(3)
+        finally:
+            self._trace(4)
 
 class _Exceptions(EnvExperiment):
     def build(self, trace):
@@ -253,6 +304,18 @@ class HostVsDeviceCase(ExperimentCase):
             _run_on_host(_Exceptions, trace=t_host)
         self.assertEqual(t_device, t_host)
 
+    def test_nested_finally(self):
+        t_device, t_host = [], []
+        self.execute(_NestedFinally, trace=t_device)
+        _run_on_host(_NestedFinally, trace=t_host)
+        self.assertEqual(t_device, t_host)
+
+    def test_nested_exceptions(self):
+        t_device, t_host = [], []
+        self.execute(_NestedExceptions, trace=t_device)
+        _run_on_host(_NestedExceptions, trace=t_host)
+        self.assertEqual(t_device, t_host)
+
     def test_rpc_exceptions(self):
         for f in self.execute, _run_on_host:
             with self.assertRaises(_MyException):

artiq/test/test_datasets.py

@@ -3,9 +3,6 @@
 import copy
 import unittest
 
-import h5py
-import numpy as np
-
 from sipyco.sync_struct import process_mod
 
 from artiq.experiment import EnvExperiment
@@ -17,7 +14,7 @@ class MockDatasetDB:
         self.data = dict()
 
     def get(self, key):
-        return self.data[key]["value"]
+        return self.data[key][1]
 
     def update(self, mod):
         # Copy mod before applying to avoid sharing references to objects
@@ -85,9 +82,9 @@ class ExperimentDatasetCase(unittest.TestCase):
     def test_append_broadcast(self):
         self.exp.set(KEY, [], broadcast=True)
         self.exp.append(KEY, 0)
-        self.assertEqual(self.dataset_db.data[KEY]["value"], [0])
+        self.assertEqual(self.dataset_db.data[KEY][1], [0])
         self.exp.append(KEY, 1)
-        self.assertEqual(self.dataset_db.data[KEY]["value"], [0, 1])
+        self.assertEqual(self.dataset_db.data[KEY][1], [0, 1])
 
     def test_append_array(self):
         for broadcast in (True, False):
@@ -106,44 +103,3 @@ class ExperimentDatasetCase(unittest.TestCase):
         with self.assertRaises(KeyError):
             self.exp.append(KEY, 0)
 
-    def test_write_hdf5_options(self):
-        data = np.random.randint(0, 1024, 1024)
-        self.exp.set(
-            KEY,
-            data,
-            hdf5_options=dict(
-                compression="gzip",
-                compression_opts=6,
-                shuffle=True,
-                fletcher32=True
-            ),
-        )
-
-        with h5py.File("test.h5", "a", "core", backing_store=False) as f:
-            self.dataset_mgr.write_hdf5(f)
-
-            self.assertTrue(np.array_equal(f["datasets"][KEY][()], data))
-            self.assertEqual(f["datasets"][KEY].compression, "gzip")
-            self.assertEqual(f["datasets"][KEY].compression_opts, 6)
-            self.assertTrue(f["datasets"][KEY].shuffle)
-            self.assertTrue(f["datasets"][KEY].fletcher32)
-
-    def test_write_hdf5_no_options(self):
-        data = np.random.randint(0, 1024, 1024)
-        self.exp.set(KEY, data)
-
-        with h5py.File("test.h5", "a", "core", backing_store=False) as f:
-            self.dataset_mgr.write_hdf5(f)
-            self.assertTrue(np.array_equal(f["datasets"][KEY][()], data))
-            self.assertIsNone(f["datasets"][KEY].compression)
-
-    def test_write_hdf5_invalid_type(self):
-        class CustomType:
-            def __init__(self, x):
-                self.x = x
-
-        self.exp.set(KEY, CustomType(42))
-
-        with h5py.File("test.h5", "w", "core", backing_store=False) as f:
-            with self.assertRaisesRegex(TypeError, "CustomType"):
-                self.dataset_mgr.write_hdf5(f)

artiq/test/test_scheduler.py

@@ -6,7 +6,6 @@ from time import time, sleep
 
 from artiq.experiment import *
 from artiq.master.scheduler import Scheduler
-from artiq.master.databases import make_dataset
 
 
 class EmptyExperiment(EnvExperiment):
@@ -288,13 +287,8 @@ class SchedulerCase(unittest.TestCase):
             nonlocal termination_ok
             self.assertEqual(
                 mod,
-                {
+                {"action": "setitem", "key": "termination_ok",
-                    "action": "setitem",
+                 "value": (False, True), "path": []})
-                    "key": "termination_ok",
-                    "value": make_dataset(value=True),
-                    "path": []
-                }
-            )
             termination_ok = True
         handlers = {
             "update_dataset": check_termination

flake.nix

@@ -189,7 +189,7 @@
           cargoDeps = rustPlatform.fetchCargoTarball {
             name = "artiq-firmware-cargo-deps";
             src = "${self}/artiq/firmware";
-            sha256 = "sha256-Lf6M4M/jdRiO5MsWSoqtOSNfRIhbze+qvg4kaiiBWW4=";
+            sha256 = "sha256-YyycMsDzR+JRcMZJd6A/CRi2J9nKmaWY/KXUnAQaZ00=";
           };
           nativeBuildInputs = [
             (pkgs.python3.withPackages(ps: [ migen misoc artiq ]))