coredevice: report nested exceptions

artiq/compiler/targets.py

@@ -212,6 +212,7 @@ class Target:
         # just after the call. Offset them back to get an address somewhere
         # inside the call instruction (or its delay slot), since that's what
         # the backtrace entry should point at.
+        last_inlined = None
         offset_addresses = [hex(addr - 1) for addr in addresses]
         with RunTool([self.tool_addr2line, "--addresses",  "--functions", "--inlines",
                       "--demangle", "--exe={library}"] + offset_addresses,
@@ -227,9 +228,11 @@ class Target:
                 if address_or_function[:2] == "0x":
                     address  = int(address_or_function[2:], 16) + 1 # remove offset
                     function = next(lines)
+                    inlined = False
                 else:
                     address  = backtrace[-1][4] # inlined
                     function = address_or_function
+                    inlined = True
                 location = next(lines)
 
                 filename, line = location.rsplit(":", 1)
@@ -240,7 +243,12 @@ class Target:
                 else:
                     line = int(line)
                 # can't get column out of addr2line D:
-                backtrace.append((filename, line, -1, function, address))
+                if inlined:
+                    last_inlined.append((filename, line, -1, function, address))
+                else:
+                    last_inlined = []
+                    backtrace.append((filename, line, -1, function, address,
+                                      last_inlined))
             return backtrace
 
     def demangle(self, names):

artiq/coredevice/comm_kernel.py

@@ -623,28 +623,59 @@ class CommKernel:
             self._flush()
 
     def _serve_exception(self, embedding_map, symbolizer, demangler):
-        name = self._read_string()
+        exception_count = self._read_int32()
-        message = self._read_string()
+        nested_exceptions = []
-        params = [self._read_int64() for _ in range(3)]
 
-        filename = self._read_string()
+        def read_exception_string():
-        line = self._read_int32()
+            # note: if length == -1, the following int32 is the object key
-        column = self._read_int32()
+            length = self._read_int32()
-        function = self._read_string()
+            if length == -1:
+                return embedding_map.retrieve_str(self._read_int32())
+            else:
+                return self._read(length).decode("utf-8")
+
+        for _ in range(exception_count):
+            name = embedding_map.retrieve_str(self._read_int32())
+            message = read_exception_string()
+            params = [self._read_int64() for _ in range(3)]
+
+            filename = read_exception_string()
+            line = self._read_int32()
+            column = self._read_int32()
+            function = read_exception_string()
+            nested_exceptions.append([name, message, params,
+                                      filename, line, column, function])
+
+        demangled_names = demangler([ex[6] for ex in nested_exceptions])
+        for i in range(exception_count):
+            nested_exceptions[i][6] = demangled_names[i]
+
+        exception_info = []
+        for _ in range(exception_count):
+            sp = self._read_int32()
+            initial_backtrace = self._read_int32()
+            current_backtrace = self._read_int32()
+            exception_info.append((sp, initial_backtrace, current_backtrace))
+
+        backtrace = []
+        stack_pointers = []
+        for _ in range(self._read_int32()):
+            backtrace.append(self._read_int32())
+            stack_pointers.append(self._read_int32())
 
-        backtrace = [self._read_int32() for _ in range(self._read_int32())]
         self._process_async_error()
 
-        traceback = list(reversed(symbolizer(backtrace))) + \
+        traceback = list(symbolizer(backtrace))
-            [(filename, line, column, *demangler([function]), None)]
+        core_exn = exceptions.CoreException(nested_exceptions, exception_info,
-        core_exn = exceptions.CoreException(name, message, params, traceback)
+                                            traceback, stack_pointers)
 
         if core_exn.id == 0:
             python_exn_type = getattr(exceptions, core_exn.name.split('.')[-1])
         else:
             python_exn_type = embedding_map.retrieve_object(core_exn.id)
 
-        python_exn = python_exn_type(message.format(*params))
+        python_exn = python_exn_type(
+            nested_exceptions[-1][1].format(*nested_exceptions[0][2]))
         python_exn.artiq_core_exception = core_exn
         raise python_exn
 

artiq/coredevice/exceptions.py

@@ -16,50 +16,94 @@ AssertionError = builtins.AssertionError
 
 class CoreException:
     """Information about an exception raised or passed through the core device."""
+    def __init__(self, exceptions, exception_info, traceback, stack_pointers):
+        self.exceptions = exceptions
+        self.exception_info = exception_info
+        self.traceback = list(traceback)
+        self.stack_pointers = stack_pointers
 
-    def __init__(self, name, message, params, traceback):
+        first_exception = exceptions[0]
+        name = first_exception[0]
         if ':' in name:
             exn_id, self.name = name.split(':', 2)
             self.id = int(exn_id)
         else:
             self.id, self.name = 0, name
-        self.message, self.params = message, params
+        self.message = first_exception[1]
-        self.traceback = list(traceback)
+        self.params = first_exception[2]
+
+    def append_backtrace(self, record, inlined=False):
+        filename, line, column, function, address = record
+        stub_globals = {"__name__": filename, "__loader__": source_loader}
+        source_line = linecache.getline(filename, line, stub_globals)
+        indentation = re.search(r"^\s*", source_line).end()
+
+        if address is None:
+            formatted_address = ""
+        elif inlined:
+            formatted_address = " (inlined)"
+        else:
+            formatted_address = " (RA=+0x{:x})".format(address)
+
+        filename = filename.replace(artiq_dir, "<artiq>")
+        lines = []
+        if column == -1:
+            lines.append("    {}".format(source_line.strip() if source_line else "<unknown>"))
+            lines.append("  File \"{file}\", line {line}, in {function}{address}".
+                         format(file=filename, line=line, function=function,
+                                address=formatted_address))
+        else:
+            lines.append("    {}^".format(" " * (column - indentation)))
+            lines.append("    {}".format(source_line.strip() if source_line else "<unknown>"))
+            lines.append("  File \"{file}\", line {line}, column {column},"
+                         " in {function}{address}".
+                         format(file=filename, line=line, column=column + 1,
+                                function=function, address=formatted_address))
+        return lines
+
+    def single_traceback(self, exception_index):
+        # note that we insert in reversed order
+        lines = []
+        last_sp = 0
+        start_backtrace_index = self.exception_info[exception_index][1]
+        zipped = list(zip(self.traceback[start_backtrace_index:],
+                          self.stack_pointers[start_backtrace_index:]))
+        exception = self.exceptions[exception_index]
+        name = exception[0]
+        message = exception[1]
+        params = exception[2]
+        if ':' in name:
+            exn_id, name = name.split(':', 2)
+            exn_id = int(exn_id)
+        else:
+            exn_id = 0
+        lines.append("{}({}): {}".format(name, exn_id, message.format(*params)))
+        zipped.append(((exception[3], exception[4], exception[5], exception[6],
+                       None, []), None))
+
+        for ((filename, line, column, function, address, inlined), sp) in zipped:
+            # backtrace of nested exceptions may be discontinuous
+            # but the stack pointer must increase monotonically
+            if sp is not None and sp <= last_sp:
+                continue
+            last_sp = sp
+
+            for record in reversed(inlined):
+                lines += self.append_backtrace(record, True)
+            lines += self.append_backtrace((filename, line, column, function,
+                                            address))
+
+        lines.append("Traceback (most recent call first):")
+
+        return "\n".join(reversed(lines))
 
     def __str__(self):
-        lines = []
+        tracebacks = [self.single_traceback(i) for i in range(len(self.exceptions))]
-        lines.append("Core Device Traceback (most recent call last):")
+        traceback_str = ('\n\nDuring handling of the above exception, ' +
-        last_address = 0
+                        'another exception occurred:\n\n').join(tracebacks)
-        for (filename, line, column, function, address) in self.traceback:
+        return 'Core Device Traceback:\n' +\
-            stub_globals = {"__name__": filename, "__loader__": source_loader}
+                traceback_str +\
-            source_line = linecache.getline(filename, line, stub_globals)
+                '\n\nEnd of Code Device Traceback\n'
-            indentation = re.search(r"^\s*", source_line).end()
-
-            if address is None:
-                formatted_address = ""
-            elif address == last_address:
-                formatted_address = " (inlined)"
-            else:
-                formatted_address = " (RA=+0x{:x})".format(address)
-            last_address = address
-
-            filename = filename.replace(artiq_dir, "<artiq>")
-            if column == -1:
-                lines.append("  File \"{file}\", line {line}, in {function}{address}".
-                             format(file=filename, line=line, function=function,
-                                    address=formatted_address))
-                lines.append("    {}".format(source_line.strip() if source_line else "<unknown>"))
-            else:
-                lines.append("  File \"{file}\", line {line}, column {column},"
-                             " in {function}{address}".
-                             format(file=filename, line=line, column=column + 1,
-                                    function=function, address=formatted_address))
-                lines.append("    {}".format(source_line.strip() if source_line else "<unknown>"))
-                lines.append("    {}^".format(" " * (column - indentation)))
-
-        lines.append("{}({}): {}".format(self.name, self.id,
-                                         self.message.format(*self.params)))
-        return "\n".join(lines)
 
 
 class InternalError(Exception):