Display full core device backtraces.

artiq/compiler/targets.py

@@ -5,6 +5,43 @@ llvm.initialize()
 llvm.initialize_all_targets()
 llvm.initialize_all_asmprinters()
 
+class RunTool:
+    def __init__(self, pattern, **tempdata):
+        self.files = []
+        self.pattern = pattern
+        self.tempdata = tempdata
+
+    def maketemp(self, data):
+        f = tempfile.NamedTemporaryFile()
+        f.write(data)
+        f.flush()
+        self.files.append(f)
+        return f
+
+    def __enter__(self):
+        tempfiles = {}
+        tempnames = {}
+        for key in self.tempdata:
+            tempfiles[key] = self.maketemp(self.tempdata[key])
+            tempnames[key] = tempfiles[key].name
+
+        cmdline = []
+        for argument in self.pattern:
+            cmdline.append(argument.format(**tempnames))
+
+        process = subprocess.Popen(cmdline, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+        stdout, stderr = process.communicate()
+        if process.returncode != 0:
+            raise Exception("{} invocation failed: {}".
+                            format(cmdline[0], stderr.decode('utf-8')))
+
+        tempfiles["__stdout__"] = stdout.decode('utf-8')
+        return tempfiles
+
+    def __exit__(self, exc_typ, exc_value, exc_trace):
+        for f in self.files:
+            f.close()
+
 class Target:
     """
     A description of the target environment where the binaries
@@ -25,35 +62,10 @@ class Target:
     features = []
     print_function = "printf"
 
+
     def __init__(self):
         self.llcontext = ll.Context()
 
-    def link(self, objects, init_fn):
-        """Link the relocatable objects into a shared library for this target."""
-        files = []
-
-        def make_tempfile(data=b""):
-            f = tempfile.NamedTemporaryFile()
-            files.append(f)
-            f.write(data)
-            f.flush()
-            return f
-
-        try:
-            output_file = make_tempfile()
-            cmdline = [self.triple + "-ld", "-shared", "--eh-frame-hdr", "-init", init_fn] + \
-                      [make_tempfile(obj).name for obj in objects] + \
-                      ["-o", output_file.name]
-            linker = subprocess.Popen(cmdline, stderr=subprocess.PIPE)
-            stdout, stderr = linker.communicate()
-            if linker.returncode != 0:
-                raise Exception("Linker invocation failed: " + stderr.decode('utf-8'))
-
-            return output_file.read()
-        finally:
-            for f in files:
-                f.close()
-
     def compile(self, module):
         """Compile the module to a relocatable object for this target."""
 
@@ -93,10 +105,50 @@ class Target:
 
         return llmachine.emit_object(llparsedmod)
 
+    def link(self, objects, init_fn):
+        """Link the relocatable objects into a shared library for this target."""
+        with RunTool([self.triple + "-ld", "-shared", "--eh-frame-hdr", "-init", init_fn] +
+                     ["{{obj{}}}".format(index) for index in range(len(objects))] +
+                     ["-o", "{output}"],
+                     output=b"",
+                     **{"obj{}".format(index): obj for index, obj in enumerate(objects)}) \
+                as results:
+            library = results["output"].read()
+
+            if os.getenv('ARTIQ_DUMP_ELF'):
+                shlib_temp = tempfile.NamedTemporaryFile(suffix=".so", delete=False)
+                shlib_temp.write(library)
+                shlib_temp.close()
+                print("====== SHARED LIBRARY DUMP ======", file=sys.stderr)
+                print("Shared library dumped as {}".format(shlib_temp.name), file=sys.stderr)
+
+            return library
+
     def compile_and_link(self, modules):
         return self.link([self.compile(module) for module in modules],
                          init_fn=modules[0].entry_point())
 
+    def strip(self, library):
+        with RunTool([self.triple + "-strip", "--strip-debug", "{library}", "-o", "{output}"],
+                     library=library, output=b"") \
+                as results:
+            return results["output"].read()
+
+    def symbolize(self, library, addresses):
+        # Addresses point one instruction past the jump; offset them back by 1.
+        offset_addresses = [hex(addr - 1) for addr in addresses]
+        with RunTool([self.triple + "-addr2line", "--functions", "--inlines",
+                      "--exe={library}"] + offset_addresses,
+                     library=library) \
+                as results:
+            lines = results["__stdout__"].rstrip().split("\n")
+            backtrace = []
+            for function_name, location, address in zip(lines[::2], lines[1::2], addresses):
+                filename, line = location.rsplit(":", 1)
+                # can't get column out of addr2line D:
+                backtrace.append((filename, int(line), -1, function_name, address))
+            return backtrace
+
 class NativeTarget(Target):
     def __init__(self):
         super().__init__()

artiq/coredevice/comm_generic.py

@@ -457,7 +457,7 @@ class CommGeneric:
 
             self._write_flush()
 
-    def _serve_exception(self):
+    def _serve_exception(self, symbolizer):
         name      = self._read_string()
         message   = self._read_string()
         params    = [self._read_int64() for _ in range(3)]
@@ -468,19 +468,18 @@ class CommGeneric:
         function  = self._read_string()
 
         backtrace = [self._read_int32() for _ in range(self._read_int32())]
-        # we don't have debug information yet.
-        # print("exception backtrace:", [hex(x) for x in backtrace])
 
-        raise core_language.ARTIQException(name, message, params,
+        traceback = list(reversed(symbolizer(backtrace))) + \
-                                           filename, line, column, function)
+                    [(filename, line, column, function, None)]
+        raise core_language.ARTIQException(name, message, params, traceback)
 
-    def serve(self, rpc_map):
+    def serve(self, rpc_map, symbolizer):
         while True:
             self._read_header()
             if self._read_type == _D2HMsgType.RPC_REQUEST:
                 self._serve_rpc(rpc_map)
             elif self._read_type == _D2HMsgType.KERNEL_EXCEPTION:
-                self._serve_exception()
+                self._serve_exception(symbolizer)
             else:
                 self._read_expect(_D2HMsgType.KERNEL_FINISHED)
                 return

artiq/coredevice/core.py

@@ -35,29 +35,26 @@ class Core:
             module = Module(stitcher)
             target = OR1KTarget()
 
-            return target.compile_and_link([module]), stitcher.rpc_map
+            library = target.compile_and_link([module])
+            stripped_library = target.strip(library)
+
+            return stitcher.rpc_map, stripped_library, \
+                   lambda addresses: target.symbolize(library, addresses)
         except diagnostic.Error as error:
             print("\n".join(error.diagnostic.render(colored=True)), file=sys.stderr)
             raise CompileError() from error
 
     def run(self, function, args, kwargs):
-        kernel_library, rpc_map = self.compile(function, args, kwargs)
+        rpc_map, kernel_library, symbolizer = self.compile(function, args, kwargs)
 
         if self.first_run:
             self.comm.check_ident()
             self.comm.switch_clock(self.external_clock)
             self.first_run = False
 
-        try:
+        self.comm.load(kernel_library)
-            self.comm.load(kernel_library)
-        except Exception as error:
-            shlib_temp = tempfile.NamedTemporaryFile(suffix=".so", delete=False)
-            shlib_temp.write(kernel_library)
-            shlib_temp.close()
-            raise RuntimeError("shared library dumped to {}".format(shlib_temp.name)) from error
-
         self.comm.run()
-        self.comm.serve(rpc_map)
+        self.comm.serve(rpc_map, symbolizer)
 
     @kernel
     def get_rtio_counter_mu(self):

artiq/language/core.py

@@ -279,8 +279,7 @@ class ARTIQException(Exception):
     """Base class for exceptions raised or passed through the core device."""
 
     # Try and create an instance of the specific class, if one exists.
-    def __new__(cls, name, message,
+    def __new__(cls, name, message, params, traceback):
-                params, filename, line, column, function):
         def find_subclass(cls):
             if cls.__name__ == name:
                 return cls
@@ -295,16 +294,13 @@ class ARTIQException(Exception):
             more_specific_cls = cls
 
         exn = Exception.__new__(more_specific_cls)
-        exn.__init__(name, message, params,
+        exn.__init__(name, message, params, traceback)
-                     filename, line, column, function)
         return exn
 
-    def __init__(self, name, message, params,
+    def __init__(self, name, message, params, traceback):
-                 filename, line, column, function):
         Exception.__init__(self, name, message, *params)
         self.name, self.message, self.params = name, message, params
-        self.filename, self.line, self.column = filename, line, column
+        self.traceback = list(traceback)
-        self.function = function
 
     def __str__(self):
         lines = []
@@ -315,11 +311,26 @@ class ARTIQException(Exception):
             lines.append("({}) {}".format(self.name, self.message.format(*self.params)))
 
         lines.append("Core Device Traceback (most recent call last):")
-        lines.append("  File \"{file}\", line {line}, column {column}, in {function}".
+        for (filename, line, column, function, address) in self.traceback:
-                     format(file=self.filename, line=self.line, column=self.column + 1,
+            source_line = linecache.getline(filename, line)
-                            function=self.function))
+            indentation = re.search(r"^\s*", source_line).end()
-        line = linecache.getline(self.filename, self.line)
+
-        lines.append("    {}".format(line.strip() if line else "<unknown>"))
+            if address is None:
-        lines.append("    {}^".format(" " * (self.column - re.search(r"^\s+", line).end())))
+                formatted_address = ""
+            else:
+                formatted_address = " (RA=0x{:x})".format(address)
+
+            if column == -1:
+                lines.append("  File \"{file}\", line {line}, in {function}{address}".
+                             format(file=filename, line=line, function=function,
+                                    address=formatted_address))
+                lines.append("    {}".format(source_line.strip() if source_line else "<unknown>"))
+            else:
+                lines.append("  File \"{file}\", line {line}, column {column},"
+                             " in {function}{address}".
+                             format(file=filename, line=line, column=column + 1,
+                                    function=function, address=formatted_address))
+                lines.append("    {}".format(source_line.strip() if source_line else "<unknown>"))
+                lines.append("    {}^".format(" " * (column - indentation)))
 
         return "\n".join(lines)