language: implemented embedding map and exception

artiq/coredevice/comm_kernel.py

@@ -553,7 +553,7 @@ class CommKernel:
         if service_id == 0:
             def service(obj, attr, value): return setattr(obj, attr, value)
         else:
-            service = embedding_map.retrieve_object(service_id)
+            service = embedding_map.retrieve_function(service_id)
         logger.debug("rpc service: [%d]%r%s %r %r -> %s", service_id, service,
                      (" (async)" if is_async else ""), args, kwargs, return_tags)
 
@@ -622,7 +622,7 @@ class CommKernel:
                                  result, result, service)
             self._flush()
 
-    def _serve_exception(self, embedding_map, symbolizer, demangler):
+    def _serve_exception(self, embedding_map, symbolizer):
         exception_count = self._read_int32()
         nested_exceptions = []
 
@@ -646,10 +646,6 @@ class CommKernel:
             nested_exceptions.append([name, message, params,
                                       filename, line, column, function])
 
-        demangled_names = demangler([ex[6] for ex in nested_exceptions])
-        for i in range(exception_count):
-            nested_exceptions[i][6] = demangled_names[i]
-
         exception_info = []
         for _ in range(exception_count):
             sp = self._read_int32()
@@ -689,13 +685,13 @@ class CommKernel:
             logger.warning(f"{(', '.join(errors[:-1]) + ' and ') if len(errors) > 1 else ''}{errors[-1]} "
                            f"reported during kernel execution")
 
-    def serve(self, embedding_map, symbolizer, demangler):
+    def serve(self, embedding_map, symbolizer):
         while True:
             self._read_header()
             if self._read_type == Reply.RPCRequest:
                 self._serve_rpc(embedding_map)
             elif self._read_type == Reply.KernelException:
-                self._serve_exception(embedding_map, symbolizer, demangler)
+                self._serve_exception(embedding_map, symbolizer)
             elif self._read_type == Reply.ClockFailure:
                 raise exceptions.ClockFailure
             else:

artiq/coredevice/core.py

@@ -1,4 +1,4 @@
-import os, sys
+import os, sys, tempfile, subprocess, io
 from numpy import int32, int64
 
 import nac3artiq
@@ -6,6 +6,7 @@ import nac3artiq
 from artiq.language.core import *
 from artiq.language import core as core_language
 from artiq.language.units import *
+from artiq.language.embedding_map import EmbeddingMap
 
 from artiq.coredevice.comm_kernel import CommKernel, CommKernelDummy
 
@@ -55,11 +56,12 @@ class Core:
         self.core = self
         self.comm.core = self
         self.compiler = nac3artiq.NAC3(target)
+        self.embedding_map = EmbeddingMap()
 
     def close(self):
         self.comm.close()
 
-    def compile(self, method, args, kwargs, file_output=None):
+    def compile(self, method, args, kwargs, embedding_map, file_output=None):
         if core_language._allow_registration:
             self.compiler.analyze(core_language._registered_functions, core_language._registered_classes)
             core_language._allow_registration = False
@@ -72,18 +74,21 @@ class Core:
             name = ""
 
         if file_output is None:
-            return self.compiler.compile_method_to_mem(obj, name, args)
+            return self.compiler.compile_method_to_mem(obj, name, args, embedding_map)
         else:
-            self.compiler.compile_method_to_file(obj, name, args, file_output)
+            self.compiler.compile_method_to_file(obj, name, args, file_output, embedding_map)
 
     def run(self, function, args, kwargs):
-        kernel_library = self.compile(function, args, kwargs)
+        kernel_library = self.compile(function, args, kwargs, self.embedding_map)
         if self.first_run:
             self.comm.check_system_info()
             self.first_run = False
+
+        symbolizer = lambda addresses: symbolize(kernel_library, addresses)
+
         self.comm.load(kernel_library)
         self.comm.run()
-        self.comm.serve(None, None, None)
+        self.comm.serve(self.embedding_map, symbolizer)
 
     @portable
     def seconds_to_mu(self, seconds: float) -> int64:
@@ -155,3 +160,96 @@ class Core:
         min_now = rtio_get_counter() + int64(125000)
         if now_mu() < min_now:
             at_mu(min_now)
+
+
+class RunTool:
+    def __init__(self, pattern, **tempdata):
+        self._pattern   = pattern
+        self._tempdata  = tempdata
+        self._tempnames = {}
+        self._tempfiles = {}
+
+    def __enter__(self):
+        for key, data in self._tempdata.items():
+            if data is None:
+                fd, filename = tempfile.mkstemp()
+                os.close(fd)
+                self._tempnames[key] = filename
+            else:
+                with tempfile.NamedTemporaryFile(delete=False) as f:
+                    f.write(data)
+                    self._tempnames[key] = f.name
+
+        cmdline = []
+        for argument in self._pattern:
+            cmdline.append(argument.format(**self._tempnames))
+
+        # https://bugs.python.org/issue17023
+        windows = os.name == "nt"
+        process = subprocess.Popen(cmdline, stdout=subprocess.PIPE, stderr=subprocess.PIPE,
+                                   universal_newlines=True, shell=windows)
+        stdout, stderr = process.communicate()
+        if process.returncode != 0:
+            raise Exception("{} invocation failed: {}".
+                            format(cmdline[0], stderr))
+
+        self._tempfiles["__stdout__"] = io.StringIO(stdout)
+        for key in self._tempdata:
+            if self._tempdata[key] is None:
+                self._tempfiles[key] = open(self._tempnames[key], "rb")
+        return self._tempfiles
+
+    def __exit__(self, exc_typ, exc_value, exc_trace):
+        for file in self._tempfiles.values():
+            file.close()
+        for filename in self._tempnames.values():
+            os.unlink(filename)
+
+
+def symbolize(library, addresses):
+    if addresses == []:
+        return []
+
+    # We got a list of return addresses, i.e. addresses of instructions
+    # just after the call. Offset them back to get an address somewhere
+    # inside the call instruction (or its delay slot), since that's what
+    # the backtrace entry should point at.
+    last_inlined = None
+    offset_addresses = [hex(addr - 1) for addr in addresses]
+    with RunTool(["llvm-addr2line", "--addresses",  "--functions", "--inlines",
+                  "--demangle", "--exe={library}"] + offset_addresses,
+                 library=library) \
+            as results:
+        lines = iter(results["__stdout__"].read().rstrip().split("\n"))
+        backtrace = []
+        while True:
+            try:
+                address_or_function = next(lines)
+            except StopIteration:
+                break
+            if address_or_function[:2] == "0x":
+                address  = int(address_or_function[2:], 16) + 1 # remove offset
+                function = next(lines)
+                inlined = False
+            else:
+                address  = backtrace[-1][4] # inlined
+                function = address_or_function
+                inlined = True
+            location = next(lines)
+
+            filename, line = location.rsplit(":", 1)
+            if filename == "??" or filename == "<synthesized>":
+                continue
+            if line == "?":
+                line = -1
+            else:
+                line = int(line)
+            # can't get column out of addr2line D:
+            if inlined:
+                last_inlined.append((filename, line, -1, function, address))
+            else:
+                last_inlined = []
+                backtrace.append((filename, line, -1, function, address,
+                                  last_inlined))
+        return backtrace
+

artiq/frontend/artiq_compile.py

@@ -8,6 +8,7 @@ from artiq import __version__ as artiq_version
 from artiq.master.databases import DeviceDB, DatasetDB
 from artiq.master.worker_db import DeviceManager, DatasetManager
 from artiq.language.environment import ProcessArgumentManager
+from artiq.language.embedding_map import EmbeddingMap
 from artiq.tools import *
 
 
@@ -46,6 +47,8 @@ def main():
     device_mgr = DeviceManager(DeviceDB(args.device_db))
     dataset_mgr = DatasetManager(DatasetDB(args.dataset_db))
 
+    embedding_map = EmbeddingMap()
+
     output = args.output
     if output is None:
         basename, ext = os.path.splitext(args.file)
@@ -60,7 +63,7 @@ def main():
 
         if not getattr(exp.run, "__artiq_kernel__", False):
             raise ValueError("Experiment entry point must be a kernel")
-        exp_inst.core.compile(exp_inst.run, [], {}, file_output=output)
+        exp_inst.core.compile(exp_inst.run, [], {}, embedding_map, file_output=output)
     finally:
         device_mgr.close_devices()
 

artiq/language/__init__.py

@@ -3,6 +3,7 @@ from artiq.language.core import *
 from artiq.language.environment import *
 from artiq.language.units import *
 from artiq.language.scan import *
+from artiq.language.embedding_map import *
 from . import import_cache
 
 __all__ = ["import_cache"]
@@ -10,3 +11,4 @@ __all__.extend(core.__all__)
 __all__.extend(environment.__all__)
 __all__.extend(units.__all__)
 __all__.extend(scan.__all__)
+__all__.extend(embedding_map.__all__)

artiq/language/core.py

@@ -104,7 +104,7 @@ def rpc(arg=None, flags={}):
         def inner_decorator(function):
             return rpc(function, flags)
         return inner_decorator
-
+    return arg
 
 @nac3
 class KernelContextManager:

artiq/language/embedding_map.py

@@ -0,0 +1,62 @@
+__all__ = [
+    'EmbeddingMap'
+]
+
+class EmbeddingMap:
+    def __init__(self):
+        self.object_inverse_map = {}
+        self.object_map = {}
+        self.string_map = {}
+        self.string_reverse_map = {}
+        self.function_map = {}
+
+        # preallocate exception names
+        self.preallocate_runtime_exception_names(["RuntimeError",
+                                          "RTIOUnderflow",
+                                          "RTIOOverflow",
+                                          "RTIODestinationUnreachable",
+                                          "DMAError",
+                                          "I2CError",
+                                          "CacheError",
+                                          "SPIError",
+                                          "0:ZeroDivisionError",
+                                          "0:IndexError"])
+
+    def preallocate_runtime_exception_names(self, names):
+        for i, name in enumerate(names):
+            if ":" not in name:
+                name = "0:artiq.coredevice.exceptions." + name
+            exn_id = self.store_str(name)
+            assert exn_id == i
+
+    def store_function(self, key, fun):
+        self.function_map[key] = fun
+        return key
+
+    def store_object(self, obj):
+        obj_id = id(obj)
+        if obj_id in self.object_inverse_map:
+            return self.object_inverse_map[obj_id]
+        key = len(self.object_map)
+        self.object_map[key] = obj
+        self.object_inverse_map[obj_id] = key
+        return key
+
+    def store_str(self, s):
+        if s in self.string_reverse_map:
+            return self.string_reverse_map[s]
+        key = len(self.string_map)
+        self.string_map[key] = s
+        self.string_reverse_map[s] = key
+        return key
+
+    def retrieve_function(self, key):
+        return self.function_map[key]
+
+    def retrieve_object(self, key):
+        return self.object_map[key]
+
+    def retrieve_str(self, key):
+        return self.string_map[key]
+
+