From f2f2e12b91ea5a7e2c25bce00c169a0e798427d7 Mon Sep 17 00:00:00 2001 From: pca006132 Date: Sat, 12 Feb 2022 22:32:21 +0800 Subject: [PATCH] language: implemented embedding map and exception --- artiq/coredevice/comm_kernel.py | 12 ++-- artiq/coredevice/core.py | 110 ++++++++++++++++++++++++++++++-- artiq/frontend/artiq_compile.py | 5 +- artiq/language/__init__.py | 2 + artiq/language/core.py | 2 +- artiq/language/embedding_map.py | 62 ++++++++++++++++++ 6 files changed, 177 insertions(+), 16 deletions(-) create mode 100644 artiq/language/embedding_map.py diff --git a/artiq/coredevice/comm_kernel.py b/artiq/coredevice/comm_kernel.py index e6551cb0a..3f9c70cdc 100644 --- a/artiq/coredevice/comm_kernel.py +++ b/artiq/coredevice/comm_kernel.py @@ -553,7 +553,7 @@ class CommKernel: if service_id == 0: def service(obj, attr, value): return setattr(obj, attr, value) else: - service = embedding_map.retrieve_object(service_id) + service = embedding_map.retrieve_function(service_id) logger.debug("rpc service: [%d]%r%s %r %r -> %s", service_id, service, (" (async)" if is_async else ""), args, kwargs, return_tags) @@ -622,7 +622,7 @@ class CommKernel: result, result, service) self._flush() - def _serve_exception(self, embedding_map, symbolizer, demangler): + def _serve_exception(self, embedding_map, symbolizer): exception_count = self._read_int32() nested_exceptions = [] @@ -646,10 +646,6 @@ class CommKernel: nested_exceptions.append([name, message, params, filename, line, column, function]) - demangled_names = demangler([ex[6] for ex in nested_exceptions]) - for i in range(exception_count): - nested_exceptions[i][6] = demangled_names[i] - exception_info = [] for _ in range(exception_count): sp = self._read_int32() @@ -689,13 +685,13 @@ class CommKernel: logger.warning(f"{(', '.join(errors[:-1]) + ' and ') if len(errors) > 1 else ''}{errors[-1]} " f"reported during kernel execution") - def serve(self, embedding_map, symbolizer, demangler): + def serve(self, embedding_map, symbolizer): while True: self._read_header() if self._read_type == Reply.RPCRequest: self._serve_rpc(embedding_map) elif self._read_type == Reply.KernelException: - self._serve_exception(embedding_map, symbolizer, demangler) + self._serve_exception(embedding_map, symbolizer) elif self._read_type == Reply.ClockFailure: raise exceptions.ClockFailure else: diff --git a/artiq/coredevice/core.py b/artiq/coredevice/core.py index 966838572..e4781fbec 100644 --- a/artiq/coredevice/core.py +++ b/artiq/coredevice/core.py @@ -1,4 +1,4 @@ -import os, sys +import os, sys, tempfile, subprocess, io from numpy import int32, int64 import nac3artiq @@ -6,6 +6,7 @@ import nac3artiq from artiq.language.core import * from artiq.language import core as core_language from artiq.language.units import * +from artiq.language.embedding_map import EmbeddingMap from artiq.coredevice.comm_kernel import CommKernel, CommKernelDummy @@ -55,11 +56,12 @@ class Core: self.core = self self.comm.core = self self.compiler = nac3artiq.NAC3(target) + self.embedding_map = EmbeddingMap() def close(self): self.comm.close() - def compile(self, method, args, kwargs, file_output=None): + def compile(self, method, args, kwargs, embedding_map, file_output=None): if core_language._allow_registration: self.compiler.analyze(core_language._registered_functions, core_language._registered_classes) core_language._allow_registration = False @@ -72,18 +74,21 @@ class Core: name = "" if file_output is None: - return self.compiler.compile_method_to_mem(obj, name, args) + return self.compiler.compile_method_to_mem(obj, name, args, embedding_map) else: - self.compiler.compile_method_to_file(obj, name, args, file_output) + self.compiler.compile_method_to_file(obj, name, args, file_output, embedding_map) def run(self, function, args, kwargs): - kernel_library = self.compile(function, args, kwargs) + kernel_library = self.compile(function, args, kwargs, self.embedding_map) if self.first_run: self.comm.check_system_info() self.first_run = False + + symbolizer = lambda addresses: symbolize(kernel_library, addresses) + self.comm.load(kernel_library) self.comm.run() - self.comm.serve(None, None, None) + self.comm.serve(self.embedding_map, symbolizer) @portable def seconds_to_mu(self, seconds: float) -> int64: @@ -155,3 +160,96 @@ class Core: min_now = rtio_get_counter() + int64(125000) if now_mu() < min_now: at_mu(min_now) + + +class RunTool: + def __init__(self, pattern, **tempdata): + self._pattern = pattern + self._tempdata = tempdata + self._tempnames = {} + self._tempfiles = {} + + def __enter__(self): + for key, data in self._tempdata.items(): + if data is None: + fd, filename = tempfile.mkstemp() + os.close(fd) + self._tempnames[key] = filename + else: + with tempfile.NamedTemporaryFile(delete=False) as f: + f.write(data) + self._tempnames[key] = f.name + + cmdline = [] + for argument in self._pattern: + cmdline.append(argument.format(**self._tempnames)) + + # https://bugs.python.org/issue17023 + windows = os.name == "nt" + process = subprocess.Popen(cmdline, stdout=subprocess.PIPE, stderr=subprocess.PIPE, + universal_newlines=True, shell=windows) + stdout, stderr = process.communicate() + if process.returncode != 0: + raise Exception("{} invocation failed: {}". + format(cmdline[0], stderr)) + + self._tempfiles["__stdout__"] = io.StringIO(stdout) + for key in self._tempdata: + if self._tempdata[key] is None: + self._tempfiles[key] = open(self._tempnames[key], "rb") + return self._tempfiles + + def __exit__(self, exc_typ, exc_value, exc_trace): + for file in self._tempfiles.values(): + file.close() + for filename in self._tempnames.values(): + os.unlink(filename) + + +def symbolize(library, addresses): + if addresses == []: + return [] + + # We got a list of return addresses, i.e. addresses of instructions + # just after the call. Offset them back to get an address somewhere + # inside the call instruction (or its delay slot), since that's what + # the backtrace entry should point at. + last_inlined = None + offset_addresses = [hex(addr - 1) for addr in addresses] + with RunTool(["llvm-addr2line", "--addresses", "--functions", "--inlines", + "--demangle", "--exe={library}"] + offset_addresses, + library=library) \ + as results: + lines = iter(results["__stdout__"].read().rstrip().split("\n")) + backtrace = [] + while True: + try: + address_or_function = next(lines) + except StopIteration: + break + if address_or_function[:2] == "0x": + address = int(address_or_function[2:], 16) + 1 # remove offset + function = next(lines) + inlined = False + else: + address = backtrace[-1][4] # inlined + function = address_or_function + inlined = True + location = next(lines) + + filename, line = location.rsplit(":", 1) + if filename == "??" or filename == "": + continue + if line == "?": + line = -1 + else: + line = int(line) + # can't get column out of addr2line D: + if inlined: + last_inlined.append((filename, line, -1, function, address)) + else: + last_inlined = [] + backtrace.append((filename, line, -1, function, address, + last_inlined)) + return backtrace + diff --git a/artiq/frontend/artiq_compile.py b/artiq/frontend/artiq_compile.py index 95be09c9c..32218c9ec 100755 --- a/artiq/frontend/artiq_compile.py +++ b/artiq/frontend/artiq_compile.py @@ -8,6 +8,7 @@ from artiq import __version__ as artiq_version from artiq.master.databases import DeviceDB, DatasetDB from artiq.master.worker_db import DeviceManager, DatasetManager from artiq.language.environment import ProcessArgumentManager +from artiq.language.embedding_map import EmbeddingMap from artiq.tools import * @@ -46,6 +47,8 @@ def main(): device_mgr = DeviceManager(DeviceDB(args.device_db)) dataset_mgr = DatasetManager(DatasetDB(args.dataset_db)) + embedding_map = EmbeddingMap() + output = args.output if output is None: basename, ext = os.path.splitext(args.file) @@ -60,7 +63,7 @@ def main(): if not getattr(exp.run, "__artiq_kernel__", False): raise ValueError("Experiment entry point must be a kernel") - exp_inst.core.compile(exp_inst.run, [], {}, file_output=output) + exp_inst.core.compile(exp_inst.run, [], {}, embedding_map, file_output=output) finally: device_mgr.close_devices() diff --git a/artiq/language/__init__.py b/artiq/language/__init__.py index 99c019a75..145471752 100644 --- a/artiq/language/__init__.py +++ b/artiq/language/__init__.py @@ -3,6 +3,7 @@ from artiq.language.core import * from artiq.language.environment import * from artiq.language.units import * from artiq.language.scan import * +from artiq.language.embedding_map import * from . import import_cache __all__ = ["import_cache"] @@ -10,3 +11,4 @@ __all__.extend(core.__all__) __all__.extend(environment.__all__) __all__.extend(units.__all__) __all__.extend(scan.__all__) +__all__.extend(embedding_map.__all__) diff --git a/artiq/language/core.py b/artiq/language/core.py index b9f5fe8a9..b2d0f2cec 100644 --- a/artiq/language/core.py +++ b/artiq/language/core.py @@ -104,7 +104,7 @@ def rpc(arg=None, flags={}): def inner_decorator(function): return rpc(function, flags) return inner_decorator - + return arg @nac3 class KernelContextManager: diff --git a/artiq/language/embedding_map.py b/artiq/language/embedding_map.py new file mode 100644 index 000000000..87d0ab9b6 --- /dev/null +++ b/artiq/language/embedding_map.py @@ -0,0 +1,62 @@ +__all__ = [ + 'EmbeddingMap' +] + +class EmbeddingMap: + def __init__(self): + self.object_inverse_map = {} + self.object_map = {} + self.string_map = {} + self.string_reverse_map = {} + self.function_map = {} + + # preallocate exception names + self.preallocate_runtime_exception_names(["RuntimeError", + "RTIOUnderflow", + "RTIOOverflow", + "RTIODestinationUnreachable", + "DMAError", + "I2CError", + "CacheError", + "SPIError", + "0:ZeroDivisionError", + "0:IndexError"]) + + def preallocate_runtime_exception_names(self, names): + for i, name in enumerate(names): + if ":" not in name: + name = "0:artiq.coredevice.exceptions." + name + exn_id = self.store_str(name) + assert exn_id == i + + def store_function(self, key, fun): + self.function_map[key] = fun + return key + + def store_object(self, obj): + obj_id = id(obj) + if obj_id in self.object_inverse_map: + return self.object_inverse_map[obj_id] + key = len(self.object_map) + self.object_map[key] = obj + self.object_inverse_map[obj_id] = key + return key + + def store_str(self, s): + if s in self.string_reverse_map: + return self.string_reverse_map[s] + key = len(self.string_map) + self.string_map[key] = s + self.string_reverse_map[s] = key + return key + + def retrieve_function(self, key): + return self.function_map[key] + + def retrieve_object(self, key): + return self.object_map[key] + + def retrieve_str(self, key): + return self.string_map[key] + +