From d446a3293ec2869aef4b2ec846b553975a38a01d Mon Sep 17 00:00:00 2001 From: whitequark Date: Wed, 16 May 2018 13:50:44 +0000 Subject: [PATCH] frontend: merge core{config,log,boot,debug,profile} into coremgmt. --- .../profiler.py} | 77 +------ artiq/frontend/aqctl_corelog.py | 81 ------- artiq/frontend/artiq_coreboot.py | 51 ----- artiq/frontend/artiq_coreconfig.py | 79 ------- artiq/frontend/artiq_coredebug.py | 43 ---- artiq/frontend/artiq_corelog.py | 56 ----- artiq/frontend/artiq_coremgmt.py | 200 ++++++++++++++++++ setup.py | 6 +- 8 files changed, 205 insertions(+), 388 deletions(-) rename artiq/{frontend/artiq_coreprofile.py => coredevice/profiler.py} (50%) mode change 100755 => 100644 delete mode 100755 artiq/frontend/aqctl_corelog.py delete mode 100755 artiq/frontend/artiq_coreboot.py delete mode 100755 artiq/frontend/artiq_coreconfig.py delete mode 100755 artiq/frontend/artiq_coredebug.py delete mode 100755 artiq/frontend/artiq_corelog.py create mode 100644 artiq/frontend/artiq_coremgmt.py diff --git a/artiq/frontend/artiq_coreprofile.py b/artiq/coredevice/profiler.py old mode 100755 new mode 100644 similarity index 50% rename from artiq/frontend/artiq_coreprofile.py rename to artiq/coredevice/profiler.py index 5b300e911..5bbbe1996 --- a/artiq/frontend/artiq_coreprofile.py +++ b/artiq/coredevice/profiler.py @@ -1,20 +1,11 @@ -#!/usr/bin/env python3 - -import argparse -import sys -import struct from collections import defaultdict import subprocess -from artiq.tools import verbosity_args, init_logger -from artiq.master.databases import DeviceDB -from artiq.coredevice.comm_mgmt import CommMgmt - class Symbolizer: - def __init__(self, binary): + def __init__(self, binary, triple): self._addr2line = subprocess.Popen([ - "or1k-linux-addr2line", "--exe=" + binary, + triple + "-addr2line", "--exe=" + binary, "--addresses", "--demangle=rust", "--functions", "--inlines" ], stdin=subprocess.PIPE, stdout=subprocess.PIPE, universal_newlines=True) @@ -39,13 +30,13 @@ class Symbolizer: class CallgrindWriter: - def __init__(self, output, binary, compression=True): + def __init__(self, output, binary, triple, compression=True): self._output = output self._binary = binary self._current = defaultdict(lambda: None) self._ids = defaultdict(lambda: {}) self._compression = compression - self._symbolizer = Symbolizer(binary) + self._symbolizer = Symbolizer(binary, triple) def _write(self, fmt, *args, **kwargs): self._output.write(fmt.format(*args, **kwargs)) @@ -93,63 +84,3 @@ class CallgrindWriter: self._spec("fn", function) self._spec("fl", file) self._write("0x{:08x} {} {}", caller, line, count) - - -def get_argparser(): - parser = argparse.ArgumentParser(description="ARTIQ core device profiling tool") - - verbosity_args(parser) - parser.add_argument("--device-db", default="device_db.py", - help="device database file (default: '%(default)s')") - - subparsers = parser.add_subparsers(dest="action") - subparsers.required = True - - p_start = subparsers.add_parser("start", - help="start profiling") - p_start.add_argument("--interval", metavar="MICROS", type=int, default=2000, - help="sampling interval, in microseconds") - p_start.add_argument("--hits-size", metavar="ENTRIES", type=int, default=8192, - help="hit buffer size") - p_start.add_argument("--edges-size", metavar="ENTRIES", type=int, default=0, - help="edge buffer size (edge profiling not implemented)") - - p_stop = subparsers.add_parser("stop", - help="stop profiling") - - p_save = subparsers.add_parser("save", - help="save profile") - p_save.add_argument("output", metavar="OUTPUT", type=argparse.FileType("w"), - help="file to save profile to, in Callgrind format") - p_save.add_argument("firmware", metavar="FIRMWARE", type=str, - help="path to firmware ELF file") - p_save.add_argument("--no-compression", default=False, action='store_true', - help="disable profile compression") - - return parser - - -def main(): - args = get_argparser().parse_args() - init_logger(args) - - core_addr = DeviceDB(args.device_db).get("core")["arguments"]["host"] - mgmt = CommMgmt(core_addr) - try: - if args.action == "start": - mgmt.start_profiler(args.interval, args.hits_size, args.edges_size) - elif args.action == "stop": - mgmt.stop_profiler() - elif args.action == "save": - hits, edges = mgmt.get_profile() - writer = CallgrindWriter(args.output, args.firmware, not args.no_compression) - writer.header() - for addr, count in hits.items(): - writer.hit(addr, count) - for (caller, callee), count in edges.items(): - writer.edge(caller, callee, count) - finally: - mgmt.close() - -if __name__ == "__main__": - main() diff --git a/artiq/frontend/aqctl_corelog.py b/artiq/frontend/aqctl_corelog.py deleted file mode 100755 index 7cee0b974..000000000 --- a/artiq/frontend/aqctl_corelog.py +++ /dev/null @@ -1,81 +0,0 @@ -#!/usr/bin/env python3 - -import argparse -import asyncio -import struct -import logging -import re - -from artiq.tools import * -from artiq.protocols.pc_rpc import Server -from artiq.protocols.logging import log_with_name -from artiq.coredevice.comm_mgmt import Request, Reply - - -def get_argparser(): - parser = argparse.ArgumentParser( - description="ARTIQ controller for core device logs") - simple_network_args(parser, 1068) - parser.add_argument("core_addr", - help="hostname or IP address of the core device") - return parser - - -class PingTarget: - def ping(self): - return True - - -async def get_logs(host): - reader, writer = await asyncio.open_connection(host, 1380) - writer.write(b"ARTIQ management\n") - writer.write(struct.pack("B", Request.PullLog.value)) - await writer.drain() - - while True: - length, = struct.unpack(">l", await reader.readexactly(4)) - log = await reader.readexactly(length) - - for line in log.decode("utf-8").splitlines(): - m = re.match(r"^\[.+?\] (TRACE|DEBUG| INFO| WARN|ERROR)\((.+?)\): (.+)$", line) - levelname = m.group(1) - if levelname == 'TRACE': - level = logging.TRACE - elif levelname == 'DEBUG': - level = logging.DEBUG - elif levelname == ' INFO': - level = logging.INFO - elif levelname == ' WARN': - level = logging.WARN - elif levelname == 'ERROR': - level = logging.ERROR - name = 'firmware.' + m.group(2).replace('::', '.') - text = m.group(3) - log_with_name(name, level, text) - - -def main(): - args = get_argparser().parse_args() - - loop = asyncio.get_event_loop() - try: - get_logs_task = asyncio.ensure_future(get_logs(args.core_addr)) - try: - server = Server({"corelog": PingTarget()}, None, True) - loop.run_until_complete(server.start(bind_address_from_args(args), args.port)) - try: - multiline_log_config(logging.TRACE) - loop.run_until_complete(server.wait_terminate()) - finally: - loop.run_until_complete(server.stop()) - finally: - get_logs_task.cancel() - try: - loop.run_until_complete(get_logs_task) - except asyncio.CancelledError: - pass - finally: - loop.close() - -if __name__ == "__main__": - main() diff --git a/artiq/frontend/artiq_coreboot.py b/artiq/frontend/artiq_coreboot.py deleted file mode 100755 index c0136292e..000000000 --- a/artiq/frontend/artiq_coreboot.py +++ /dev/null @@ -1,51 +0,0 @@ -#!/usr/bin/env python3 - -import argparse -import sys -import struct - -from artiq.tools import verbosity_args, init_logger -from artiq.master.databases import DeviceDB -from artiq.coredevice.comm_mgmt import CommMgmt - - -def get_argparser(): - parser = argparse.ArgumentParser(description="ARTIQ core device boot tool") - - verbosity_args(parser) - parser.add_argument("--device-db", default="device_db.py", - help="device database file (default: '%(default)s')") - - subparsers = parser.add_subparsers(dest="action") - - p_reboot = subparsers.add_parser("reboot", - help="reboot the currently running firmware") - - p_hotswap = subparsers.add_parser("hotswap", - help="load the specified firmware in RAM") - - p_hotswap.add_argument("image", metavar="IMAGE", type=argparse.FileType("rb"), - help="runtime image to be executed") - - return parser - - -def main(): - args = get_argparser().parse_args() - init_logger(args) - - core_addr = DeviceDB(args.device_db).get("core")["arguments"]["host"] - mgmt = CommMgmt(core_addr) - try: - if args.action == "reboot": - mgmt.reboot() - elif args.action == "hotswap": - mgmt.hotswap(args.image.read()) - else: - print("An action needs to be specified.", file=sys.stderr) - sys.exit(1) - finally: - mgmt.close() - -if __name__ == "__main__": - main() diff --git a/artiq/frontend/artiq_coreconfig.py b/artiq/frontend/artiq_coreconfig.py deleted file mode 100755 index 6f1ff2e42..000000000 --- a/artiq/frontend/artiq_coreconfig.py +++ /dev/null @@ -1,79 +0,0 @@ -#!/usr/bin/env python3 - -import argparse -import struct - -from artiq.tools import verbosity_args, init_logger -from artiq.master.databases import DeviceDB -from artiq.master.worker_db import DeviceManager - - -def get_argparser(): - parser = argparse.ArgumentParser(description="ARTIQ core device " - "configuration tool") - - verbosity_args(parser) - parser.add_argument("--device-db", default="device_db.py", - help="device database file (default: '%(default)s')") - - subparsers = parser.add_subparsers(dest="action") - subparsers.required = True - - p_read = subparsers.add_parser("read", - help="read key from core device config") - p_read.add_argument("key", metavar="KEY", type=str, - help="key to be read from core device config") - - p_write = subparsers.add_parser("write", - help="write key-value records to core " - "device config") - p_write.add_argument("-s", "--string", nargs=2, action="append", - default=[], metavar=("KEY", "STRING"), type=str, - help="key-value records to be written to core device " - "config") - p_write.add_argument("-f", "--file", nargs=2, action="append", - type=str, default=[], - metavar=("KEY", "FILENAME"), - help="key and file whose content to be written to " - "core device config") - - p_delete = subparsers.add_parser("delete", - help="delete key from core device config") - p_delete.add_argument("key", metavar="KEY", nargs=argparse.REMAINDER, - default=[], type=str, - help="key to be deleted from core device config") - - subparsers.add_parser("erase", help="fully erase core device config") - return parser - - -def main(): - args = get_argparser().parse_args() - init_logger(args) - device_mgr = DeviceManager(DeviceDB(args.device_db)) - try: - comm = device_mgr.get("core").comm - comm.check_system_info() - - if args.action == "read": - value = comm.flash_storage_read(args.key) - if not value: - print("Key {} does not exist".format(args.key)) - else: - print(value) - elif args.action == "write": - for key, value in args.string: - comm.flash_storage_write(key, value.encode("utf-8")) - for key, filename in args.file: - with open(filename, "rb") as fi: - comm.flash_storage_write(key, fi.read()) - elif args.action == "delete": - for key in args.key: - comm.flash_storage_remove(key) - elif args.action == "erase": - comm.flash_storage_erase() - finally: - device_mgr.close_devices() - -if __name__ == "__main__": - main() diff --git a/artiq/frontend/artiq_coredebug.py b/artiq/frontend/artiq_coredebug.py deleted file mode 100755 index 55631e037..000000000 --- a/artiq/frontend/artiq_coredebug.py +++ /dev/null @@ -1,43 +0,0 @@ -#!/usr/bin/env python3 - -import argparse -import sys -import struct - -from artiq.tools import verbosity_args, init_logger -from artiq.master.databases import DeviceDB -from artiq.coredevice.comm_mgmt import CommMgmt - - -def get_argparser(): - parser = argparse.ArgumentParser(description="ARTIQ core device debug tool") - - verbosity_args(parser) - parser.add_argument("--device-db", default="device_db.py", - help="device database file (default: '%(default)s')") - - subparsers = parser.add_subparsers(dest="action") - - p_allocator = subparsers.add_parser("allocator", - help="show heap layout") - - return parser - - -def main(): - args = get_argparser().parse_args() - init_logger(args) - - core_addr = DeviceDB(args.device_db).get("core")["arguments"]["host"] - mgmt = CommMgmt(core_addr) - try: - if args.action == "allocator": - mgmt.debug_allocator() - else: - print("An action needs to be specified.", file=sys.stderr) - sys.exit(1) - finally: - mgmt.close() - -if __name__ == "__main__": - main() diff --git a/artiq/frontend/artiq_corelog.py b/artiq/frontend/artiq_corelog.py deleted file mode 100755 index 0f7cd8759..000000000 --- a/artiq/frontend/artiq_corelog.py +++ /dev/null @@ -1,56 +0,0 @@ -#!/usr/bin/env python3 - -import argparse - -from artiq.tools import verbosity_args, init_logger -from artiq.master.databases import DeviceDB -from artiq.coredevice.comm_mgmt import CommMgmt - - -def get_argparser(): - parser = argparse.ArgumentParser(description="ARTIQ core device " - "log tool") - verbosity_args(parser) - parser.add_argument("--device-db", default="device_db.py", - help="device database file (default: '%(default)s')") - - subparsers = parser.add_subparsers(dest="action") - - p_clear = subparsers.add_parser("clear", - help="clear log buffer") - - p_set_level = subparsers.add_parser("set_level", - help="set minimum level for messages to be logged") - p_set_level.add_argument("level", metavar="LEVEL", type=str, - help="log level (one of: OFF ERROR WARN INFO DEBUG TRACE)") - - p_set_uart_level = subparsers.add_parser("set_uart_level", - help="set minimum level for messages to be logged " - "to UART") - p_set_uart_level.add_argument("level", metavar="LEVEL", type=str, - help="log level (one of: OFF ERROR WARN INFO DEBUG TRACE)") - - return parser - - -def main(): - args = get_argparser().parse_args() - init_logger(args) - - core_addr = DeviceDB(args.device_db).get("core")["arguments"]["host"] - mgmt = CommMgmt(core_addr) - try: - if args.action == "set_level": - mgmt.set_log_level(args.level) - elif args.action == "set_uart_level": - mgmt.set_uart_log_level(args.level) - elif args.action == "clear": - mgmt.clear_log() - else: - print(mgmt.get_log(), end="") - finally: - mgmt.close() - - -if __name__ == "__main__": - main() diff --git a/artiq/frontend/artiq_coremgmt.py b/artiq/frontend/artiq_coremgmt.py new file mode 100644 index 000000000..6013b7359 --- /dev/null +++ b/artiq/frontend/artiq_coremgmt.py @@ -0,0 +1,200 @@ +#!/usr/bin/env python3 + +import argparse +import struct + +from artiq.tools import verbosity_args, init_logger +from artiq.master.databases import DeviceDB +from artiq.master.worker_db import DeviceManager +from artiq.coredevice.comm_kernel import CommKernel +from artiq.coredevice.comm_mgmt import CommMgmt +from artiq.coredevice.profiler import CallgrindWriter + + +def get_argparser(): + parser = argparse.ArgumentParser(description="ARTIQ core device " + "management tool") + + verbosity_args(parser) + parser.add_argument("--device-db", default="device_db.py", + help="device database file (default: '%(default)s')") + + tools = parser.add_subparsers(dest="tool") + tools.required = True + + # logging + t_log = tools.add_parser("log", + help="read logs and change log levels") + + subparsers = t_log.add_subparsers(dest="action") + + p_clear = subparsers.add_parser("clear", + help="clear log buffer") + + p_set_level = subparsers.add_parser("set_level", + help="set minimum level for messages to be logged") + p_set_level.add_argument("level", metavar="LEVEL", type=str, + help="log level (one of: OFF ERROR WARN INFO DEBUG TRACE)") + + p_set_uart_level = subparsers.add_parser("set_uart_level", + help="set minimum level for messages to be logged " + "to UART") + p_set_uart_level.add_argument("level", metavar="LEVEL", type=str, + help="log level (one of: OFF ERROR WARN INFO DEBUG TRACE)") + + # configuration + t_config = tools.add_parser("config", + help="read and change core device configuration") + + subparsers = t_config.add_subparsers(dest="action") + subparsers.required = True + + p_read = subparsers.add_parser("read", + help="read key from core device config") + p_read.add_argument("key", metavar="KEY", type=str, + help="key to be read from core device config") + + p_write = subparsers.add_parser("write", + help="write key-value records to core " + "device config") + p_write.add_argument("-s", "--string", nargs=2, action="append", + default=[], metavar=("KEY", "STRING"), type=str, + help="key-value records to be written to core device " + "config") + p_write.add_argument("-f", "--file", nargs=2, action="append", + type=str, default=[], + metavar=("KEY", "FILENAME"), + help="key and file whose content to be written to " + "core device config") + + p_delete = subparsers.add_parser("delete", + help="delete key from core device config") + p_delete.add_argument("key", metavar="KEY", nargs=argparse.REMAINDER, + default=[], type=str, + help="key to be deleted from core device config") + + subparsers.add_parser("erase", help="fully erase core device config") + + # booting + t_boot = tools.add_parser("reboot", + help="reboot the currently running firmware") + + t_hotswap = tools.add_parser("hotswap", + help="load the specified firmware in RAM") + + t_hotswap.add_argument("image", metavar="IMAGE", type=argparse.FileType("rb"), + help="runtime image to be executed") + + # profiling + t_profile = tools.add_parser("profile", + help="account for communications CPU time") + + subparsers = t_profile.add_subparsers(dest="action") + subparsers.required = True + + p_start = subparsers.add_parser("start", + help="start profiling") + p_start.add_argument("--interval", metavar="MICROS", type=int, default=2000, + help="sampling interval, in microseconds") + p_start.add_argument("--hits-size", metavar="ENTRIES", type=int, default=8192, + help="hit buffer size") + p_start.add_argument("--edges-size", metavar="ENTRIES", type=int, default=0, + help="edge buffer size (edge profiling not implemented)") + + p_stop = subparsers.add_parser("stop", + help="stop profiling") + + p_save = subparsers.add_parser("save", + help="save profile") + p_save.add_argument("output", metavar="OUTPUT", type=argparse.FileType("w"), + help="file to save profile to, in Callgrind format") + p_save.add_argument("firmware", metavar="FIRMWARE", type=str, + help="path to firmware ELF file") + p_save.add_argument("--no-compression", + dest="compression", default=True, action="store_false", + help="disable profile compression") + + # misc debug + t_debug = tools.add_parser("debug", + help="specialized debug functions") + + subparsers = t_debug.add_subparsers(dest="action") + subparsers.required = True + + p_allocator = subparsers.add_parser("allocator", + help="show heap layout") + + return parser + + +def main(): + args = get_argparser().parse_args() + init_logger(args) + + device_mgr = DeviceManager(DeviceDB(args.device_db)) + try: + core_addr = DeviceDB(args.device_db).get("core")["arguments"]["host"] + kern = CommKernel(core_addr) + mgmt = CommMgmt(core_addr) + + kern.check_system_info() + + if args.tool == "log": + if args.action == "set_level": + mgmt.set_log_level(args.level) + if args.action == "set_uart_level": + mgmt.set_uart_log_level(args.level) + if args.action == "clear": + mgmt.clear_log() + if args.action == None: + print(mgmt.get_log(), end="") + + if args.tool == "config": + if args.action == "read": + value = kern.flash_storage_read(args.key) + if not value: + print("Key {} does not exist".format(args.key)) + else: + print(value) + if args.action == "write": + for key, value in args.string: + kern.flash_storage_write(key, value.encode("utf-8")) + for key, filename in args.file: + with open(filename, "rb") as fi: + kern.flash_storage_write(key, fi.read()) + if args.action == "delete": + for key in args.key: + kern.flash_storage_remove(key) + if args.action == "erase": + kern.flash_storage_erase() + + if args.tool == "reboot": + mgmt.reboot() + + if args.tool == "hotswap": + mgmt.hotswap(args.image.read()) + + if args.tool == "profile": + if args.action == "start": + mgmt.start_profiler(args.interval, args.hits_size, args.edges_size) + elif args.action == "stop": + mgmt.stop_profiler() + elif args.action == "save": + hits, edges = mgmt.get_profile() + writer = CallgrindWriter(args.output, args.firmware, + "or1k-linux", args.compression) + writer.header() + for addr, count in hits.items(): + writer.hit(addr, count) + for (caller, callee), count in edges.items(): + writer.edge(caller, callee, count) + + if args.tool == "debug": + if args.action == "allocator": + mgmt.debug_allocator() + + finally: + device_mgr.close_devices() + +if __name__ == "__main__": + main() diff --git a/setup.py b/setup.py index 8806a4f65..a85f23ae5 100755 --- a/setup.py +++ b/setup.py @@ -22,11 +22,7 @@ console_scripts = [ "artiq_client = artiq.frontend.artiq_client:main", "artiq_compile = artiq.frontend.artiq_compile:main", "artiq_coreanalyzer = artiq.frontend.artiq_coreanalyzer:main", - "artiq_coreconfig = artiq.frontend.artiq_coreconfig:main", - "artiq_corelog = artiq.frontend.artiq_corelog:main", - "artiq_coreboot = artiq.frontend.artiq_coreboot:main", - "artiq_coredebug = artiq.frontend.artiq_coredebug:main", - "artiq_coreprofile = artiq.frontend.artiq_coreprofile:main", + "artiq_coremgmt = artiq.frontend.artiq_coremgmt:main", "artiq_ctlmgr = artiq.frontend.artiq_ctlmgr:main", "artiq_devtool = artiq.frontend.artiq_devtool:main", "artiq_pcap = artiq.frontend.artiq_pcap:main",