runtime: new serial protocol, support multiple entry points and log messages

artiq/devices/core.py

@@ -24,5 +24,6 @@ class Core:
         fold_constants(func_def)
 
         binary = get_runtime_binary(self.runtime_env, func_def)
-        self.core_com.run(binary)
+        self.core_com.load(binary)
+        self.core_com.run(func_def.name)
         self.core_com.serve(rpc_map)

artiq/devices/corecom_dummy.py

@@ -16,12 +16,15 @@ class CoreCom:
     def get_runtime_env(self):
         return _RuntimeEnvironment(10*ns)
 
-    def run(self, kcode):
+    def load(self, kcode):
         print("================")
         print(" LLVM IR")
         print("================")
         print(kcode)
 
+    def run(self, kname):
+        print("RUN: "+kname)
+
     def serve(self, rpc_map):
         print("================")
         print(" RPC map")

artiq/devices/corecom_serial.py

@@ -12,11 +12,23 @@ class UnsupportedDevice(Exception):
     pass
 
 
-class _MsgType(Enum):
+class _H2DMsgType(Enum):
-    REQUEST_IDENT = 0x01
+    REQUEST_IDENT = 1
-    LOAD_KERNEL = 0x02
+    LOAD_OBJECT = 2
-    KERNEL_FINISHED = 0x03
+    RUN_KERNEL = 3
-    RPC_REQUEST = 0x04
+
+
+class _D2HMsgType(Enum):
+    LOG = 1
+    MESSAGE_UNRECOGNIZED = 2
+    IDENT = 3
+    OBJECT_LOADED = 4
+    INCORRECT_LENGTH = 5
+    CRC_FAILED = 6
+    OBJECT_UNRECOGNIZED = 7
+    KERNEL_FINISHED = 8
+    KERNEL_STARTUP_FAILED = 9
+    RPC_REQUEST = 10
 
 
 def _write_exactly(f, data):
@@ -63,9 +75,8 @@ class CoreCom:
     def __exit__(self, type, value, traceback):
         self.close()
 
-    def get_runtime_env(self):
+    def _get_device_msg(self):
-        _write_exactly(self.port, struct.pack(
+        while True:
-            ">lb", 0x5a5a5a5a, _MsgType.REQUEST_IDENT.value))
             # FIXME: when loading immediately after a board reset,
             # we erroneously get some zeros back.
             # Ignore them with a warning for now.
@@ -79,6 +90,24 @@ class CoreCom:
             if spurious_zero_count:
                 print("Warning: received {} spurious zeros"
                       .format(spurious_zero_count))
+            msg = _D2HMsgType(reply)
+            if msg == _D2HMsgType.LOG:
+                (length, ) = struct.unpack(">h", _read_exactly(self.port, 2))
+                log_message = ""
+                for i in range(length):
+                    (c, ) = struct.unpack("b", _read_exactly(self.port, 1))
+                    log_message += chr(c)
+                print("DEVICE LOG: " + log_message)
+            else:
+                return msg
+
+    def get_runtime_env(self):
+        _write_exactly(self.port, struct.pack(
+            ">lb", 0x5a5a5a5a, _H2DMsgType.REQUEST_IDENT.value))
+        msg = self._get_device_msg()
+        if msg != _D2HMsgType.IDENT:
+            raise IOError("Incorrect reply from device: "+str(msg))
+        (reply, ) = struct.unpack("b", _read_exactly(self.port, 1))
         runtime_id = chr(reply)
         for i in range(3):
             (reply, ) = struct.unpack("b", _read_exactly(self.port, 1))
@@ -88,32 +117,28 @@ class CoreCom:
         (ref_period, ) = struct.unpack(">l", _read_exactly(self.port, 4))
         return Environment(ref_period*units.ps)
 
-    def run(self, kcode):
+    def load(self, kcode):
         _write_exactly(self.port, struct.pack(
             ">lblL",
-            0x5a5a5a5a, _MsgType.LOAD_KERNEL.value,
+            0x5a5a5a5a, _H2DMsgType.LOAD_OBJECT.value,
             len(kcode), zlib.crc32(kcode)))
         _write_exactly(self.port, kcode)
-        (reply, ) = struct.unpack("b", _read_exactly(self.port, 1))
+        msg = self._get_device_msg()
-        if reply != 0x4f:
+        if msg != _D2HMsgType.OBJECT_LOADED:
-            raise IOError("Incorrect reply from device: "+hex(reply))
+            raise IOError("Incorrect reply from device: "+str(msg))
 
-    def _wait_sync(self):
+    def run(self, kname):
-        recognized = 0
+        _write_exactly(self.port, struct.pack(
-        while recognized < 4:
+            ">lbl", 0x5a5a5a5a, _H2DMsgType.RUN_KERNEL.value, len(kname)))
-            (c, ) = struct.unpack("b", _read_exactly(self.port, 1))
+        for c in kname:
-            if c == 0x5a:
+            _write_exactly(self.port, struct.pack("b", ord(c)))
-                recognized += 1
-            else:
-                recognized = 0
 
     def serve(self, rpc_map):
         while True:
-            self._wait_sync()
+            msg = self._get_device_msg()
-            msg = _MsgType(*struct.unpack("b", _read_exactly(self.port, 1)))
+            if msg == _D2HMsgType.KERNEL_FINISHED:
-            if msg == _MsgType.KERNEL_FINISHED:
                 return
-            elif msg == _MsgType.RPC_REQUEST:
+            elif msg == _D2HMsgType.RPC_REQUEST:
                 rpc_num, n_args = struct.unpack(">hb",
                                                 _read_exactly(self.port, 3))
                 args = []
@@ -124,3 +149,5 @@ class CoreCom:
                 if r is None:
                     r = 0
                 _write_exactly(self.port, struct.pack(">l", r))
+            else:
+                raise IOError("Incorrect request from device: "+str(msg))

soc/runtime/Makefile

@@ -3,7 +3,7 @@ include $(MSCDIR)/software/common.mak
 BOARD=papilio_pro
 SERIAL=/dev/ttyUSB1
 
-OBJECTS=isr.o elf_loader.o symbols.o corecom_serial.o gpio.o rtio.o dds.o main.o
+OBJECTS=isr.o elf_loader.o services.o corecom_serial.o gpio.o rtio.o dds.o main.o
 
 all: runtime.bin
 

soc/runtime/corecom.h

@@ -1,8 +1,11 @@
 #ifndef __CORECOM_H
 #define __CORECOM_H
 
-int ident_and_download_kernel(void *buffer, int maxlength);
+typedef int (*object_loader)(void *, int);
-int rpc(int rpc_num, int n_args, ...);
+typedef int (*kernel_runner)(const char *);
-void kernel_finished(void);
+
+void corecom_serve(object_loader load_object, kernel_runner run_kernel);
+int corecom_rpc(int rpc_num, int n_args, ...);
+void corecom_log(const char *fmt, ...);
 
 #endif /* __CORECOM_H */

soc/runtime/corecom_serial.c

@@ -5,11 +5,29 @@
 
 #include "corecom.h"
 
+/* host to device */
 enum {
-    MSGTYPE_REQUEST_IDENT = 0x01,
+    MSGTYPE_REQUEST_IDENT = 1,
-    MSGTYPE_LOAD_KERNEL = 0x02,
+    MSGTYPE_LOAD_OBJECT,
-    MSGTYPE_KERNEL_FINISHED = 0x03,
+    MSGTYPE_RUN_KERNEL,
-    MSGTYPE_RPC_REQUEST = 0x04,
+};
+
+/* device to host */
+enum {
+    MSGTYPE_LOG = 1,
+    MSGTYPE_MESSAGE_UNRECOGNIZED,
+
+    MSGTYPE_IDENT,
+
+    MSGTYPE_OBJECT_LOADED,
+    MSGTYPE_INCORRECT_LENGTH,
+    MSGTYPE_CRC_FAILED,
+    MSGTYPE_OBJECT_UNRECOGNIZED,
+
+    MSGTYPE_KERNEL_FINISHED,
+    MSGTYPE_KERNEL_STARTUP_FAILED,
+
+    MSGTYPE_RPC_REQUEST,
 };
 
 static int receive_int(void)
@@ -66,48 +84,73 @@ static void receive_sync(void)
     }
 }
 
-static void send_sync(void)
+static void receive_and_load_object(object_loader load_object)
-{
-    send_int(0x5a5a5a5a);
-}
-
-int ident_and_download_kernel(void *buffer, int maxlength)
 {
     int length;
-    unsigned int crc;
     int i;
+    unsigned char buffer[256*1024];
+    unsigned int crc;
+
+    length = receive_int();
+    if(length > sizeof(buffer)) {
+        send_char(MSGTYPE_INCORRECT_LENGTH);
+        return;
+    }
+    crc = receive_int();
+    for(i=0;i<length;i++)
+        buffer[i] = receive_char();
+    if(crc32(buffer, length) != crc) {
+        send_char(MSGTYPE_CRC_FAILED);
+        return;
+    }
+    if(load_object(buffer, length))
+        send_char(MSGTYPE_OBJECT_LOADED);
+    else
+        send_char(MSGTYPE_OBJECT_UNRECOGNIZED);
+}
+
+static void receive_and_run_kernel(kernel_runner run_kernel)
+{
+    int length;
+    int i;
+    char kernel_name[256];
+    int r;
+
+    length = receive_int();
+    if(length > (sizeof(kernel_name)-1)) {
+        send_char(MSGTYPE_INCORRECT_LENGTH);
+        return;
+    }
+    for(i=0;i<length;i++)
+        kernel_name[i] = receive_char();
+    kernel_name[length] = 0;
+
+    r = run_kernel(kernel_name);
+    send_char(r ? MSGTYPE_KERNEL_FINISHED : MSGTYPE_KERNEL_STARTUP_FAILED);
+}
+
+void corecom_serve(object_loader load_object, kernel_runner run_kernel)
+{
     char msgtype;
-    unsigned char *_buffer = buffer;
 
     while(1) {
         receive_sync();
         msgtype = receive_char();
         if(msgtype == MSGTYPE_REQUEST_IDENT) {
+            send_char(MSGTYPE_IDENT);
             send_int(0x41524f52); /* "AROR" - ARTIQ runtime on OpenRISC */
             send_int(1000000000000LL/identifier_frequency_read()); /* RTIO clock period in picoseconds */
-        } else if(msgtype == MSGTYPE_LOAD_KERNEL) {
+        } else if(msgtype == MSGTYPE_LOAD_OBJECT)
-            length = receive_int();
+            receive_and_load_object(load_object);
-            if(length > maxlength) {
+        else if(msgtype == MSGTYPE_RUN_KERNEL)
-                send_char(0x4c); /* Incorrect length */
+            receive_and_run_kernel(run_kernel);
-                return -1;
+        else
-            }
+            send_char(MSGTYPE_MESSAGE_UNRECOGNIZED);
-            crc = receive_int();
-            for(i=0;i<length;i++)
-                _buffer[i] = receive_char();
-            if(crc32(buffer, length) != crc) {
-                send_char(0x43); /* CRC failed */
-                return -1;
-            }
-            send_char(0x4f); /* kernel reception OK */
-            return length;
-        } else
-            return -1;
     }
 }
 
-int rpc(int rpc_num, int n_args, ...)
+int corecom_rpc(int rpc_num, int n_args, ...)
 {
-    send_sync();
     send_char(MSGTYPE_RPC_REQUEST);
     send_sint(rpc_num);
     send_char(n_args);
@@ -121,8 +164,19 @@ int rpc(int rpc_num, int n_args, ...)
     return receive_int();
 }
 
-void kernel_finished(void)
+void corecom_log(const char *fmt, ...)
 {
-    send_sync();
+    va_list args;
-    send_char(MSGTYPE_KERNEL_FINISHED);
+    int len;
+    char outbuf[256];
+    int i;
+
+    va_start(args, fmt);
+    len = vscnprintf(outbuf, sizeof(outbuf), fmt, args);
+    va_end(args);
+
+    send_char(MSGTYPE_LOG);
+    send_sint(len);
+    for(i=0;i<len;i++)
+        send_char(outbuf[i]);
 }

soc/runtime/elf_loader.c

@@ -1,6 +1,6 @@
-#include <stdio.h>
 #include <string.h>
 
+#include "corecom.h"
 #include "elf_loader.h"
 
 #define EI_NIDENT 16
@@ -85,12 +85,12 @@ struct elf32_sym {
 
 #define SANITIZE_OFFSET_SIZE(offset, size) \
     if(offset > 0x10000000) { \
-        printf("Incorrect offset in ELF data"); \
+        corecom_log("Incorrect offset in ELF data"); \
-        return NULL; \
+        return 0; \
     } \
     if((offset + size) > elf_length) { \
-        printf("Attempted to access past the end of ELF data"); \
+        corecom_log("Attempted to access past the end of ELF data"); \
-        return NULL; \
+        return 0; \
     }
 
 #define GET_POINTER_SAFE(target, target_type, offset) \
@@ -121,11 +121,11 @@ static int fixup(void *dest, int dest_length, struct elf32_rela *rela, void *tar
         val = _target - (_dest + offset);
         _dest[offset] = (_dest[offset] & 0xfc000000) | (val & 0x03ffffff);
     } else
-        printf("Unsupported relocation type: %d\n", type);
+        corecom_log("Unsupported relocation type: %d", type);
     return 1;
 }
 
-void *load_elf(symbol_resolver resolver, const char *entry_name, void *elf_data, int elf_length, void *dest, int dest_length)
+int load_elf(symbol_resolver resolver, symbol_callback callback, void *elf_data, int elf_length, void *dest, int dest_length)
 {
     struct elf32_ehdr *ehdr;
     struct elf32_shdr *strtable;
@@ -137,22 +137,20 @@ void *load_elf(symbol_resolver resolver, const char *entry_name, void *elf_data,
     unsigned int symtaboff, symtabsize;
     unsigned int strtaboff, strtabsize;
 
-    void *entry_point;
-
 
     /* validate ELF */
     GET_POINTER_SAFE(ehdr, struct elf32_ehdr, 0);
     if(memcmp(ehdr->ident, elf_magic_header, sizeof(elf_magic_header)) != 0) {
-        printf("Incorrect ELF header\n");
+        corecom_log("Incorrect ELF header");
-        return NULL;
+        return 0;
     }
     if(ehdr->type != ET_REL) {
-        printf("ELF is not relocatable\n");
+        corecom_log("ELF is not relocatable");
-        return NULL;
+        return 0;
     }
     if(ehdr->machine != EM_OR1K) {
-        printf("ELF is for a different machine\n");
+        corecom_log("ELF is for a different machine");
-        return NULL;
+        return 0;
     }
 
     /* extract section info */
@@ -192,8 +190,8 @@ void *load_elf(symbol_resolver resolver, const char *entry_name, void *elf_data,
 
     /* load .text section */
     if(textsize > dest_length) {
-        printf(".text section is too large\n");
+        corecom_log(".text section is too large");
-        return NULL;
+        return 0;
     }
     memcpy(dest, (char *)elf_data + textoff, textsize);
 
@@ -211,35 +209,32 @@ void *load_elf(symbol_resolver resolver, const char *entry_name, void *elf_data,
             name = (char *)elf_data + strtaboff + sym->name;
             target = resolver(name);
             if(target == NULL) {
-                printf("Undefined symbol: %s\n", name);
+                corecom_log("Undefined symbol: %s", name);
-                return NULL;
+                return 0;
             }
             if(!fixup(dest, dest_length, rela, target))
-                return NULL;
+                return 0;
         } else {
-            printf("Unsupported relocation\n");
+            corecom_log("Unsupported relocation");
-            return NULL;
+            return 0;
         }
     }
 
-    /* find entry point */
+    /* list provided functions via callback */
-    entry_point = NULL;
     for(i=0;i<symtabsize;i+=sizeof(struct elf32_sym)) {
         struct elf32_sym *sym;
 
         GET_POINTER_SAFE(sym, struct elf32_sym, symtaboff + i);
         if((ELF32_ST_TYPE(sym->info) == STT_FUNC) && (sym->name != 0)) {
             char *name;
+            void *target;
 
             name = (char *)elf_data + strtaboff + sym->name;
-            if(strcmp(name, entry_name) == 0) {
+            target = (char *)dest + sym->value;
-                entry_point = (char *)dest + sym->value;
+            if(!callback(name, target))
-                break;
+                return 0;
         }
     }
-    }
-    if(entry_point == NULL)
-        printf("Failed to find entry point\n");
 
-    return entry_point;
+    return 1;
 }

soc/runtime/elf_loader.h

@@ -6,9 +6,10 @@ struct symbol {
     void *target;
 };
 
-void *find_symbol(const struct symbol *symbols, const char *name);
+typedef void * (*symbol_resolver)(const char *);
+typedef int (*symbol_callback)(const char *, void *);
 
-typedef void * (*symbol_resolver)(const char *name);
+void *find_symbol(const struct symbol *symbols, const char *name);
-void *load_elf(symbol_resolver resolver, const char *entry_name, void *elf_data, int elf_length, void *dest, int dest_length);
+int load_elf(symbol_resolver resolver, symbol_callback callback, void *elf_data, int elf_length, void *dest, int dest_length);
 
 #endif /* __ELF_LOADER_H */

soc/runtime/main.c

@@ -1,42 +1,90 @@
 #include <stdio.h>
+#include <string.h>
 #include <irq.h>
 #include <uart.h>
 #include <system.h>
 
 #include "corecom.h"
 #include "elf_loader.h"
-#include "symbols.h"
+#include "services.h"
 #include "rtio.h"
 #include "dds.h"
 
+static unsigned char kcode[256*1024];
+
+static struct symbol symtab[128];
+static int _symtab_count;
+static char _symtab_strings[128*16];
+static char *_symtab_strptr;
+
+
+static void symtab_init(void)
+{
+    memset(symtab, 0, sizeof(symtab));
+    _symtab_count = 0;
+    _symtab_strptr = _symtab_strings;
+}
+
+static int symtab_add(const char *name, void *target)
+{
+    if(_symtab_count >= sizeof(symtab)/sizeof(symtab[0])) {
+        corecom_log("Too many provided symbols in object");
+        symtab_init();
+        return 0;
+    }
+    symtab[_symtab_count].name = _symtab_strptr;
+    symtab[_symtab_count].target = target;
+    _symtab_count++;
+
+    while(1) {
+        if(_symtab_strptr >= &_symtab_strings[sizeof(_symtab_strings)]) {
+            corecom_log("Provided symbol string table overflow");
+            symtab_init();
+            return 0;
+        }
+        *_symtab_strptr = *name;
+        _symtab_strptr++;
+        if(*name == 0)
+            break;
+        name++;
+    }
+
+    return 1;
+}
+
+static int load_object(void *buffer, int length)
+{
+    symtab_init();
+    return load_elf(
+        resolve_service_symbol, symtab_add,
+        buffer, length, kcode, sizeof(kcode));
+}
+
 typedef void (*kernel_function)(void);
 
+static int run_kernel(const char *kernel_name)
+{
+    kernel_function k;
+
+    k = find_symbol(symtab, kernel_name);
+    if(k == NULL) {
+        corecom_log("Failed to find kernel entry point '%s' in object", kernel_name);
+        return 0;
+    }
+    rtio_init();
+    flush_cpu_icache();
+    k();
+    return 1;
+}
+
 int main(void)
 {
-    unsigned char kbuf[256*1024];
-    unsigned char kcode[256*1024];
-    kernel_function k;
-    int length;
-
     irq_setmask(0);
     irq_setie(1);
     uart_init();
     
     puts("ARTIQ runtime built "__DATE__" "__TIME__"\n");
-
-    while(1) {
-        length = ident_and_download_kernel(kbuf, sizeof(kbuf));
-        if(length > 0) {
-            k = load_elf(resolve_symbol, "run", kbuf, length, kcode, sizeof(kcode));
-            if(k != NULL) {
-                rtio_init();
     dds_init();
-                flush_cpu_icache();
+    corecom_serve(load_object, run_kernel);
-                k();
-                kernel_finished();
-            }
-        }
-    }
-
     return 0;
 }

soc/runtime/services.c

@@ -5,10 +5,10 @@
 #include "gpio.h"
 #include "rtio.h"
 #include "dds.h"
-#include "symbols.h"
+#include "services.h"
 
 static const struct symbol syscalls[] = {
-    {"rpc", rpc},
+    {"rpc", corecom_rpc},
     {"gpio_set", gpio_set},
     {"rtio_oe", rtio_oe},
     {"rtio_set", rtio_set},
@@ -34,7 +34,7 @@ static const struct symbol compiler_rt[] = {
     {"modsi3", &__modsi3},
     {"ledf2", &__ledf2},
     {"gedf2", &__gedf2},
-    {"unorddf2", &__gedf2},
+    {"unorddf2", &__unorddf2},
     {"negsf2", &__negsf2},
     {"negdf2", &__negdf2},
     {"addsf3", &__addsf3},
@@ -68,7 +68,7 @@ static const struct symbol compiler_rt[] = {
     {NULL, NULL}
 };
 
-void *resolve_symbol(const char *name)
+void *resolve_service_symbol(const char *name)
 {
     if(strncmp(name, "__", 2) != 0)
         return NULL;

soc/runtime/services.h

@@ -0,0 +1,6 @@
+#ifndef __SERVICES_H
+#define __SERVICES_H
+
+void *resolve_service_symbol(const char *name);
+
+#endif /* __SERVICES_H */

soc/runtime/symbols.h

@@ -1,6 +0,0 @@
-#ifndef __SYMBOLS_H
-#define __SYMBOLS_H
-
-void *resolve_symbol(const char *name);
-
-#endif /* __SYMBOLS_H */