runtime: implement mailbox, use it for kernel startup, exceptions and termination

artiq/coredevice/runtime_exceptions.py

@@ -5,8 +5,8 @@ from artiq.language.core import RuntimeException
 
 # Must be kept in sync with soc/runtime/exceptions.h
 
-class OutOfMemory(RuntimeException):
+class InternalError(RuntimeException):
-    """Raised when the runtime fails to allocate memory.
+    """Raised when the runtime encounters an internal error condition.
 
     """
     eid = 1

soc/runtime/Makefile

@@ -1,19 +1,20 @@
 include $(MSCDIR)/software/common.mak
 
-OBJECTS_SERVICES=exception_jmp.o exceptions.o rtio.o dds.o
+OBJECTS := isr.o elf_loader.o services.o comm_serial.o test_mode.o main.o
-OBJECTS=isr.o elf_loader.o services.o comm_serial.o test_mode.o main.o
+OBJECTS_KSUPPORT := exception_jmp.o exceptions.o rtio.o dds.o
 
 # NOTE: this does not handle dependencies well. Run "make clean"
 # when switching between UP and AMP.
 UNIPROCESSOR := $(shell echo -e "\#include <generated/csr.h>\nCSR_KERNEL_CPU_BASE" | $(CC_normal) $(CFLAGS) -E - | tail -n 1 | grep -c CSR_KERNEL_CPU_BASE)
 
 ifeq ($(UNIPROCESSOR),0)
-OBJECTS += kernelcpu.o ksupport_data.o
+OBJECTS += mailbox.o kernelcpu.o ksupport_data.o
+OBJECTS_KSUPPORT += mailbox.o ksupport.o
 CFLAGS += -DARTIQ_AMP
 SERVICE_TABLE_INPUT = ksupport.elf
 else
 ifeq ($(UNIPROCESSOR),1)
-OBJECTS += $(OBJECTS_SERVICES)
+OBJECTS += $(OBJECTS_KSUPPORT)
 else
 $(error failed to determine UP/AMP build)
 endif
@@ -32,8 +33,6 @@ all: runtime.bin
 	$(MSCDIR)/mkmscimg.py -f -o $@ $<
 
 runtime.elf: $(OBJECTS) libs
-
-%.elf:
 	$(LD) $(LDFLAGS) \
 		-T linker.ld \
 		-N -o $@ \
@@ -44,7 +43,7 @@ runtime.elf: $(OBJECTS) libs
 		-lbase -lcompiler-rt
 	@chmod -x $@
 
-ksupport.elf: $(OBJECTS_SERVICES) ksupport.o
+ksupport.elf: $(OBJECTS_KSUPPORT)
 	$(LD) $(LDFLAGS) \
 		-T ksupport.ld \
 		-N -o $@ \
@@ -74,8 +73,8 @@ libs:
 	$(MAKE) -C $(MSCDIR)/software/libbase
 
 clean:
-	$(RM) $(OBJECTS) $(OBJECTS:.o=.d) $(OBJECTS_SERVICES) $(OBJECTS_SERVICES:.o=.d)
+	$(RM) $(OBJECTS) $(OBJECTS:.o=.d) $(OBJECTS_KSUPPORT) $(OBJECTS_KSUPPORT:.o=.d)
 	$(RM) runtime.elf runtime.bin runtime.fbi .*~ *~
-	$(RM) service_table.h ksupport.d ksupport.o ksupport.elf ksupport.bin
+	$(RM) service_table.h ksupport.elf ksupport.bin
 
 .PHONY: all main.o clean libs load

soc/runtime/comm.h

@@ -2,13 +2,15 @@
 #define __COMM_H
 
 enum {
+    KERNEL_RUN_INVALID_STATUS,
+
     KERNEL_RUN_FINISHED,
     KERNEL_RUN_EXCEPTION,
     KERNEL_RUN_STARTUP_FAILED
 };
 
 typedef int (*object_loader)(void *, int);
-typedef int (*kernel_runner)(const char *, int *);
+typedef int (*kernel_runner)(const char *, int *, long long int *);
 
 void comm_serve(object_loader load_object, kernel_runner run_kernel);
 int comm_rpc(int rpc_num, ...);

soc/runtime/comm_serial.c

@@ -132,6 +132,7 @@ static void receive_and_run_kernel(kernel_runner run_kernel)
     int i;
     char kernel_name[256];
     int r, eid;
+    long long int eparams[3];
 
     length = receive_int();
     if(length > (sizeof(kernel_name)-1)) {
@@ -142,7 +143,7 @@ static void receive_and_run_kernel(kernel_runner run_kernel)
         kernel_name[i] = receive_char();
     kernel_name[length] = 0;
 
-    r = run_kernel(kernel_name, &eid);
+    r = run_kernel(kernel_name, &eid, eparams);
     switch(r) {
         case KERNEL_RUN_FINISHED:
             send_char(MSGTYPE_KERNEL_FINISHED);
@@ -151,12 +152,7 @@ static void receive_and_run_kernel(kernel_runner run_kernel)
             send_char(MSGTYPE_KERNEL_EXCEPTION);
             send_int(eid);
             for(i=0;i<3;i++)
-#ifdef ARTIQ_AMP
+                send_llint(eparams[i]);
-#warning TODO
-                send_llint(0LL);
-#else
-                send_llint(exception_params[i]);
-#endif
             break;
         case KERNEL_RUN_STARTUP_FAILED:
             send_char(MSGTYPE_KERNEL_STARTUP_FAILED);

soc/runtime/exceptions.c

@@ -1,7 +1,11 @@
 #include <generated/csr.h>
 
 #include "exceptions.h"
-#ifndef ARTIQ_AMP
+
+#ifdef ARTIQ_AMP
+#include "mailbox.h"
+#include "messages.h"
+#else
 #include "comm.h"
 #endif
 
@@ -14,12 +18,12 @@ struct exception_context {
 static struct exception_context exception_contexts[MAX_EXCEPTION_CONTEXTS];
 static int ec_top;
 static int stored_id;
-long long int exception_params[3];
+static long long int stored_params[3];
 
 void *exception_push(void)
 {
     if(ec_top >= MAX_EXCEPTION_CONTEXTS)
-        exception_raise(EID_OUT_OF_MEMORY);
+        exception_raise(EID_INTERNAL_ERROR);
     return exception_contexts[ec_top++].jb;
 }
 
@@ -28,8 +32,13 @@ void exception_pop(int levels)
     ec_top -= levels;
 }
 
-int exception_getid(void)
+int exception_getid(long long int *eparams)
 {
+    int i;
+
+    if(eparams)
+        for(i=0;i<3;i++)
+            eparams[i] = stored_params[i];
     return stored_id;
 }
 
@@ -44,13 +53,20 @@ void exception_raise_params(int id,
 {
     if(ec_top > 0) {
         stored_id = id;
-        exception_params[0] = p0;
+        stored_params[0] = p0;
-        exception_params[1] = p1;
+        stored_params[1] = p1;
-        exception_params[2] = p2;
+        stored_params[2] = p2;
         exception_longjmp(exception_contexts[--ec_top].jb);
     } else {
 #ifdef ARTIQ_AMP
-#warning TODO
+    struct msg_exception msg;
+    int i;
+
+    msg.type = MESSAGE_TYPE_EXCEPTION;
+    msg.eid = EID_INTERNAL_ERROR;
+    for(i=0;i<3;i++)
+        msg.eparams[i] = 0;
+    mailbox_send_and_wait(&msg);
 #else
         comm_log("ERROR: uncaught exception, ID=%d\n", id);
 #endif

soc/runtime/exceptions.h

@@ -3,21 +3,19 @@
 
 enum {
     EID_NONE = 0,
-    EID_OUT_OF_MEMORY = 1,
+    EID_INTERNAL_ERROR = 1,
     EID_RPC_EXCEPTION = 2,
     EID_RTIO_UNDERFLOW = 3,
     EID_RTIO_SEQUENCE_ERROR = 4,
     EID_RTIO_OVERFLOW = 5,
 };
 
-extern long long int exception_params[3];
-
 int exception_setjmp(void *jb) __attribute__((returns_twice));
 void exception_longjmp(void *jb) __attribute__((noreturn));
 
 void *exception_push(void);
 void exception_pop(int levels);
-int exception_getid(void);
+int exception_getid(long long int *eparams);
 void exception_raise(int id) __attribute__((noreturn));
 void exception_raise_params(int id,
     long long int p0, long long int p1,

soc/runtime/kernelcpu.c

@@ -1,9 +1,9 @@
 #include <stdio.h>
 #include <string.h>
-#include <system.h>
 
 #include <generated/csr.h>
 
+#include "mailbox.h"
 #include "kernelcpu.h"
 
 extern char _binary_ksupport_bin_start;
@@ -13,8 +13,8 @@ void kernelcpu_start(void *addr)
 {
     memcpy((void *)KERNELCPU_EXEC_ADDRESS, &_binary_ksupport_bin_start,
         &_binary_ksupport_bin_end - &_binary_ksupport_bin_start);
-    KERNELCPU_MAILBOX = (unsigned int)addr;
+    mailbox_acknowledge();
-    flush_l2_cache();
+    mailbox_send(addr);
     kernel_cpu_reset_write(0);
 }
 

soc/runtime/kernelcpu.h

@@ -6,8 +6,6 @@
 #define KERNELCPU_EXEC_ADDRESS 0x40020000
 #define KERNELCPU_PAYLOAD_ADDRESS 0x40024000
 
-#define KERNELCPU_MAILBOX MMPTR(0xd0000000)
-
 void kernelcpu_start(void *addr);
 void kernelcpu_stop(void);
 

soc/runtime/ksupport.c

@@ -1,14 +1,21 @@
-#include "kernelcpu.h"
 #include "exceptions.h"
-#include "comm.h"
+#include "mailbox.h"
+#include "messages.h"
 #include "rtio.h"
 #include "dds.h"
 
 void exception_handler(unsigned long vect, unsigned long *sp);
 void exception_handler(unsigned long vect, unsigned long *sp)
 {
-    /* TODO: report hardware exception to comm CPU */
+    struct msg_exception msg;
-    for(;;);
+    int i;
+
+    msg.type = MESSAGE_TYPE_EXCEPTION;
+    msg.eid = EID_INTERNAL_ERROR;
+    for(i=0;i<3;i++)
+        msg.eparams[i] = 0;
+    mailbox_send_and_wait(&msg);
+    while(1);
 }
 
 typedef void (*kernel_function)(void);
@@ -19,17 +26,26 @@ int main(void)
     kernel_function k;
     void *jb;
 
-    k = (kernel_function)KERNELCPU_MAILBOX;
-
     jb = exception_push();
-    if(exception_setjmp(jb))
+    if(exception_setjmp(jb)) {
-        KERNELCPU_MAILBOX = KERNEL_RUN_EXCEPTION;
+        struct msg_exception msg;
-    else {
+
+        msg.type = MESSAGE_TYPE_EXCEPTION;
+        msg.eid = exception_getid(msg.eparams);
+        mailbox_send_and_wait(&msg);
+    } else {
+        struct msg_finished msg;
+
+        k = mailbox_receive();
+        if(!k)
+            exception_raise(EID_INTERNAL_ERROR);
         dds_init();
         rtio_init();
         k();
         exception_pop(1);
-        KERNELCPU_MAILBOX = KERNEL_RUN_FINISHED;
+
+        msg.type = MESSAGE_TYPE_FINISHED;
+        mailbox_send_and_wait(&msg);
     }
     while(1);
 }

soc/runtime/mailbox.c

@@ -0,0 +1,81 @@
+#include <stdlib.h>
+#include <system.h>
+#include <spr-defs.h>
+#include <hw/common.h>
+
+#include "mailbox.h"
+
+#define KERNELCPU_MAILBOX MMPTR(0xd0000000)
+
+static unsigned int last_transmission;
+
+static void _flush_cpu_dcache(void)
+{
+    unsigned long dccfgr;
+    unsigned long cache_set_size;
+    unsigned long cache_ways;
+    unsigned long cache_block_size;
+    unsigned long cache_size;
+    int i;
+
+    dccfgr = mfspr(SPR_DCCFGR);
+    cache_ways = 1 << (dccfgr & SPR_ICCFGR_NCW);
+    cache_set_size = 1 << ((dccfgr & SPR_DCCFGR_NCS) >> 3);
+    cache_block_size = (dccfgr & SPR_DCCFGR_CBS) ? 32 : 16;
+    cache_size = cache_set_size * cache_ways * cache_block_size;
+
+    for (i = 0; i < cache_size; i += cache_block_size)
+        mtspr(SPR_DCBIR, i);
+}
+
+/* TODO: do not use L2 cache in AMP systems */
+static void _flush_l2_cache(void)
+{
+    unsigned int i;
+    register unsigned int addr;
+    register unsigned int dummy;
+
+    for(i=0;i<2*8192/4;i++) {
+        addr = 0x40000000 + i*4;
+        __asm__ volatile("l.lwz %0, 0(%1)\n":"=r"(dummy):"r"(addr));
+    }
+}
+
+void mailbox_send(void *ptr)
+{
+    _flush_l2_cache();
+    last_transmission = (unsigned int)ptr;
+    KERNELCPU_MAILBOX = last_transmission;
+}
+
+int mailbox_acknowledged(void)
+{
+    return KERNELCPU_MAILBOX != last_transmission;
+}
+
+void mailbox_send_and_wait(void *ptr)
+{
+    mailbox_send(ptr);
+    while(!mailbox_acknowledged());
+}
+
+void *mailbox_receive(void)
+{
+    unsigned int r;
+
+    r = KERNELCPU_MAILBOX;
+    if(r == last_transmission)
+        return NULL;
+    else {
+        if(r) {
+            _flush_l2_cache();
+            _flush_cpu_dcache();
+        }
+        return (void *)r;
+    }
+}
+
+void mailbox_acknowledge(void)
+{
+    KERNELCPU_MAILBOX = 0;
+}

soc/runtime/mailbox.h

@@ -0,0 +1,11 @@
+#ifndef __MAILBOX_H
+#define __MAILBOX_H
+
+void mailbox_send(void *ptr);
+int mailbox_acknowledged(void);
+void mailbox_send_and_wait(void *ptr);
+
+void *mailbox_receive(void);
+void mailbox_acknowledge(void);
+
+#endif /* __MAILBOX_H */

soc/runtime/main.c

@@ -10,12 +10,17 @@
 #include "test_mode.h"
 #include "comm.h"
 #include "elf_loader.h"
-#include "kernelcpu.h"
 #include "exceptions.h"
 #include "services.h"
 #include "rtio.h"
 #include "dds.h"
 
+#ifdef ARTIQ_AMP
+#include "kernelcpu.h"
+#include "mailbox.h"
+#include "messages.h"
+#endif
+
 static struct symbol symtab[128];
 static int _symtab_count;
 static char _symtab_strings[128*16];
@@ -63,15 +68,44 @@ static int load_object(void *buffer, int length)
         buffer, length, (void *)KERNELCPU_PAYLOAD_ADDRESS, 4*1024*1024);
 }
 
+
+#ifdef ARTIQ_AMP
+static int process_msg(struct msg_unknown *umsg, int *eid, long long int *eparams)
+{
+    int i;
+
+    switch(umsg->type) {
+        case MESSAGE_TYPE_FINISHED:
+            return KERNEL_RUN_FINISHED;
+        case MESSAGE_TYPE_EXCEPTION: {
+            struct msg_exception *msg = (struct msg_exception *)umsg;
+
+            *eid = msg->eid;
+            for(i=0;i<3;i++)
+                eparams[i] = msg->eparams[i];
+            return KERNEL_RUN_EXCEPTION;
+        }
+        default:
+            *eid = EID_INTERNAL_ERROR;
+            for(i=0;i<3;i++)
+                eparams[i] = 0;
+            return KERNEL_RUN_EXCEPTION;
+    }
+}
+#endif
+
 typedef void (*kernel_function)(void);
 
-static int run_kernel(const char *kernel_name, int *eid)
+static int run_kernel(const char *kernel_name, int *eid, long long int *eparams)
 {
     kernel_function k;
-#ifndef ARTIQ_AMP
+#ifdef ARTIQ_AMP
+    int r;
+#else
     void *jb;
 #endif
 
+
     k = find_symbol(symtab, kernel_name);
     if(k == NULL) {
         comm_log("Failed to find kernel entry point '%s' in object", kernel_name);
@@ -80,20 +114,22 @@ static int run_kernel(const char *kernel_name, int *eid)
 
 #ifdef ARTIQ_AMP
     kernelcpu_start(k);
-    *eid = 0;
     while(1) {
-        unsigned int r;
+        struct msg_unknown *umsg;
 
-        r = KERNELCPU_MAILBOX;
+        umsg = mailbox_receive();
-        if(r < 0x40000000) {
+        r = KERNEL_RUN_INVALID_STATUS;
+        if(umsg)
+            r = process_msg(umsg, eid, eparams);
+        if(r != KERNEL_RUN_INVALID_STATUS)
+            break;
+    }
     kernelcpu_stop();
     return r;
-        }
-    }
 #else
     jb = exception_push();
     if(exception_setjmp(jb)) {
-        *eid = exception_getid();
+        *eid = exception_getid(eparams);
         return KERNEL_RUN_EXCEPTION;
     } else {
         dds_init();

soc/runtime/messages.h

@@ -0,0 +1,23 @@
+#ifndef __MESSAGES_H
+#define __MESSAGES_H
+
+enum {
+    MESSAGE_TYPE_FINISHED,
+    MESSAGE_TYPE_EXCEPTION
+};
+
+struct msg_unknown {
+    int type;
+};
+
+struct msg_finished {
+    int type;
+};
+
+struct msg_exception {
+    int type;
+    int eid;
+    long long int eparams[3];
+};
+
+#endif /* __MESSAGES_H */