Merge branch 'master' of github.com:m-labs/artiq

artiq/compiler/analyses/constness.py

@@ -24,7 +24,10 @@ class Constness(algorithm.Visitor):
         self.in_assign = old_in_assign
 
     def visit_AttributeT(self, node):
-        self.generic_visit(node)
+        old_in_assign, self.in_assign = self.in_assign, False
+        self.visit(node.value)
+        self.in_assign = old_in_assign
+
         if self.in_assign:
             typ = node.value.type.find()
             if types.is_instance(typ) and node.attr in typ.constant_attributes:

artiq/compiler/transforms/llvm_ir_generator.py

@@ -340,11 +340,11 @@ class LLVMIRGenerator:
             llty = ll.FunctionType(llvoid, [])
         elif name == "memcmp":
             llty = ll.FunctionType(lli32, [llptr, llptr, lli32])
-        elif name == "send_rpc":
+        elif name == "rpc_send":
             llty = ll.FunctionType(llvoid, [lli32, llsliceptr, llptrptr])
-        elif name == "send_async_rpc":
+        elif name == "rpc_send_async":
             llty = ll.FunctionType(llvoid, [lli32, llsliceptr, llptrptr])
-        elif name == "recv_rpc":
+        elif name == "rpc_recv":
             llty = ll.FunctionType(lli32, [llptr])
         elif name == "now":
             llty = lli64
@@ -359,7 +359,7 @@ class LLVMIRGenerator:
             llglobal = ll.Function(self.llmodule, llty, name)
             if name in ("__artiq_raise", "__artiq_reraise", "llvm.trap"):
                 llglobal.attributes.add("noreturn")
-            if name in ("rtio_log", "send_rpc", "send_async_rpc",
+            if name in ("rtio_log", "rpc_send", "rpc_send_async",
                         "watchdog_set", "watchdog_clear",
                         self.target.print_function):
                 llglobal.attributes.add("nounwind")
@@ -1242,10 +1242,10 @@ class LLVMIRGenerator:
             self.llbuilder.store(llargslot, llargptr)
 
         if fun_type.async:
-            self.llbuilder.call(self.llbuiltin("send_async_rpc"),
+            self.llbuilder.call(self.llbuiltin("rpc_send_async"),
                                 [llservice, lltagptr, llargs])
         else:
-            self.llbuilder.call(self.llbuiltin("send_rpc"),
+            self.llbuilder.call(self.llbuiltin("rpc_send"),
                                 [llservice, lltagptr, llargs])
 
         # Don't waste stack space on saved arguments.
@@ -1257,7 +1257,7 @@ class LLVMIRGenerator:
         # T result = {
         #   void *ret_ptr = alloca(sizeof(T));
         #   void *ptr = ret_ptr;
-        #   loop: int size = recv_rpc(ptr);
+        #   loop: int size = rpc_recv(ptr);
         #   // Non-zero: Provide `size` bytes of extra storage for variable-length data.
         #   if(size) { ptr = alloca(size); goto loop; }
         #   else *(T*)ret_ptr
@@ -1278,12 +1278,12 @@ class LLVMIRGenerator:
         llphi = self.llbuilder.phi(llslotgen.type, name="rpc.ptr")
         llphi.add_incoming(llslotgen, llprehead)
         if llunwindblock:
-            llsize = self.llbuilder.invoke(self.llbuiltin("recv_rpc"), [llphi],
+            llsize = self.llbuilder.invoke(self.llbuiltin("rpc_recv"), [llphi],
                                            llheadu, llunwindblock,
                                            name="rpc.size.next")
             self.llbuilder.position_at_end(llheadu)
         else:
-            llsize = self.llbuilder.call(self.llbuiltin("recv_rpc"), [llphi],
+            llsize = self.llbuilder.call(self.llbuiltin("rpc_recv"), [llphi],
                                          name="rpc.size.next")
         lldone = self.llbuilder.icmp_unsigned('==', llsize, ll.Constant(llsize.type, 0),
                                               name="rpc.done")

artiq/coredevice/dma.py

@@ -0,0 +1,55 @@
+from artiq.language.core import syscall, kernel
+from artiq.language.types import TStr, TNone
+
+from numpy import int64
+
+
+@syscall
+def dma_record_start() -> TNone:
+    raise NotImplementedError("syscall not simulated")
+
+
+@syscall
+def dma_record_stop(name: TStr) -> TNone:
+    raise NotImplementedError("syscall not simulated")
+
+
+class DMARecordContextManager:
+    def __init__(self):
+        self.name = ""
+        self.saved_now_mu = int64(0)
+
+    @kernel
+    def __enter__(self):
+        """Starts recording a DMA trace. All RTIO operations are redirected to
+        a newly created DMA buffer after this call, and ``now`` is reset to zero."""
+        dma_record_start() # this may raise, so do it before altering now
+        self.saved_now_mu = now_mu()
+        at_mu(0)
+
+    @kernel
+    def __exit__(self, type, value, traceback):
+        """Stops recording a DMA trace. All recorded RTIO operations are stored
+        in a newly created trace called ``self.name``, and ``now`` is restored
+        to the value it had before ``__enter__`` was called."""
+        dma_record_stop(self.name) # see above
+        at_mu(self.saved_now_mu)
+
+
+class CoreDMA:
+    """Core device Direct Memory Access (DMA) driver.
+
+    Gives access to the DMA functionality of the core device.
+    """
+
+    kernel_invariants = {"core", "recorder"}
+
+    def __init__(self, dmgr, core_device="core"):
+        self.core     = dmgr.get(core_device)
+        self.recorder = DMARecordContextManager()
+
+    @kernel
+    def record(self, name):
+        """Returns a context manager that will record a DMA trace called ``name``."""
+        self.recorder.name = name
+        return self.recorder

artiq/coredevice/exceptions.py

@@ -120,6 +120,10 @@ class RTIOOverflow(Exception):
     """
     artiq_builtin = True
 
+class DMAError(Exception):
+    """Raised when performing an invalid DMA operation."""
+    artiq_builtin = True
+
 class DDSError(Exception):
     """Raised when attempting to start a DDS batch while already in a batch,
     when too many commands are batched, and when DDS channel settings are

artiq/examples/master/device_db.pyon

@@ -20,6 +20,11 @@
         "module": "artiq.coredevice.cache",
         "class": "CoreCache"
     },
+    "core_dma": {
+        "type": "local",
+        "module": "artiq.coredevice.dma",
+        "class": "CoreDMA"
+    },
     "core_dds": {
         "type": "local",
         "module": "artiq.coredevice.dds",

artiq/firmware/Cargo.lock

@@ -10,6 +10,13 @@ version = "0.0.0"
 name = "alloc_none"
 version = "0.0.0"
 
+[[package]]
+name = "amp"
+version = "0.0.0"
+dependencies = [
+ "board 0.0.0",
+]
+
 [[package]]
 name = "board"
 version = "0.0.0"
@@ -62,6 +69,10 @@ dependencies = [
  "std_artiq 0.0.0",
 ]
 
+[[package]]
+name = "dyld"
+version = "0.0.0"
+
 [[package]]
 name = "fringe"
 version = "1.1.0"
@@ -89,10 +100,13 @@ name = "ksupport"
 version = "0.0.0"
 dependencies = [
  "alloc_none 0.0.0",
+ "amp 0.0.0",
  "board 0.0.0",
  "build_artiq 0.0.0",
  "byteorder 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)",
  "cslice 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)",
+ "dyld 0.0.0",
+ "proto 0.0.0",
  "std_artiq 0.0.0",
 ]
 
@@ -125,11 +139,23 @@ name = "managed"
 version = "0.2.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 
+[[package]]
+name = "proto"
+version = "0.0.0"
+dependencies = [
+ "byteorder 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)",
+ "cslice 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)",
+ "dyld 0.0.0",
+ "log 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)",
+ "std_artiq 0.0.0",
+]
+
 [[package]]
 name = "runtime"
 version = "0.0.0"
 dependencies = [
  "alloc_artiq 0.0.0",
+ "amp 0.0.0",
  "board 0.0.0",
  "build_artiq 0.0.0",
  "byteorder 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)",
@@ -139,6 +165,7 @@ dependencies = [
  "fringe 1.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
  "log 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)",
  "logger_artiq 0.0.0",
+ "proto 0.0.0",
  "smoltcp 0.2.1 (git+https://github.com/m-labs/smoltcp?rev=e8ece3e)",
  "std_artiq 0.0.0",
 ]

artiq/firmware/ksupport/Cargo.toml

@@ -13,8 +13,11 @@ crate-type = ["staticlib"]
 build_artiq = { path = "../libbuild_artiq" }
 
 [dependencies]
-alloc_none = { path = "../liballoc_none" }
-std_artiq = { path = "../libstd_artiq" }
-board = { path = "../libboard" }
 byteorder = { version = "1.0", default-features = false }
 cslice = { version = "0.3" }
+alloc_none = { path = "../liballoc_none" }
+std_artiq = { path = "../libstd_artiq" }
+dyld = { path = "../libdyld" }
+board = { path = "../libboard" }
+proto = { path = "../libproto" }
+amp = { path = "../libamp" }

artiq/firmware/ksupport/Makefile

@@ -11,8 +11,7 @@ LDFLAGS += --eh-frame-hdr \
 	-L../libcompiler-rt \
 	-L../libbase \
 	-L../libm \
-	-L../libunwind \
+	-L../libunwind
-	-L../libdyld
 
 RUSTFLAGS += -Cpanic=unwind
 
@@ -24,7 +23,7 @@ $(RUSTOUT)/libksupport.a:
 
 ksupport.elf: $(RUSTOUT)/libksupport.a glue.o
 	$(LD) $(LDFLAGS) -T $(KSUPPORT_DIRECTORY)/ksupport.ld -o $@ $^ \
-		-lbase -lm -lcompiler-rt -ldyld -lunwind
+		-lunwind -lcompiler-rt -lbase -lm
 	@chmod -x $@
 
 %.o: $(KSUPPORT_DIRECTORY)/%.c

artiq/firmware/ksupport/api.rs

@@ -12,12 +12,11 @@ macro_rules! api {
     }
 }
 
-pub fn resolve(required: &str) -> usize {
+pub fn resolve(required: &[u8]) -> Option<u32> {
     unsafe {
         API.iter()
-           .find(|&&(exported, _)| exported == required)
+           .find(|&&(exported, _)| exported.as_bytes() == required)
-           .map(|&(_, ptr)| ptr as usize)
+           .map(|&(_, ptr)| ptr as u32)
-           .unwrap_or(0)
     }
 }
 
@@ -87,9 +86,9 @@ static mut API: &'static [(&'static str, *const ())] = &[
     api!(watchdog_set = ::watchdog_set),
     api!(watchdog_clear = ::watchdog_clear),
 
-    api!(send_rpc = ::send_rpc),
+    api!(rpc_send = ::rpc_send),
-    api!(send_async_rpc = ::send_async_rpc),
+    api!(rpc_send_async = ::rpc_send_async),
-    api!(recv_rpc = ::recv_rpc),
+    api!(rpc_recv = ::rpc_recv),
 
     api!(cache_get = ::cache_get),
     api!(cache_put = ::cache_put),
@@ -106,6 +105,9 @@ static mut API: &'static [(&'static str, *const ())] = &[
     api!(rtio_input_timestamp = ::rtio::input_timestamp),
     api!(rtio_input_data = ::rtio::input_data),
 
+    api!(dma_record_start = ::dma_record_start),
+    api!(dma_record_stop = ::dma_record_stop),
+
     api!(drtio_get_channel_state = ::rtio::drtio_dbg::get_channel_state),
     api!(drtio_reset_channel_state = ::rtio::drtio_dbg::reset_channel_state),
     api!(drtio_get_fifo_space = ::rtio::drtio_dbg::get_fifo_space),

artiq/firmware/ksupport/dyld.rs

@@ -1,56 +0,0 @@
-use core::{ptr, slice, str};
-use libc::{c_void, c_char, c_int, size_t};
-
-#[allow(non_camel_case_types)]
-#[repr(C)]
-#[derive(Default)]
-struct dyld_info {
-    __opaque: [usize; 7]
-}
-
-extern {
-    fn dyld_load(shlib: *const c_void, base: usize,
-                 resolve: extern fn(*mut c_void, *const c_char) -> usize,
-                 resolve_data: *mut c_void,
-                 info: *mut dyld_info, error_out: *mut *const c_char) -> c_int;
-    fn dyld_lookup(symbol: *const c_char, info: *const dyld_info) -> *const c_void;
-
-    fn strlen(ptr: *const c_char) -> size_t;
-}
-
-pub struct Library {
-    lower: dyld_info
-}
-
-impl Library {
-    pub unsafe fn load<F>(shlib: &[u8], base: usize, mut resolve: F)
-                -> Result<Library, &'static str>
-            where F: Fn(&str) -> usize {
-        extern fn wrapper<F>(data: *mut c_void, name: *const c_char) -> usize
-                where F: Fn(&str) -> usize {
-            unsafe {
-                let f = data as *mut F;
-                let name = slice::from_raw_parts(name as *const u8, strlen(name));
-                (*f)(str::from_utf8_unchecked(name))
-            }
-        }
-
-        let mut library = Library { lower: dyld_info::default() };
-        let mut error: *const c_char = ptr::null();
-        if dyld_load(shlib.as_ptr() as *const _, base,
-                     wrapper::<F>, &mut resolve as *mut _ as *mut _,
-                     &mut library.lower, &mut error) == 0 {
-            let error = slice::from_raw_parts(error as *const u8, strlen(error));
-            Err(str::from_utf8_unchecked(error))
-        } else {
-            Ok(library)
-        }
-    }
-
-    pub unsafe fn lookup(&self, symbol: &str) -> usize {
-        assert!(symbol.len() < 32);
-        let mut buf = [0u8; 32];
-        buf[0..symbol.as_bytes().len()].copy_from_slice(symbol.as_bytes());
-        dyld_lookup(&buf as *const _ as *const c_char, &self.lower) as usize
-    }
-}

artiq/firmware/ksupport/eh.rs

@@ -433,3 +433,28 @@ pub unsafe extern fn reraise() -> ! {
         uw::_Unwind_Resume(&mut INFLIGHT.uw_exception)
     }
 }
+
+// Stub implementations for the functions the panic_unwind crate expects to be provided.
+// These all do nothing in libunwind, but aren't built for OR1K.
+pub mod stubs {
+    #![allow(bad_style, unused_variables)]
+
+    use super::{uw, c_int};
+
+    #[export_name="_Unwind_GetIPInfo"]
+    pub unsafe extern fn _Unwind_GetIPInfo(ctx: *mut uw::_Unwind_Context,
+                                           ip_before_insn: *mut c_int) -> uw::_Unwind_Word {
+        *ip_before_insn = 0;
+        uw::_Unwind_GetIP(ctx)
+    }
+
+    #[export_name="_Unwind_GetTextRelBase"]
+    pub unsafe extern fn _Unwind_GetTextRelBase(ctx: *mut uw::_Unwind_Context) -> uw::_Unwind_Ptr {
+        unimplemented!()
+    }
+
+    #[export_name="_Unwind_GetDataRelBase"]
+    pub unsafe extern fn _Unwind_GetDataRelBase(ctx: *mut uw::_Unwind_Context) -> uw::_Unwind_Ptr {
+        unimplemented!()
+    }
+}

artiq/firmware/ksupport/lib.rs

@@ -1,35 +1,70 @@
 #![feature(lang_items, asm, libc, panic_unwind, unwind_attributes)]
 #![no_std]
 
-extern crate alloc_none;
-#[macro_use]
-extern crate std_artiq as std;
 extern crate unwind;
 extern crate libc;
 extern crate byteorder;
-extern crate board;
 extern crate cslice;
 
-#[path = "../runtime/mailbox.rs"]
+extern crate alloc_none;
-mod mailbox;
+extern crate std_artiq as std;
 
-#[path = "../runtime/proto.rs"]
+extern crate board;
-mod proto;
+extern crate dyld;
-#[path = "../runtime/kernel_proto.rs"]
+extern crate proto;
-mod kernel_proto;
+extern crate amp;
-#[path = "../runtime/rpc_proto.rs"]
-mod rpc_proto;
 
-mod dyld;
+use core::{mem, ptr, slice, str};
-mod api;
-
-use core::{mem, ptr, str};
 use std::io::Cursor;
 use cslice::{CSlice, AsCSlice};
-use kernel_proto::*;
 use dyld::Library;
+use proto::{kernel_proto, rpc_proto};
+use proto::kernel_proto::*;
+use amp::{mailbox, rpc_queue};
 
-macro_rules! artiq_raise {
+fn send(request: &Message) {
+    unsafe { mailbox::send(request as *const _ as usize) }
+    while !mailbox::acknowledged() {}
+}
+
+fn recv<R, F: FnOnce(&Message) -> R>(f: F) -> R {
+    while mailbox::receive() == 0 {}
+    let result = f(unsafe { mem::transmute::<usize, &Message>(mailbox::receive()) });
+    mailbox::acknowledge();
+    result
+}
+
+macro_rules! recv {
+    ($p:pat => $e:expr) => {
+        recv(move |request| {
+            if let $p = request {
+                $e
+            } else {
+                send(&Log(format_args!("unexpected reply: {:?}\n", request)));
+                loop {}
+            }
+        })
+    }
+}
+
+#[no_mangle]
+#[lang = "panic_fmt"]
+pub extern fn panic_fmt(args: core::fmt::Arguments, file: &'static str, line: u32) -> ! {
+    send(&Log(format_args!("panic at {}:{}: {}\n", file, line, args)));
+    send(&RunAborted);
+    loop {}
+}
+
+macro_rules! print {
+    ($($arg:tt)*) => ($crate::send(&$crate::kernel_proto::Log(format_args!($($arg)*))));
+}
+
+macro_rules! println {
+    ($fmt:expr) => (print!(concat!($fmt, "\n")));
+    ($fmt:expr, $($arg:tt)*) => (print!(concat!($fmt, "\n"), $($arg)*));
+}
+
+macro_rules! raise {
     ($name:expr, $message:expr, $param0:expr, $param1:expr, $param2:expr) => ({
         use cslice::AsCSlice;
         let exn = $crate::eh::Exception {
@@ -46,58 +81,16 @@ macro_rules! artiq_raise {
         unsafe { $crate::eh::raise(&exn) }
     });
     ($name:expr, $message:expr) => ({
-        artiq_raise!($name, $message, 0, 0, 0)
+        raise!($name, $message, 0, 0, 0)
     });
 }
 
-fn send(request: &Message) {
+pub mod eh;
-    unsafe { mailbox::send(request as *const _ as usize) }
+mod api;
-    while !mailbox::acknowledged() {}
-}
-
-fn recv<R, F: FnOnce(&Message) -> R>(f: F) -> R {
-    while mailbox::receive() == 0 {}
-    let result = f(unsafe { mem::transmute::<usize, &Message>(mailbox::receive()) });
-    mailbox::acknowledge();
-    result
-}
-
-macro_rules! recv {
-    ($p:pat => $e:expr) => {
-        recv(|request| {
-            if let $p = request {
-                $e
-            } else {
-                send(&Log(format_args!("unexpected reply: {:?}", request)));
-                loop {}
-            }
-        })
-    }
-}
-
-macro_rules! print {
-    ($($arg:tt)*) => ($crate::send(&$crate::kernel_proto::Log(format_args!($($arg)*))));
-}
-
-macro_rules! println {
-    ($fmt:expr) => (print!(concat!($fmt, "\n")));
-    ($fmt:expr, $($arg:tt)*) => (print!(concat!($fmt, "\n"), $($arg)*));
-}
-
-#[path = "../runtime/rpc_queue.rs"]
-mod rpc_queue;
 mod rtio;
-mod eh;
-
-#[no_mangle]
-#[lang = "panic_fmt"]
-pub extern fn panic_fmt(args: core::fmt::Arguments, file: &'static str, line: u32) -> ! {
-    println!("panic at {}:{}: {}", file, line, args);
-    send(&RunAborted);
-    loop {}
-}
 
 static mut NOW: u64 = 0;
+static mut LIBRARY: Option<Library<'static>> = None;
 
 #[no_mangle]
 pub extern fn send_to_core_log(text: CSlice<u8>) {
@@ -115,13 +108,7 @@ pub extern fn send_to_rtio_log(timestamp: i64, text: CSlice<u8>) {
     rtio::log(timestamp, text.as_ref())
 }
 
-extern fn abort() -> ! {
+extern fn rpc_send(service: u32, tag: CSlice<u8>, data: *const *const ()) {
-    println!("kernel called abort()");
-    send(&RunAborted);
-    loop {}
-}
-
-extern fn send_rpc(service: u32, tag: CSlice<u8>, data: *const *const ()) {
     while !rpc_queue::empty() {}
     send(&RpcSend {
         async:   false,
@@ -131,7 +118,7 @@ extern fn send_rpc(service: u32, tag: CSlice<u8>, data: *const *const ()) {
     })
 }
 
-extern fn send_async_rpc(service: u32, tag: CSlice<u8>, data: *const *const ()) {
+extern fn rpc_send_async(service: u32, tag: CSlice<u8>, data: *const *const ()) {
     while rpc_queue::full() {}
     rpc_queue::enqueue(|mut slice| {
         let length = {
@@ -139,7 +126,7 @@ extern fn send_async_rpc(service: u32, tag: CSlice<u8>, data: *const *const ())
             rpc_proto::send_args(&mut writer, service, tag.as_ref(), data)?;
             writer.position()
         };
-        proto::write_u32(&mut slice, length as u32)
+        proto::io::write_u32(&mut slice, length as u32)
     }).unwrap_or_else(|err| {
         assert!(err.kind() == std::io::ErrorKind::WriteZero);
 
@@ -153,7 +140,7 @@ extern fn send_async_rpc(service: u32, tag: CSlice<u8>, data: *const *const ())
     })
 }
 
-extern fn recv_rpc(slot: *mut ()) -> usize {
+extern fn rpc_recv(slot: *mut ()) -> usize {
     send(&RpcRecvRequest(slot));
     recv!(&RpcRecvReply(ref result) => {
         match result {
@@ -202,7 +189,7 @@ fn terminate(exception: &eh::Exception, mut backtrace: &mut [usize]) -> ! {
 
 extern fn watchdog_set(ms: i64) -> i32 {
     if ms < 0 {
-        artiq_raise!("ValueError", "cannot set a watchdog with a negative timeout")
+        raise!("ValueError", "cannot set a watchdog with a negative timeout")
     }
 
     send(&WatchdogSetRequest { ms: ms as u64 });
@@ -227,27 +214,82 @@ extern fn cache_put(key: CSlice<u8>, list: CSlice<i32>) {
     });
     recv!(&CachePutReply { succeeded } => {
         if !succeeded {
-            artiq_raise!("CacheError", "cannot put into a busy cache row")
+            raise!("CacheError", "cannot put into a busy cache row")
         }
     })
 }
 
 extern fn i2c_start(busno: i32) {
-    send(&I2CStartRequest { busno: busno as u8 });
+    send(&I2cStartRequest { busno: busno as u8 });
 }
 
 extern fn i2c_stop(busno: i32) {
-    send(&I2CStopRequest { busno: busno as u8 });
+    send(&I2cStopRequest { busno: busno as u8 });
 }
 
 extern fn i2c_write(busno: i32, data: i32) -> bool {
-    send(&I2CWriteRequest { busno: busno as u8, data: data as u8 });
+    send(&I2cWriteRequest { busno: busno as u8, data: data as u8 });
-    recv!(&I2CWriteReply { ack } => ack)
+    recv!(&I2cWriteReply { ack } => ack)
 }
 
 extern fn i2c_read(busno: i32, ack: bool) -> i32 {
-    send(&I2CReadRequest { busno: busno as u8, ack: ack });
+    send(&I2cReadRequest { busno: busno as u8, ack: ack });
-    recv!(&I2CReadReply { data } => data) as i32
+    recv!(&I2cReadReply { data } => data) as i32
+}
+
+static mut DMA_RECORDING: bool = false;
+
+extern fn dma_record_start() {
+    unsafe {
+        if DMA_RECORDING {
+            raise!("DMAError", "DMA is already recording")
+        }
+
+        let library = LIBRARY.as_ref().unwrap();
+        library.rebind(b"rtio_output",
+                       dma_record_output as *const () as u32).unwrap();
+        library.rebind(b"rtio_output_wide",
+                       dma_record_output_wide as *const () as u32).unwrap();
+
+        DMA_RECORDING = true;
+        send(&DmaRecordStart);
+    }
+}
+
+extern fn dma_record_stop(name: CSlice<u8>) {
+    unsafe {
+        if !DMA_RECORDING {
+            raise!("DMAError", "DMA is not recording")
+        }
+
+        let library = LIBRARY.as_ref().unwrap();
+        library.rebind(b"rtio_output",
+                       rtio::output as *const () as u32).unwrap();
+        library.rebind(b"rtio_output_wide",
+                       rtio::output_wide as *const () as u32).unwrap();
+
+        DMA_RECORDING = false;
+        send(&DmaRecordStop(str::from_utf8(name.as_ref()).unwrap()));
+    }
+}
+
+extern fn dma_record_output(timestamp: i64, channel: i32, address: i32, data: i32) {
+    send(&DmaRecordAppend {
+        timestamp: timestamp as u64,
+        channel:   channel as u32,
+        address:   address as u32,
+        data:      &[data as u32]
+    })
+}
+
+extern fn dma_record_output_wide(timestamp: i64, channel: i32, address: i32, data: CSlice<i32>) {
+    assert!(data.len() <= 16); // enforce the hardware limit
+    send(&DmaRecordAppend {
+        timestamp: timestamp as u64,
+        channel:   channel as u32,
+        address:   address as u32,
+        data:      unsafe { mem::transmute::<&[i32], &[u32]>(data.as_ref()) }
+    })
 }
 
 unsafe fn attribute_writeback(typeinfo: *const ()) {
@@ -262,9 +304,6 @@ unsafe fn attribute_writeback(typeinfo: *const ()) {
         objects:    *const *const ()
     }
 
-    // artiq_compile'd kernels don't include type information
-    if typeinfo.is_null() { return }
-
     let mut tys = typeinfo as *const *const Type;
     while !(*tys).is_null() {
         let ty = *tys;
@@ -281,7 +320,7 @@ unsafe fn attribute_writeback(typeinfo: *const ()) {
                 attributes = attributes.offset(1);
 
                 if (*attribute).tag.len() > 0 {
-                    send_async_rpc(0, (*attribute).tag, [
+                    rpc_send_async(0, (*attribute).tag, [
                         &object as *const _ as *const (),
                         &(*attribute).name as *const _ as *const (),
                         (object as usize + (*attribute).offset) as *const ()
@@ -294,8 +333,12 @@ unsafe fn attribute_writeback(typeinfo: *const ()) {
 
 #[no_mangle]
 pub unsafe fn main() {
+    let image = slice::from_raw_parts_mut(kernel_proto::KERNELCPU_PAYLOAD_ADDRESS as *mut u8,
+                                          kernel_proto::KERNELCPU_LAST_ADDRESS -
+                                          kernel_proto::KERNELCPU_PAYLOAD_ADDRESS);
+
     let library = recv!(&LoadRequest(library) => {
-        match Library::load(library, kernel_proto::KERNELCPU_PAYLOAD_ADDRESS, api::resolve) {
+        match Library::load(library, image, &api::resolve) {
             Err(error) => {
                 send(&LoadReply(Err(error)));
                 loop {}
@@ -307,16 +350,23 @@ pub unsafe fn main() {
         }
     });
 
-    let __bss_start = library.lookup("__bss_start");
+    let __bss_start = library.lookup(b"__bss_start").unwrap();
-    let _end = library.lookup("_end");
+    let _end = library.lookup(b"_end").unwrap();
-    ptr::write_bytes(__bss_start as *mut u8, 0, _end - __bss_start);
+    let __modinit__ = library.lookup(b"__modinit__").unwrap();
+    let typeinfo = library.lookup(b"typeinfo");
+
+    LIBRARY = Some(library);
+
+    ptr::write_bytes(__bss_start as *mut u8, 0, (_end - __bss_start) as usize);
 
     send(&NowInitRequest);
     recv!(&NowInitReply(now) => NOW = now);
-    (mem::transmute::<usize, fn()>(library.lookup("__modinit__")))();
+    (mem::transmute::<u32, fn()>(__modinit__))();
     send(&NowSave(NOW));
 
-    attribute_writeback(library.lookup("typeinfo") as *const ());
+    if let Some(typeinfo) = typeinfo {
+        attribute_writeback(typeinfo as *const ());
+    }
 
     send(&RunFinished);
 
@@ -325,6 +375,11 @@ pub unsafe fn main() {
 
 #[no_mangle]
 pub extern fn exception_handler(vect: u32, _regs: *const u32, pc: u32, ea: u32) {
-    println!("exception {:?} at PC 0x{:x}, EA 0x{:x}", vect, pc, ea);
+    panic!("exception {:?} at PC 0x{:x}, EA 0x{:x}", vect, pc, ea)
-    send(&RunAborted)
+}
+
+// We don't export this because libbase does.
+// #[no_mangle]
+pub extern fn abort() {
+    panic!("aborted")
 }

artiq/firmware/ksupport/rtio.rs

@@ -1,6 +1,3 @@
-#[path = "../runtime/kernel_proto.rs"]
-mod kernel_proto;
-
 use core::ptr::{read_volatile, write_volatile};
 use cslice::CSlice;
 use board::csr;
@@ -16,7 +13,7 @@ const RTIO_I_STATUS_EMPTY:          u32 = 1;
 const RTIO_I_STATUS_OVERFLOW:       u32 = 2;
 
 pub extern fn init() {
-    send(&RTIOInitRequest);
+    send(&RtioInitRequest);
 }
 
 pub extern fn get_counter() -> i64 {
@@ -46,25 +43,25 @@ unsafe fn process_exceptional_status(timestamp: i64, channel: i32, status: u32)
     }
     if status & RTIO_O_STATUS_UNDERFLOW != 0 {
         csr::rtio::o_underflow_reset_write(1);
-        artiq_raise!("RTIOUnderflow",
+        raise!("RTIOUnderflow",
             "RTIO underflow at {0} mu, channel {1}, slack {2} mu",
             timestamp, channel as i64, timestamp - get_counter())
     }
     if status & RTIO_O_STATUS_SEQUENCE_ERROR != 0 {
         csr::rtio::o_sequence_error_reset_write(1);
-        artiq_raise!("RTIOSequenceError",
+        raise!("RTIOSequenceError",
             "RTIO sequence error at {0} mu, channel {1}",
             timestamp, channel as i64, 0)
     }
     if status & RTIO_O_STATUS_COLLISION != 0 {
         csr::rtio::o_collision_reset_write(1);
-        artiq_raise!("RTIOCollision",
+        raise!("RTIOCollision",
             "RTIO collision at {0} mu, channel {1}",
             timestamp, channel as i64, 0)
     }
     if status & RTIO_O_STATUS_BUSY != 0 {
         csr::rtio::o_busy_reset_write(1);
-        artiq_raise!("RTIOBusy",
+        raise!("RTIOBusy",
             "RTIO busy on channel {0}",
             channel as i64, 0, 0)
     }
@@ -122,7 +119,7 @@ pub extern fn input_timestamp(timeout: i64, channel: i32) -> u64 {
         }
 
         if status & RTIO_I_STATUS_OVERFLOW != 0 {
-            artiq_raise!("RTIOOverflow",
+            raise!("RTIOOverflow",
                 "RTIO input overflow on channel {0}",
                 channel as i64, 0, 0);
         }
@@ -145,7 +142,7 @@ pub extern fn input_data(channel: i32) -> i32 {
 
             if status & RTIO_I_STATUS_OVERFLOW != 0 {
                 csr::rtio::i_overflow_reset_write(1);
-                artiq_raise!("RTIOOverflow",
+                raise!("RTIOOverflow",
                     "RTIO input overflow on channel {0}",
                     channel as i64, 0, 0);
             }
@@ -193,31 +190,31 @@ pub mod drtio_dbg {
     pub struct ChannelState(i32, i64);
 
     pub extern fn get_channel_state(channel: i32) -> ChannelState {
-        send(&DRTIOChannelStateRequest { channel: channel as u32 });
+        send(&DrtioChannelStateRequest { channel: channel as u32 });
-        recv!(&DRTIOChannelStateReply { fifo_space, last_timestamp }
+        recv!(&DrtioChannelStateReply { fifo_space, last_timestamp }
               => ChannelState(fifo_space as i32, last_timestamp as i64))
     }
 
     pub extern fn reset_channel_state(channel: i32) {
-        send(&DRTIOResetChannelStateRequest { channel: channel as u32 })
+        send(&DrtioResetChannelStateRequest { channel: channel as u32 })
     }
 
     pub extern fn get_fifo_space(channel: i32) {
-        send(&DRTIOGetFIFOSpaceRequest { channel: channel as u32 })
+        send(&DrtioGetFifoSpaceRequest { channel: channel as u32 })
     }
 
     #[repr(C)]
     pub struct PacketCounts(i32, i32);
 
     pub extern fn get_packet_counts() -> PacketCounts {
-        send(&DRTIOPacketCountRequest);
+        send(&DrtioPacketCountRequest);
-        recv!(&DRTIOPacketCountReply { tx_cnt, rx_cnt }
+        recv!(&DrtioPacketCountReply { tx_cnt, rx_cnt }
               => PacketCounts(tx_cnt as i32, rx_cnt as i32))
     }
 
     pub extern fn get_fifo_space_req_count() -> i32 {
-        send(&DRTIOFIFOSpaceReqCountRequest);
+        send(&DrtioFifoSpaceReqCountRequest);
-        recv!(&DRTIOFIFOSpaceReqCountReply { cnt }
+        recv!(&DrtioFifoSpaceReqCountReply { cnt }
               => cnt as i32)
     }
 }

artiq/firmware/libamp/Cargo.toml

@@ -0,0 +1,11 @@
+[package]
+authors = ["M-Labs"]
+name = "amp"
+version = "0.0.0"
+
+[lib]
+name = "amp"
+path = "lib.rs"
+
+[dependencies]
+board = { path = "../libboard" }

artiq/firmware/libamp/lib.rs

@@ -0,0 +1,6 @@
+#![no_std]
+
+extern crate board;
+
+pub mod mailbox;
+pub mod rpc_queue;

artiq/firmware/libamp/rpc_queue.rs

@@ -1,5 +1,3 @@
-#![allow(dead_code)]
-
 use core::ptr::{read_volatile, write_volatile};
 use core::slice;
 use board::{mem, cache};

artiq/firmware/libdyld/Cargo.toml

@@ -0,0 +1,8 @@
+[package]
+authors = ["M-Labs"]
+name = "dyld"
+version = "0.0.0"
+
+[lib]
+name = "dyld"
+path = "lib.rs"

artiq/firmware/libdyld/lib.rs

@@ -0,0 +1,338 @@
+#![no_std]
+#![feature(untagged_unions, ptr_unaligned)]
+
+use core::{mem, ptr, fmt, slice, str, convert};
+use elf::*;
+
+pub mod elf;
+
+fn read_unaligned<T: Copy>(data: &[u8], offset: usize) -> Result<T, ()> {
+    if data.len() < offset + mem::size_of::<T>() {
+        Err(())
+    } else {
+        let ptr = data.as_ptr().wrapping_offset(offset as isize) as *const T;
+        Ok(unsafe { ptr::read_unaligned(ptr) })
+    }
+}
+
+fn get_ref<T: Copy>(data: &[u8], offset: usize) -> Result<&T, ()> {
+    if data.len() < offset + mem::size_of::<T>() {
+        Err(())
+    } else if (data.as_ptr() as usize + offset) & (mem::align_of::<T>() - 1) != 0 {
+        Err(())
+    } else {
+        let ptr = data.as_ptr().wrapping_offset(offset as isize) as *const T;
+        Ok(unsafe { &*ptr })
+    }
+}
+
+fn get_ref_slice<T: Copy>(data: &[u8], offset: usize, len: usize) -> Result<&[T], ()> {
+    if data.len() < offset + mem::size_of::<T>() * len {
+        Err(())
+    } else if (data.as_ptr() as usize + offset) & (mem::align_of::<T>() - 1) != 0 {
+        Err(())
+    } else {
+        let ptr = data.as_ptr().wrapping_offset(offset as isize) as *const T;
+        Ok(unsafe { slice::from_raw_parts(ptr, len) })
+    }
+}
+
+fn elf_hash(name: &[u8]) -> u32 {
+    let mut h: u32 = 0;
+    for c in name {
+        h = (h << 4) + *c as u32;
+        let g = h & 0xf0000000;
+        if g != 0 {
+            h ^= g >> 24;
+            h &= !g;
+        }
+    }
+    h
+}
+
+#[derive(Debug)]
+pub enum Error<'a> {
+    Parsing(&'static str),
+    Lookup(&'a [u8])
+}
+
+impl<'a> convert::From<&'static str> for Error<'a> {
+    fn from(desc: &'static str) -> Error<'a> {
+        Error::Parsing(desc)
+    }
+}
+
+impl<'a> fmt::Display for Error<'a> {
+    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+        match self {
+            &Error::Parsing(desc) =>
+                write!(f, "parse error: {}", desc),
+            &Error::Lookup(sym) =>
+                match str::from_utf8(sym) {
+                    Ok(sym) => write!(f, "symbol lookup error: {}", sym),
+                    Err(_)  => write!(f, "symbol lookup error: {:?}", sym)
+                }
+        }
+    }
+}
+
+pub struct Library<'a> {
+    image_off:   Elf32_Addr,
+    image_sz:    usize,
+    strtab:      &'a [u8],
+    symtab:      &'a [Elf32_Sym],
+    pltrel:      &'a [Elf32_Rela],
+    hash_bucket: &'a [Elf32_Word],
+    hash_chain:  &'a [Elf32_Word],
+}
+
+impl<'a> Library<'a> {
+    pub fn lookup(&self, name: &[u8]) -> Option<Elf32_Word> {
+        let hash = elf_hash(name);
+        let mut index = self.hash_bucket[hash as usize % self.hash_bucket.len()] as usize;
+
+        loop {
+            if index == STN_UNDEF { return None }
+
+            let sym = &self.symtab[index];
+            let sym_name_off = sym.st_name as usize;
+            match self.strtab.get(sym_name_off..sym_name_off + name.len()) {
+                Some(sym_name) if sym_name == name => {
+                    if ELF32_ST_BIND(sym.st_info) & STB_GLOBAL == 0 {
+                        return None
+                    }
+
+                    match sym.st_shndx {
+                        SHN_UNDEF => return None,
+                        SHN_ABS => return Some(sym.st_value),
+                        _ => return Some(self.image_off + sym.st_value)
+                    }
+                }
+                _ => (),
+            }
+
+            index = self.hash_chain[index] as usize;
+        }
+    }
+
+    fn name_starting_at(&self, offset: usize) -> Result<&'a [u8], Error<'a>> {
+        let size = self.strtab.iter().skip(offset).position(|&x| x == 0)
+                              .ok_or("symbol in symbol table not null-terminated")?;
+        Ok(self.strtab.get(offset..offset + size)
+                      .ok_or("cannot read symbol name")?)
+    }
+
+    fn update_rela(&self, rela: &Elf32_Rela, value: Elf32_Word) -> Result<(), Error<'a>> {
+        if rela.r_offset as usize + mem::size_of::<Elf32_Addr>() > self.image_sz {
+            return Err("relocation out of image bounds")?
+        }
+
+        let ptr = (self.image_off + rela.r_offset) as *mut Elf32_Addr;
+        Ok(unsafe { *ptr = value })
+    }
+
+    // This is unsafe because it mutates global data (the PLT).
+    pub unsafe fn rebind(&self, name: &[u8], addr: Elf32_Word) -> Result<(), Error<'a>> {
+        for rela in self.pltrel.iter() {
+            match ELF32_R_TYPE(rela.r_info) {
+                R_OR1K_32 | R_OR1K_GLOB_DAT | R_OR1K_JMP_SLOT => {
+                    let sym = self.symtab.get(ELF32_R_SYM(rela.r_info) as usize)
+                                         .ok_or("symbol out of bounds of symbol table")?;
+                    let sym_name = self.name_starting_at(sym.st_name as usize)?;
+
+                    if sym_name == name {
+                        self.update_rela(rela, addr)?
+                    }
+                }
+
+                // No associated symbols for other relocation types.
+                _ => ()
+            }
+        }
+        Ok(())
+    }
+
+    fn resolve_rela(&self, rela: &Elf32_Rela, resolve: &Fn(&[u8]) -> Option<Elf32_Word>)
+            -> Result<(), Error<'a>> {
+        let sym;
+        if ELF32_R_SYM(rela.r_info) == 0 {
+            sym = None;
+        } else {
+            sym = Some(self.symtab.get(ELF32_R_SYM(rela.r_info) as usize)
+                                  .ok_or("symbol out of bounds of symbol table")?)
+        }
+
+        let value;
+        match ELF32_R_TYPE(rela.r_info) {
+            R_OR1K_NONE =>
+                return Ok(()),
+
+            R_OR1K_RELATIVE =>
+                value = self.image_off + rela.r_addend as Elf32_Word,
+
+            R_OR1K_32 | R_OR1K_GLOB_DAT | R_OR1K_JMP_SLOT => {
+                let sym = sym.ok_or("relocation requires an associated symbol")?;
+                let sym_name = self.name_starting_at(sym.st_name as usize)?;
+
+                // First, try to resolve against itself.
+                match self.lookup(sym_name) {
+                    Some(addr) => value = addr,
+                    None => {
+                        // Second, call the user-provided function.
+                        match resolve(sym_name) {
+                            Some(addr) => value = addr,
+                            None => {
+                                // We couldn't find it anywhere.
+                                return Err(Error::Lookup(sym_name))
+                            }
+                        }
+                    }
+                }
+            }
+
+            _ => return Err("unsupported relocation type")?
+        }
+
+        self.update_rela(rela, value)
+    }
+
+    pub fn load(data: &[u8], image: &'a mut [u8], resolve: &Fn(&[u8]) -> Option<Elf32_Word>)
+            -> Result<Library<'a>, Error<'a>> {
+        let ehdr = read_unaligned::<Elf32_Ehdr>(data, 0)
+                                  .map_err(|()| "cannot read ELF header")?;
+
+        const IDENT: [u8; EI_NIDENT] = [
+            ELFMAG0,    ELFMAG1,     ELFMAG2,    ELFMAG3,
+            ELFCLASS32, ELFDATA2MSB, EV_CURRENT, ELFOSABI_NONE,
+            /* ABI version */ 0, /* padding */ 0, 0, 0, 0, 0, 0, 0
+        ];
+
+        #[cfg(target_arch = "or1k")]
+        const ARCH: u16 = EM_OPENRISC;
+        #[cfg(not(target_arch = "or1k"))]
+        const ARCH: u16 = EM_NONE;
+
+        if ehdr.e_ident != IDENT || ehdr.e_type != ET_DYN || ehdr.e_machine != ARCH {
+            return Err("not a shared library for current architecture")?
+        }
+
+        let mut dyn_off = None;
+        for i in 0..ehdr.e_phnum {
+            let phdr_off = ehdr.e_phoff as usize + mem::size_of::<Elf32_Phdr>() * i as usize;
+            let phdr = read_unaligned::<Elf32_Phdr>(data, phdr_off)
+                                      .map_err(|()| "cannot read program header")?;
+
+            match phdr.p_type {
+                PT_LOAD => {
+                    if (phdr.p_vaddr + phdr.p_filesz) as usize > image.len() ||
+                            (phdr.p_offset + phdr.p_filesz) as usize > data.len() {
+                        return Err("program header requests an out of bounds load")?
+                    }
+                    let dst = image.get_mut(phdr.p_vaddr as usize..
+                                            (phdr.p_vaddr + phdr.p_filesz) as usize)
+                                   .ok_or("cannot write to program header destination")?;
+                    let src = data.get(phdr.p_offset as usize..
+                                       (phdr.p_offset + phdr.p_filesz) as usize)
+                                  .ok_or("cannot read from program header source")?;
+                    dst.copy_from_slice(src);
+                }
+
+                PT_DYNAMIC =>
+                    dyn_off = Some(phdr.p_vaddr),
+
+                _ => ()
+            }
+        }
+
+        let (mut strtab_off, mut strtab_sz) = (0, 0);
+        let (mut symtab_off, mut symtab_sz) = (0, 0);
+        let (mut rela_off,   mut rela_sz)   = (0, 0);
+        let (mut pltrel_off, mut pltrel_sz) = (0, 0);
+        let (mut hash_off,   mut hash_sz)   = (0, 0);
+        let mut sym_ent  = 0;
+        let mut rela_ent = 0;
+        let mut nbucket  = 0;
+        let mut nchain   = 0;
+
+        let dyn_off = dyn_off.ok_or("cannot find a dynamic header")?;
+        for i in 0.. {
+            let dyn_off = dyn_off as usize + i * mem::size_of::<Elf32_Dyn>();
+            let dyn = get_ref::<Elf32_Dyn>(image, dyn_off)
+                              .map_err(|()| "cannot read dynamic header")?;
+
+            let val = unsafe { dyn.d_un.d_val } as usize;
+            match dyn.d_tag {
+                DT_NULL     => break,
+                DT_REL      => return Err("relocations with implicit addend are not supported")?,
+                DT_STRTAB   => strtab_off = val,
+                DT_STRSZ    => strtab_sz  = val,
+                DT_SYMTAB   => symtab_off = val,
+                DT_SYMENT   => sym_ent    = val,
+                DT_RELA     => rela_off   = val,
+                DT_RELASZ   => rela_sz    = val / mem::size_of::<Elf32_Rela>(),
+                DT_RELAENT  => rela_ent   = val,
+                DT_JMPREL   => pltrel_off = val,
+                DT_PLTRELSZ => pltrel_sz  = val / mem::size_of::<Elf32_Rela>(),
+                DT_HASH     => {
+                    nbucket  = *get_ref::<Elf32_Word>(image, val + 0)
+                                        .map_err(|()| "cannot read hash bucket count")? as usize;
+                    nchain   = *get_ref::<Elf32_Word>(image, val + 4)
+                                        .map_err(|()| "cannot read hash chain count")? as usize;
+                    hash_off = val + 8;
+                    hash_sz  = nbucket + nchain;
+                }
+                _ => ()
+            }
+        }
+
+        if sym_ent != mem::size_of::<Elf32_Sym>() {
+            return Err("incorrect symbol entry size")?
+        }
+        if rela_ent != mem::size_of::<Elf32_Rela>() {
+            return Err("incorrect relocation entry size")?
+        }
+
+        // These are the same--there are as many chains as buckets, and the chains only contain
+        // the symbols that overflowed the bucket.
+        symtab_sz = nchain;
+
+        // Drop the mutability. See also the comment below.
+        let image = &*image;
+
+        let strtab = get_ref_slice::<u8>(image, strtab_off, strtab_sz)
+                                   .map_err(|()| "cannot read string table")?;
+        let symtab = get_ref_slice::<Elf32_Sym>(image, symtab_off, symtab_sz)
+                                   .map_err(|()| "cannot read symbol table")?;
+        let rela   = get_ref_slice::<Elf32_Rela>(image, rela_off, rela_sz)
+                                   .map_err(|()| "cannot read rela entries")?;
+        let pltrel = get_ref_slice::<Elf32_Rela>(image, pltrel_off, pltrel_sz)
+                                   .map_err(|()| "cannot read pltrel entries")?;
+        let hash   = get_ref_slice::<Elf32_Word>(image, hash_off, hash_sz)
+                                   .map_err(|()| "cannot read hash entries")?;
+
+        let library = Library {
+            image_off:   image.as_ptr() as Elf32_Word,
+            image_sz:    image.len(),
+            strtab:      strtab,
+            symtab:      symtab,
+            pltrel:      pltrel,
+            hash_bucket: &hash[..nbucket],
+            hash_chain:  &hash[nbucket..nbucket + nchain],
+        };
+
+        // If a borrow exists anywhere, the borrowed memory cannot be mutated except
+        // through that pointer or it's UB. However, we need to retain pointers
+        // to the symbol tables and relocations, and at the same time mutate the code
+        // to resolve the relocations.
+        //
+        // To avoid invoking UB, we drop the only pointer to the entire area (which is
+        // unique since it's a &mut); we retain pointers to the various tables, but
+        // we never write to the memory they refer to, so it's safe.
+        mem::drop(image);
+
+        for r in rela   { library.resolve_rela(r, resolve)? }
+        for r in pltrel { library.resolve_rela(r, resolve)? }
+
+        Ok(library)
+    }
+}

artiq/firmware/libproto/Cargo.toml

@@ -0,0 +1,15 @@
+[package]
+authors = ["M-Labs"]
+name = "proto"
+version = "0.0.0"
+
+[lib]
+name = "proto"
+path = "lib.rs"
+
+[dependencies]
+byteorder = { version = "1.0", default-features = false }
+cslice = { version = "0.3" }
+log = { version = "0.3", default-features = false, optional = true }
+std_artiq = { path = "../libstd_artiq", features = ["alloc"] }
+dyld = { path = "../libdyld" }

artiq/firmware/libproto/analyzer_proto.rs

@@ -1,5 +1,5 @@
 use std::io::{self, Write};
-use proto::*;
+use io::*;
 
 #[derive(Debug)]
 pub struct Header {

artiq/firmware/libproto/io.rs

@@ -1,5 +1,3 @@
-#![allow(dead_code)]
-
 use std::io::{self, Read, Write};
 use std::vec::Vec;
 use std::string::String;
@@ -55,7 +53,8 @@ pub fn write_u64(writer: &mut Write, value: u64) -> io::Result<()> {
 
 pub fn read_bytes(reader: &mut Read) -> io::Result<Vec<u8>> {
     let length = read_u32(reader)?;
-    let mut value = vec![0; length as usize];
+    let mut value = Vec::new();
+    value.resize(length as usize, 0);
     reader.read_exact(&mut value)?;
     Ok(value)
 }

artiq/firmware/libproto/kernel_proto.rs

@@ -1,6 +1,5 @@
-#![allow(dead_code)]
-
 use core::fmt;
+use dyld;
 
 pub const KERNELCPU_EXEC_ADDRESS:    usize = 0x40800000;
 pub const KERNELCPU_PAYLOAD_ADDRESS: usize = 0x40840000;
@@ -22,22 +21,31 @@ pub struct Exception<'a> {
 #[derive(Debug)]
 pub enum Message<'a> {
     LoadRequest(&'a [u8]),
-    LoadReply(Result<(), &'a str>),
+    LoadReply(Result<(), dyld::Error<'a>>),
 
     NowInitRequest,
     NowInitReply(u64),
     NowSave(u64),
 
-    RTIOInitRequest,
+    RtioInitRequest,
 
-    DRTIOChannelStateRequest { channel: u32 },
+    DmaRecordStart,
-    DRTIOChannelStateReply { fifo_space: u16, last_timestamp: u64 },
+    DmaRecordAppend {
-    DRTIOResetChannelStateRequest { channel: u32 },
+        timestamp: u64,
-    DRTIOGetFIFOSpaceRequest { channel: u32 },
+        channel:   u32,
-    DRTIOPacketCountRequest,
+        address:   u32,
-    DRTIOPacketCountReply { tx_cnt: u32, rx_cnt: u32 },
+        data:      &'a [u32]
-    DRTIOFIFOSpaceReqCountRequest,
+    },
-    DRTIOFIFOSpaceReqCountReply { cnt: u32 },
+    DmaRecordStop(&'a str),
+
+    DrtioChannelStateRequest { channel: u32 },
+    DrtioChannelStateReply { fifo_space: u16, last_timestamp: u64 },
+    DrtioResetChannelStateRequest { channel: u32 },
+    DrtioGetFifoSpaceRequest { channel: u32 },
+    DrtioPacketCountRequest,
+    DrtioPacketCountReply { tx_cnt: u32, rx_cnt: u32 },
+    DrtioFifoSpaceReqCountRequest,
+    DrtioFifoSpaceReqCountReply { cnt: u32 },
 
     RunFinished,
     RunException {
@@ -64,12 +72,12 @@ pub enum Message<'a> {
     CachePutRequest { key: &'a str, value: &'a [i32] },
     CachePutReply   { succeeded: bool },
 
-    I2CStartRequest { busno: u8 },
+    I2cStartRequest { busno: u8 },
-    I2CStopRequest { busno: u8 },
+    I2cStopRequest { busno: u8 },
-    I2CWriteRequest { busno: u8, data: u8 },
+    I2cWriteRequest { busno: u8, data: u8 },
-    I2CWriteReply { ack: bool },
+    I2cWriteReply { ack: bool },
-    I2CReadRequest { busno: u8, ack: bool },
+    I2cReadRequest { busno: u8, ack: bool },
-    I2CReadReply { data: u8 },
+    I2cReadReply { data: u8 },
 
     Log(fmt::Arguments<'a>),
     LogSlice(&'a str)

artiq/firmware/libproto/lib.rs

@@ -0,0 +1,21 @@
+#![no_std]
+
+extern crate byteorder;
+extern crate cslice;
+#[cfg(feature = "log")]
+#[macro_use]
+extern crate log;
+
+extern crate dyld;
+extern crate std_artiq as std;
+
+pub mod io;
+
+// Internal protocols.
+pub mod kernel_proto;
+
+// External protocols.
+pub mod analyzer_proto;
+pub mod moninj_proto;
+pub mod session_proto;
+pub mod rpc_proto;

artiq/firmware/libproto/moninj_proto.rs

@@ -1,5 +1,5 @@
 use std::io::{self, Read, Write};
-use proto::*;
+use io::*;
 
 #[derive(Debug)]
 pub enum HostMessage {

artiq/firmware/libproto/rpc_proto.rs

@@ -1,9 +1,7 @@
-#![allow(dead_code)]
-
-use core::str;
 use std::io::{self, Read, Write};
+use std::str;
 use cslice::{CSlice, CMutSlice};
-use proto::*;
+use io::*;
 use self::tag::{Tag, TagIterator, split_tag};
 
 unsafe fn recv_value(reader: &mut Read, tag: Tag, data: &mut *mut (),
@@ -76,7 +74,7 @@ unsafe fn recv_value(reader: &mut Read, tag: Tag, data: &mut *mut (),
 pub fn recv_return(reader: &mut Read, tag_bytes: &[u8], data: *mut (),
                    alloc: &Fn(usize) -> io::Result<*mut ()>) -> io::Result<()> {
     let mut it = TagIterator::new(tag_bytes);
-    #[cfg(not(ksupport))]
+    #[cfg(feature = "log")]
     trace!("recv ...->{}", it);
 
     let tag = it.next().expect("truncated tag");
@@ -162,7 +160,7 @@ pub fn send_args(writer: &mut Write, service: u32, tag_bytes: &[u8],
     let (arg_tags_bytes, return_tag_bytes) = split_tag(tag_bytes);
 
     let mut args_it = TagIterator::new(arg_tags_bytes);
-    #[cfg(not(ksupport))]
+    #[cfg(feature = "log")]
     {
         let return_it = TagIterator::new(return_tag_bytes);
         trace!("send<{}>({})->{}", service, args_it, return_it);

artiq/firmware/libproto/session_proto.rs

@@ -1,6 +1,7 @@
-use std::prelude::v1::*;
 use std::io::{self, Read, Write};
-use proto::*;
+use std::vec::Vec;
+use std::string::String;
+use io::*;
 
 fn read_sync(reader: &mut Read) -> io::Result<()> {
     let mut sync = [0; 4];

artiq/firmware/runtime/Cargo.toml

@@ -13,15 +13,17 @@ path = "lib.rs"
 build_artiq = { path = "../libbuild_artiq" }
 
 [dependencies]
+byteorder = { version = "1.0", default-features = false }
+cslice = { version = "0.3" }
+log = { version = "0.3", default-features = false, features = ["max_level_debug"] }
+fringe = { version = "= 1.1.0", default-features = false, features = ["alloc"] }
 alloc_artiq = { path = "../liballoc_artiq" }
 std_artiq = { path = "../libstd_artiq", features = ["alloc"] }
 logger_artiq = { path = "../liblogger_artiq" }
-cslice = { version = "0.3" }
-log = { version = "0.3", default-features = false, features = ["max_level_debug"] }
 board = { path = "../libboard", features = ["uart_console"] }
+proto = { path = "../libproto", features = ["log"] }
+amp = { path = "../libamp" }
 drtioaux = { path = "../libdrtioaux" }
-fringe = { version = "= 1.1.0", default-features = false, features = ["alloc"] }
-byteorder = { version = "1.0", default-features = false }
 
 [dependencies.compiler_builtins]
 git = "https://github.com/rust-lang-nursery/compiler-builtins"

artiq/firmware/runtime/lib.rs

@@ -2,23 +2,28 @@
 #![feature(compiler_builtins_lib, repr_simd, const_fn)]
 
 extern crate compiler_builtins;
-extern crate alloc_artiq;
-#[macro_use]
-extern crate std_artiq as std;
 extern crate cslice;
 #[macro_use]
 extern crate log;
-extern crate logger_artiq;
 extern crate byteorder;
 extern crate fringe;
 extern crate smoltcp;
+
+extern crate alloc_artiq;
+#[macro_use]
+extern crate std_artiq as std;
+extern crate logger_artiq;
 #[macro_use]
 extern crate board;
+extern crate proto;
+extern crate amp;
 #[cfg(has_drtio)]
 extern crate drtioaux;
 
 use std::boxed::Box;
 use smoltcp::wire::{EthernetAddress, IpAddress};
+use proto::{analyzer_proto, moninj_proto, rpc_proto, session_proto, kernel_proto};
+use amp::{mailbox, rpc_queue};
 
 macro_rules! borrow_mut {
     ($x:expr) => ({
@@ -32,21 +37,11 @@ macro_rules! borrow_mut {
 mod config;
 mod ethmac;
 mod rtio_mgt;
-mod mailbox;
-mod rpc_queue;
 
 mod urc;
 mod sched;
 mod cache;
-
+mod rtio_dma;
-mod proto;
-mod kernel_proto;
-mod session_proto;
-#[cfg(any(has_rtio_moninj, has_drtio))]
-mod moninj_proto;
-#[cfg(has_rtio_analyzer)]
-mod analyzer_proto;
-mod rpc_proto;
 
 mod kernel;
 mod session;

artiq/firmware/runtime/rtio_dma.rs

@@ -0,0 +1,58 @@
+use std::vec::Vec;
+use std::string::String;
+use std::btree_map::BTreeMap;
+use std::mem;
+use proto::io;
+
+#[derive(Debug)]
+struct Entry {
+    data: Vec<u8>
+}
+
+#[derive(Debug)]
+pub struct Manager {
+    entries: BTreeMap<String, Entry>,
+    recording: Vec<u8>
+}
+
+impl Manager {
+    pub fn new() -> Manager {
+        Manager {
+            entries: BTreeMap::new(),
+            recording: Vec::new()
+        }
+    }
+
+    pub fn record_start(&mut self) {
+        self.recording.clear();
+    }
+
+    pub fn record_append(&mut self, timestamp: u64, channel: u32,
+                         address: u32, data: &[u32]) {
+        // See gateware/rtio/dma.py.
+        let length = /*length*/1 + /*channel*/3 + /*timestamp*/8 + /*address*/2 +
+                     /*data*/data.len() * 4;
+        let writer = &mut self.recording;
+        io::write_u8(writer, length as u8).unwrap();
+        io::write_u8(writer, (channel >> 24) as u8).unwrap();
+        io::write_u8(writer, (channel >> 16) as u8).unwrap();
+        io::write_u8(writer, (channel >>  8) as u8).unwrap();
+        io::write_u64(writer, timestamp).unwrap();
+        io::write_u16(writer, address as u16).unwrap();
+        for &word in data {
+            io::write_u32(writer, word).unwrap();
+        }
+    }
+
+    pub fn record_stop(&mut self, name: &str) {
+        let mut recorded = Vec::new();
+        mem::swap(&mut self.recording, &mut recorded);
+        recorded.shrink_to_fit();
+
+        info!("recorded DMA data: {:?}", recorded);
+
+        self.entries.insert(String::from(name), Entry {
+            data: recorded
+        });
+    }
+}

artiq/firmware/runtime/session.rs

@@ -6,6 +6,7 @@ use std::error::Error;
 use {config, rtio_mgt, mailbox, rpc_queue, kernel};
 use logger_artiq::BufferLogger;
 use cache::Cache;
+use rtio_dma::Manager as DmaManager;
 use urc::Urc;
 use sched::{ThreadHandle, Io};
 use sched::{TcpListener, TcpStream};
@@ -34,6 +35,7 @@ fn io_error(msg: &str) -> io::Error {
 struct Congress {
     now: u64,
     cache: Cache,
+    dma_manager: DmaManager,
     finished_cleanly: Cell<bool>
 }
 
@@ -42,6 +44,7 @@ impl Congress {
         Congress {
             now: 0,
             cache: Cache::new(),
+            dma_manager: DmaManager::new(),
             finished_cleanly: Cell::new(true)
         }
     }
@@ -179,7 +182,7 @@ unsafe fn kern_load(io: &Io, session: &mut Session, library: &[u8]) -> io::Resul
                 session.kernel_state = KernelState::Loaded;
                 Ok(())
             }
-            &kern::LoadReply(Err(error)) => {
+            &kern::LoadReply(Err(ref error)) => {
                 kernel::stop();
                 Err(io::Error::new(io::ErrorKind::Other,
                                    format!("cannot load kernel: {}", error)))
@@ -347,8 +350,7 @@ fn process_host_message(io: &Io,
     }
 }
 
-fn process_kern_message(io: &Io,
+fn process_kern_message(io: &Io, mut stream: Option<&mut TcpStream>,
-                        mut stream: Option<&mut TcpStream>,
                         session: &mut Session) -> io::Result<bool> {
     kern_recv_notrace(io, |request| {
         match (request, session.kernel_state) {
@@ -388,32 +390,45 @@ fn process_kern_message(io: &Io,
                 kern_acknowledge()
             }
 
-            &kern::RTIOInitRequest => {
+            &kern::RtioInitRequest => {
                 info!("resetting RTIO");
                 rtio_mgt::init_core();
                 kern_acknowledge()
             }
 
-            &kern::DRTIOChannelStateRequest { channel } => {
+            &kern::DmaRecordStart => {
+                session.congress.dma_manager.record_start();
+                kern_acknowledge()
+            }
+            &kern::DmaRecordAppend { timestamp, channel, address, data } => {
+                session.congress.dma_manager.record_append(timestamp, channel, address, data);
+                kern_acknowledge()
+            }
+            &kern::DmaRecordStop(name) => {
+                session.congress.dma_manager.record_stop(name);
+                kern_acknowledge()
+            }
+
+            &kern::DrtioChannelStateRequest { channel } => {
                 let (fifo_space, last_timestamp) = rtio_mgt::drtio_dbg::get_channel_state(channel);
-                kern_send(io, &kern::DRTIOChannelStateReply { fifo_space: fifo_space,
+                kern_send(io, &kern::DrtioChannelStateReply { fifo_space: fifo_space,
                                                                   last_timestamp: last_timestamp })
             }
-            &kern::DRTIOResetChannelStateRequest { channel } => {
+            &kern::DrtioResetChannelStateRequest { channel } => {
                 rtio_mgt::drtio_dbg::reset_channel_state(channel);
                 kern_acknowledge()
             }
-            &kern::DRTIOGetFIFOSpaceRequest { channel } => {
+            &kern::DrtioGetFifoSpaceRequest { channel } => {
                 rtio_mgt::drtio_dbg::get_fifo_space(channel);
                 kern_acknowledge()
             }
-            &kern::DRTIOPacketCountRequest => {
+            &kern::DrtioPacketCountRequest => {
                 let (tx_cnt, rx_cnt) = rtio_mgt::drtio_dbg::get_packet_counts();
-                kern_send(io, &kern::DRTIOPacketCountReply { tx_cnt: tx_cnt, rx_cnt: rx_cnt })
+                kern_send(io, &kern::DrtioPacketCountReply { tx_cnt: tx_cnt, rx_cnt: rx_cnt })
             }
-            &kern::DRTIOFIFOSpaceReqCountRequest => {
+            &kern::DrtioFifoSpaceReqCountRequest => {
                 let cnt = rtio_mgt::drtio_dbg::get_fifo_space_req_count();
-                kern_send(io, &kern::DRTIOFIFOSpaceReqCountReply { cnt: cnt })
+                kern_send(io, &kern::DrtioFifoSpaceReqCountReply { cnt: cnt })
             }
 
             &kern::WatchdogSetRequest { ms } => {
@@ -454,41 +469,41 @@ fn process_kern_message(io: &Io,
             }
 
             #[cfg(has_i2c)]
-            &kern::I2CStartRequest { busno } => {
+            &kern::I2cStartRequest { busno } => {
                 board::i2c::start(busno);
                 kern_acknowledge()
             }
             #[cfg(has_i2c)]
-            &kern::I2CStopRequest { busno } => {
+            &kern::I2cStopRequest { busno } => {
                 board::i2c::stop(busno);
                 kern_acknowledge()
             }
             #[cfg(has_i2c)]
-            &kern::I2CWriteRequest { busno, data } => {
+            &kern::I2cWriteRequest { busno, data } => {
                 let ack = board::i2c::write(busno, data);
-                kern_send(io, &kern::I2CWriteReply { ack: ack })
+                kern_send(io, &kern::I2cWriteReply { ack: ack })
             }
             #[cfg(has_i2c)]
-            &kern::I2CReadRequest { busno, ack } => {
+            &kern::I2cReadRequest { busno, ack } => {
                 let data = board::i2c::read(busno, ack);
-                kern_send(io, &kern::I2CReadReply { data: data })
+                kern_send(io, &kern::I2cReadReply { data: data })
             }
 
             #[cfg(not(has_i2c))]
-            &kern::I2CStartRequest { .. } => {
+            &kern::I2cStartRequest { .. } => {
                 kern_acknowledge()
             }
             #[cfg(not(has_i2c))]
-            &kern::I2CStopRequest { .. } => {
+            &kern::I2cStopRequest { .. } => {
                 kern_acknowledge()
             }
             #[cfg(not(has_i2c))]
-            &kern::I2CWriteRequest { .. } => {
+            &kern::I2cWriteRequest { .. } => {
-                kern_send(io, &kern::I2CWriteReply { ack: false })
+                kern_send(io, &kern::I2cWriteReply { ack: false })
             }
             #[cfg(not(has_i2c))]
-            &kern::I2CReadRequest { .. } => {
+            &kern::I2cReadRequest { .. } => {
-                kern_send(io, &kern::I2CReadReply { data: 0xff })
+                kern_send(io, &kern::I2cReadReply { data: 0xff })
             }
 
             &kern::RunFinished => {

artiq/firmware/satman/Makefile

@@ -1,26 +1,15 @@
 include ../include/generated/variables.mak
 include $(MISOC_DIRECTORY)/software/common.mak
 
-PYTHON ?= python3
+LDFLAGS += -L../libbase
 
-RUSTOUT := cargo/or1k-unknown-none/debug
+RUSTFLAGS += -Cpanic=abort
-
-LDFLAGS += --gc-sections \
-	-L../libbase
-
-RUSTFLAGS = \
-	-C target-feature=+mul,+div,+ffl1,+cmov,+addc \
-	-C opt-level=s \
-	-C panic=abort
-export RUSTFLAGS
 
 all: satman.bin satman.fbi
 
 .PHONY: $(RUSTOUT)/libsatman.a
 $(RUSTOUT)/libsatman.a:
-	CARGO_TARGET_DIR=$(realpath .)/cargo \
+	$(cargo) --manifest-path $(SATMAN_DIRECTORY)/Cargo.toml
-		cargo build --target=or1k-unknown-none \
-			--manifest-path $(realpath $(SATMAN_DIRECTORY))/Cargo.toml
 
 satman.elf: $(RUSTOUT)/libsatman.a
 	$(LD) $(LDFLAGS) -T $(SATMAN_DIRECTORY)/satman.ld -o $@ $^
@@ -34,7 +23,7 @@ satman.elf: $(RUSTOUT)/libsatman.a
 	@echo " MSCIMG  " $@ && $(PYTHON) -m misoc.tools.mkmscimg -f -o $@ $<
 
 clean:
-	$(RM) satman.elf satman.bin satman.fbi .*~ *~
+	$(RM) satman.elf satman.bin satman.fbi
 	$(RM) -rf cargo
 
 .PHONY: all clean

artiq/test/lit/embedding/async_rpc.py

@@ -4,7 +4,7 @@
 from artiq.language.core import *
 from artiq.language.types import *
 
-# CHECK: call void @send_async_rpc
+# CHECK: call void @rpc_send_async
 
 @rpc(flags={"async"})
 def foo():

artiq/test/lit/embedding/invariant_nested.py

@@ -0,0 +1,22 @@
+# RUN: %python -m artiq.compiler.testbench.embedding %s
+
+from artiq.language.core import *
+from artiq.language.types import *
+
+
+class ClassA:
+    def __init__(self):
+        self.foo = False
+
+
+class ClassB:
+    kernel_invariants = {"bar"}
+
+    def __init__(self):
+        self.bar = ClassA()
+
+obj = ClassB()
+
+@kernel
+def entrypoint():
+    obj.bar.foo = True

doc/manual/core_drivers_reference.rst

@@ -21,6 +21,12 @@ These drivers are for the core device and the peripherals closely integrated int
 .. automodule:: artiq.coredevice.dds
     :members:
 
+:mod:`artiq.coredevice.dma` module
+----------------------------------
+
+.. automodule:: artiq.coredevice.dma
+    :members:
+
 :mod:`artiq.coredevice.spi` module
 ----------------------------------