runtime: rewrite rtio support code in Rust.

artiq/coredevice/rtio.py

@@ -3,8 +3,7 @@ from artiq.language.types import TInt64, TInt32, TNone
 
 
 @syscall(flags={"nowrite"})
-def rtio_output(time_mu: TInt64, channel: TInt32, addr: TInt32, data: TInt32
+def rtio_output(time_mu: TInt64, channel: TInt32, addr: TInt32, data: TInt32) -> TNone:
-                ) -> TNone:
     raise NotImplementedError("syscall not simulated")
 
 

artiq/runtime.rs/libksupport/api.rs

@@ -98,12 +98,12 @@ static mut API: &'static [(&'static str, *const ())] = &[
     api!(cache_put = ::cache_put),
 
     /* direct syscalls */
-    api!(rtio_init),
+    api!(rtio_init = ::rtio::init),
-    api!(rtio_get_counter),
+    api!(rtio_get_counter = ::rtio::get_counter),
     api!(rtio_log),
-    api!(rtio_output),
+    api!(rtio_output = ::rtio::output),
-    api!(rtio_input_timestamp),
+    api!(rtio_input_timestamp = ::rtio::input_timestamp),
-    api!(rtio_input_data),
+    api!(rtio_input_data = ::rtio::input_data),
 
     api!(i2c_init),
     api!(i2c_start),

artiq/runtime.rs/libksupport/lib.rs

@@ -23,6 +23,34 @@ mod rpc_proto;
 mod dyld;
 mod api;
 
+#[allow(improper_ctypes)]
+extern {
+    fn __artiq_raise(exn: *const ::kernel_proto::Exception) -> !;
+}
+
+macro_rules! artiq_raise {
+    ($name:expr, $message:expr, $param0:expr, $param1:expr, $param2:expr) => ({
+        let exn = $crate::kernel_proto::Exception {
+            name:     concat!("0:artiq.coredevice.exceptions.", $name, "\0").as_bytes().as_ptr(),
+            file:     concat!(file!(), "\0").as_bytes().as_ptr(),
+            line:     line!(),
+            column:   column!(),
+            // https://github.com/rust-lang/rfcs/pull/1719
+            function: "(Rust function)\0".as_bytes().as_ptr(),
+            message:  concat!($message, "\0").as_bytes().as_ptr(),
+            param:    [$param0, $param1, $param2],
+            phantom:  ::core::marker::PhantomData
+        };
+        #[allow(unused_unsafe)]
+        unsafe { $crate::__artiq_raise(&exn as *const _) }
+    });
+    ($name:expr, $message:expr) => ({
+        artiq_raise!($name, $message, 0, 0, 0)
+    });
+}
+
+mod rtio;
+
 use core::{mem, ptr, slice, str};
 use std::io::Cursor;
 use libc::{c_char, size_t};
@@ -91,12 +119,17 @@ extern fn panic_fmt(args: core::fmt::Arguments, file: &'static str, line: u32) -
 static mut NOW: u64 = 0;
 
 #[no_mangle]
-pub extern fn send_to_log(ptr: *const u8, len: usize) {
+pub extern fn send_to_core_log(ptr: *const u8, len: usize) {
     send(&LogSlice(unsafe {
         str::from_utf8_unchecked(slice::from_raw_parts(ptr, len))
     }))
 }
 
+#[no_mangle]
+pub extern fn send_to_rtio_log(timestamp: i64, ptr: *const u8, len: usize) {
+    rtio::log(timestamp, unsafe { slice::from_raw_parts(ptr, len) })
+}
+
 extern fn abort() -> ! {
     println!("kernel called abort()");
     send(&RunAborted);
@@ -151,28 +184,6 @@ extern fn recv_rpc(slot: *mut ()) -> usize {
     })
 }
 
-#[allow(improper_ctypes)]
-extern {
-    fn __artiq_raise(exn: *const ::kernel_proto::Exception) -> !;
-}
-
-macro_rules! artiq_raise {
-    ($name:expr, $message:expr) => ({
-        let exn = Exception {
-            name:     concat!("0:artiq.coredevice.exceptions.", $name, "\0").as_bytes().as_ptr(),
-            file:     concat!(file!(), "\0").as_bytes().as_ptr(),
-            line:     line!(),
-            column:   column!(),
-            // https://github.com/rust-lang/rfcs/pull/1719
-            function: "(Rust function)\0".as_bytes().as_ptr(),
-            message:  concat!($message, "\0").as_bytes().as_ptr(),
-            param:    [0; 3],
-            phantom:  ::core::marker::PhantomData
-        };
-        unsafe { __artiq_raise(&exn as *const _) }
-    })
-}
-
 #[no_mangle]
 pub extern fn __artiq_terminate(exception: *const kernel_proto::Exception,
                             backtrace_data: *mut usize,

artiq/runtime.rs/libksupport/rtio.rs

@@ -0,0 +1,148 @@
+use board::csr;
+
+const RTIO_O_STATUS_FULL:           u32 = 1;
+const RTIO_O_STATUS_UNDERFLOW:      u32 = 2;
+const RTIO_O_STATUS_SEQUENCE_ERROR: u32 = 4;
+const RTIO_O_STATUS_COLLISION:      u32 = 8;
+const RTIO_O_STATUS_BUSY:           u32 = 16;
+const RTIO_I_STATUS_EMPTY:          u32 = 1;
+const RTIO_I_STATUS_OVERFLOW:       u32 = 2;
+
+pub extern fn init() {
+    unsafe {
+        csr::rtio::reset_write(1);
+        csr::rtio::reset_write(0);
+        csr::rtio::reset_phy_write(0);
+    }
+}
+
+pub extern fn get_counter() -> i64 {
+    unsafe {
+        csr::rtio::counter_update_write(1);
+        csr::rtio::counter_read() as i64
+    }
+}
+
+#[inline(never)]
+unsafe fn process_exceptional_status(timestamp: i64, channel: u32, status: u32) {
+    if status & RTIO_O_STATUS_FULL != 0 {
+        while csr::rtio::o_status_read() & RTIO_O_STATUS_FULL != 0 {}
+    }
+    if status & RTIO_O_STATUS_UNDERFLOW != 0 {
+        csr::rtio::o_underflow_reset_write(1);
+        artiq_raise!("RTIOUnderflow",
+            "RTIO underflow at {0} mu, channel {1}, slack {2} mu",
+            timestamp, channel as i64, timestamp - get_counter())
+    }
+    if status & RTIO_O_STATUS_SEQUENCE_ERROR != 0 {
+        csr::rtio::o_sequence_error_reset_write(1);
+        artiq_raise!("RTIOSequenceError",
+            "RTIO sequence error at {0} mu, channel {1}",
+            timestamp, channel as i64, 0)
+    }
+    if status & RTIO_O_STATUS_COLLISION != 0 {
+        csr::rtio::o_collision_reset_write(1);
+        artiq_raise!("RTIOCollision",
+            "RTIO collision at {0} mu, channel {1}",
+            timestamp, channel as i64, 0)
+    }
+    if status & RTIO_O_STATUS_BUSY != 0 {
+        csr::rtio::o_busy_reset_write(1);
+        artiq_raise!("RTIOBusy",
+            "RTIO busy on channel {0}",
+            channel as i64, 0, 0)
+    }
+}
+
+pub extern fn output(timestamp: i64, channel: u32, addr: u32, data: u32) {
+    unsafe {
+        csr::rtio::chan_sel_write(channel);
+        csr::rtio::o_timestamp_write(timestamp as u64);
+        csr::rtio::o_address_write(addr);
+        csr::rtio::o_data_write(data);
+        csr::rtio::o_we_write(1);
+        let status = csr::rtio::o_status_read();
+        if status != 0 {
+            process_exceptional_status(timestamp, channel, status);
+        }
+    }
+}
+
+pub extern fn input_timestamp(timeout: i64, channel: u32) -> u64 {
+    unsafe {
+        csr::rtio::chan_sel_write(channel);
+        let mut status;
+        loop {
+            status = csr::rtio::i_status_read();
+            if status == 0 { break }
+
+            if status & RTIO_I_STATUS_OVERFLOW != 0 {
+                csr::rtio::i_overflow_reset_write(1);
+                break
+            }
+            if get_counter() >= timeout {
+                // check empty flag again to prevent race condition.
+                // now we are sure that the time limit has been exceeded.
+                let status = csr::rtio::i_status_read();
+                if status & RTIO_I_STATUS_EMPTY != 0 { break }
+            }
+            // input FIFO is empty - keep waiting
+        }
+
+        if status & RTIO_I_STATUS_OVERFLOW != 0 {
+            artiq_raise!("RTIOOverflow",
+                "RTIO input overflow on channel {0}",
+                channel as i64, 0, 0);
+        }
+        if status & RTIO_I_STATUS_EMPTY != 0 {
+            return !0
+        }
+
+        let timestamp = csr::rtio::i_timestamp_read();
+        csr::rtio::i_re_write(1);
+        timestamp
+    }
+}
+
+pub extern fn input_data(channel: u32) -> u32 {
+    unsafe {
+        csr::rtio::chan_sel_write(channel);
+        loop {
+            let status = csr::rtio::i_status_read();
+            if status == 0 { break }
+
+            if status & RTIO_I_STATUS_OVERFLOW != 0 {
+                csr::rtio::i_overflow_reset_write(1);
+                artiq_raise!("RTIOOverflow",
+                    "RTIO input overflow on channel {0}",
+                    channel as i64, 0, 0);
+            }
+        }
+
+        let data = csr::rtio::i_data_read();
+        csr::rtio::i_re_write(1);
+        data
+    }
+}
+
+pub fn log(timestamp: i64, data: &[u8]) {
+    unsafe {
+        csr::rtio::chan_sel_write(csr::CONFIG_RTIO_LOG_CHANNEL);
+        csr::rtio::o_timestamp_write(timestamp as u64);
+
+        let mut word: u32 = 0;
+        for i in 0..data.len() {
+            word <<= 8;
+            word |= data[i] as u32;
+            if i % 4 == 0 {
+                csr::rtio::o_data_write(word);
+                csr::rtio::o_we_write(1);
+                word = 0;
+            }
+        }
+
+        word <<= 8;
+        csr::rtio::o_data_write(word);
+        csr::rtio::o_we_write(1);
+    }
+}

artiq/runtime.rs/src/kernel_proto.rs

@@ -17,7 +17,7 @@ pub struct Exception<'a> {
     pub column:   u32,
     pub function: *const u8,
     pub message:  *const u8,
-    pub param:    [u64; 3],
+    pub param:    [i64; 3],
     pub phantom:  PhantomData<&'a str>
 }
 

artiq/runtime.rs/src/session_proto.rs

@@ -30,7 +30,7 @@ pub enum Request {
     RpcException {
         name:     String,
         message:  String,
-        param:    [u64; 3],
+        param:    [i64; 3],
         file:     String,
         line:     u32,
         column:   u32,
@@ -59,9 +59,9 @@ impl Request {
             8  => Request::RpcException {
                 name:     try!(read_string(reader)),
                 message:  try!(read_string(reader)),
-                param:    [try!(read_u64(reader)),
+                param:    [try!(read_u64(reader).map(|x| x as i64)),
-                           try!(read_u64(reader)),
+                           try!(read_u64(reader).map(|x| x as i64)),
-                           try!(read_u64(reader))],
+                           try!(read_u64(reader).map(|x| x as i64))],
                 file:     try!(read_string(reader)),
                 line:     try!(read_u32(reader)),
                 column:   try!(read_u32(reader)),
@@ -99,7 +99,7 @@ pub enum Reply<'a> {
     KernelException {
         name:      &'a str,
         message:   &'a str,
-        param:     [u64; 3],
+        param:     [i64; 3],
         file:      &'a str,
         line:      u32,
         column:    u32,
@@ -157,9 +157,9 @@ impl<'a> Reply<'a> {
                 try!(write_u8(writer, 9));
                 try!(write_string(writer, name));
                 try!(write_string(writer, message));
-                try!(write_u64(writer, param[0]));
+                try!(write_u64(writer, param[0] as u64));
-                try!(write_u64(writer, param[1]));
+                try!(write_u64(writer, param[1] as u64));
-                try!(write_u64(writer, param[2]));
+                try!(write_u64(writer, param[2] as u64));
                 try!(write_string(writer, file));
                 try!(write_u32(writer, line));
                 try!(write_u32(writer, column));

artiq/runtime/Makefile

@@ -4,7 +4,7 @@ include $(MISOC_DIRECTORY)/software/common.mak
 PYTHON ?= python3.5
 
 OBJECTS := flash_storage.o main.o
-OBJECTS_KSUPPORT := ksupport_glue.o artiq_personality.o rtio.o i2c.o
+OBJECTS_KSUPPORT := ksupport_glue.o artiq_personality.o i2c.o
 
 RUSTOUT_DIRECTORY := cargo/or1k-unknown-none/debug
 CORE_IO_COMMIT := d40c593f42fafbac1ff3d827f6df96338b5b7d8b

artiq/runtime/ksupport_glue.c

@@ -6,7 +6,8 @@
 #include <link.h>
 #include <dlfcn.h>
 
-void send_to_log(const char *ptr, size_t length);
+void send_to_core_log(const char *ptr, size_t length);
+void send_to_rtio_log(long long int timestamp, const char *ptr, size_t length);
 
 #define KERNELCPU_EXEC_ADDRESS    0x40800000
 #define KERNELCPU_PAYLOAD_ADDRESS 0x40840000
@@ -16,20 +17,18 @@ void send_to_log(const char *ptr, size_t length);
 /* called by libunwind */
 int fprintf(FILE *stream, const char *fmt, ...)
 {
-    size_t size;
-    char *buf;
     va_list args;
 
     va_start(args, fmt);
-    size = vsnprintf(NULL, 0, fmt, args);
+    size_t size = vsnprintf(NULL, 0, fmt, args);
-    buf = __builtin_alloca(size + 1);
+    char *buf = __builtin_alloca(size + 1);
     va_end(args);
 
     va_start(args, fmt);
     vsnprintf(buf, size + 1, fmt, args);
     va_end(args);
 
-    send_to_log(buf, size);
+    send_to_core_log(buf, size);
     return 0;
 }
 
@@ -103,19 +102,35 @@ double round(double x)
 int core_log(const char *fmt, ...);
 int core_log(const char *fmt, ...)
 {
-    size_t size;
-    char *buf;
     va_list args;
 
     va_start(args, fmt);
-    size = vsnprintf(NULL, 0, fmt, args);
+    size_t size = vsnprintf(NULL, 0, fmt, args);
-    buf = __builtin_alloca(size + 1);
+    char *buf = __builtin_alloca(size + 1);
     va_end(args);
 
     va_start(args, fmt);
     vsnprintf(buf, size + 1, fmt, args);
     va_end(args);
 
-    send_to_log(buf, size);
+    send_to_core_log(buf, size);
     return 0;
 }
+
+/* called by kernel */
+void rtio_log(long long int timestamp, const char *fmt, ...);
+void rtio_log(long long int timestamp, const char *fmt, ...)
+{
+    va_list args;
+
+    va_start(args, fmt);
+    size_t size = vsnprintf(NULL, 0, fmt, args);
+    char *buf = __builtin_alloca(size + 1);
+    va_end(args);
+
+    va_start(args, fmt);
+    vsnprintf(buf, size + 1, fmt, args);
+    va_end(args);
+
+    send_to_rtio_log(timestamp, buf, size);
+}

artiq/runtime/rtio.c

@@ -1,166 +0,0 @@
-#include <generated/csr.h>
-
-#include "artiq_personality.h"
-#include "rtio.h"
-
-void rtio_init(void)
-{
-    rtio_reset_write(1);
-    rtio_reset_write(0);
-    rtio_reset_phy_write(0);
-}
-
-long long int rtio_get_counter(void)
-{
-    rtio_counter_update_write(1);
-    return rtio_counter_read();
-}
-
-static void rtio_process_exceptional_status(
-        long long int timestamp, int channel, int status)
-{
-    if(status & RTIO_O_STATUS_FULL)
-        while(rtio_o_status_read() & RTIO_O_STATUS_FULL);
-    if(status & RTIO_O_STATUS_UNDERFLOW) {
-        rtio_o_underflow_reset_write(1);
-        artiq_raise_from_c("RTIOUnderflow",
-            "RTIO underflow at {0} mu, channel {1}, slack {2} mu",
-            timestamp, channel, timestamp - rtio_get_counter());
-    }
-    if(status & RTIO_O_STATUS_SEQUENCE_ERROR) {
-        rtio_o_sequence_error_reset_write(1);
-        artiq_raise_from_c("RTIOSequenceError",
-            "RTIO sequence error at {0} mu, channel {1}",
-            timestamp, channel, 0);
-    }
-    if(status & RTIO_O_STATUS_COLLISION) {
-        rtio_o_collision_reset_write(1);
-        artiq_raise_from_c("RTIOCollision",
-            "RTIO collision at {0} mu, channel {1}",
-            timestamp, channel, 0);
-    }
-    if(status & RTIO_O_STATUS_BUSY) {
-        rtio_o_busy_reset_write(1);
-        artiq_raise_from_c("RTIOBusy",
-            "RTIO busy on channel {0}",
-            channel, 0, 0);
-    }
-}
-
-
-void rtio_output(long long int timestamp, int channel, unsigned int addr,
-        unsigned int data)
-{
-    int status;
-
-    rtio_chan_sel_write(channel);
-    rtio_o_timestamp_write(timestamp);
-#ifdef CSR_RTIO_O_ADDRESS_ADDR
-    rtio_o_address_write(addr);
-#endif
-    rtio_o_data_write(data);
-    rtio_o_we_write(1);
-    status = rtio_o_status_read();
-    if(status)
-        rtio_process_exceptional_status(timestamp, channel, status);
-}
-
-
-long long int rtio_input_timestamp(long long int timeout, int channel)
-{
-    long long int r;
-    int status;
-
-    rtio_chan_sel_write(channel);
-    while((status = rtio_i_status_read())) {
-        if(status & RTIO_I_STATUS_OVERFLOW) {
-            rtio_i_overflow_reset_write(1);
-            break;
-        }
-        if(rtio_get_counter() >= timeout) {
-            /* check empty flag again to prevent race condition.
-             * now we are sure that the time limit has been exceeded.
-             */
-            status = rtio_i_status_read();
-            if(status & RTIO_I_STATUS_EMPTY)
-                break;
-        }
-        /* input FIFO is empty - keep waiting */
-    }
-
-    if (status & RTIO_I_STATUS_OVERFLOW)
-        artiq_raise_from_c("RTIOOverflow",
-                "RTIO input overflow on channel {0}",
-                channel, 0, 0);
-    if (status & RTIO_I_STATUS_EMPTY)
-        return -1;
-
-    r = rtio_i_timestamp_read();
-    rtio_i_re_write(1);
-    return r;
-}
-
-
-unsigned int rtio_input_data(int channel)
-{
-    unsigned int data;
-    int status;
-
-    rtio_chan_sel_write(channel);
-    while((status = rtio_i_status_read())) {
-        if(status & RTIO_I_STATUS_OVERFLOW) {
-            rtio_i_overflow_reset_write(1);
-            artiq_raise_from_c("RTIOOverflow",
-                    "RTIO input overflow on channel {0}",
-                    channel, 0, 0);
-        }
-    }
-
-    data = rtio_i_data_read();
-    rtio_i_re_write(1);
-    return data;
-}
-
-
-void rtio_log_va(long long int timestamp, const char *fmt, va_list args)
-{
-#ifdef CONFIG_RTIO_LOG_CHANNEL
-    // This executes on the kernel CPU's stack, which is specifically designed
-    // for allocation of this kind of massive buffers.
-    int   len = vsnprintf(NULL, 0, fmt, args);
-    char *buf = __builtin_alloca(len + 1);
-    vsnprintf(buf, len + 1, fmt, args);
-
-    rtio_chan_sel_write(CONFIG_RTIO_LOG_CHANNEL);
-    rtio_o_timestamp_write(timestamp);
-
-    int i = 0;
-    unsigned int word = 0;
-    while(1) {
-        word <<= 8;
-        word |= *buf & 0xff;
-        if(*buf == 0) {
-            rtio_o_data_write(word);
-            rtio_o_we_write(1);
-            break;
-        }
-        buf++;
-        i++;
-        if(i == 4) {
-            rtio_o_data_write(word);
-            rtio_o_we_write(1);
-            word = 0;
-            i = 0;
-        }
-    }
-#endif
-}
-
-void rtio_log(long long int timestamp, const char *fmt, ...)
-{
-    va_list args;
-
-    va_start(args, fmt);
-    rtio_log_va(timestamp, fmt, args);
-    va_end(args);
-}

artiq/runtime/rtio.h

@@ -1,33 +0,0 @@
-#ifndef __RTIO_H
-#define __RTIO_H
-
-#include <stdarg.h>
-
-#define RTIO_O_STATUS_FULL 1
-#define RTIO_O_STATUS_UNDERFLOW 2
-#define RTIO_O_STATUS_SEQUENCE_ERROR 4
-#define RTIO_O_STATUS_COLLISION 8
-#define RTIO_O_STATUS_BUSY 16
-#define RTIO_I_STATUS_EMPTY 1
-#define RTIO_I_STATUS_OVERFLOW 2
-
-void rtio_init(void);
-long long int rtio_get_counter(void);
-void rtio_log(long long int timestamp, const char *format, ...);
-void rtio_log_va(long long int timestamp, const char *format, va_list args);
-void rtio_output(long long int timestamp, int channel, unsigned int address,
-        unsigned int data);
-
-/*
- * Waits at least until timeout and returns the timestamp of the first
- * input event on the chanel, -1 if there was no event.
- */
-long long int rtio_input_timestamp(long long int timeout, int channel);
-
-/*
- * Assumes that there is or will be an event in the channel and returns only
- * its data.
- */
-unsigned int rtio_input_data(int channel);
-
-#endif /* __RTIO_H */