diff --git a/artiq/coredevice/rtio.py b/artiq/coredevice/rtio.py index f4d0e2c82..2f564dc4b 100644 --- a/artiq/coredevice/rtio.py +++ b/artiq/coredevice/rtio.py @@ -3,8 +3,7 @@ from artiq.language.types import TInt64, TInt32, TNone @syscall(flags={"nowrite"}) -def rtio_output(time_mu: TInt64, channel: TInt32, addr: TInt32, data: TInt32 - ) -> TNone: +def rtio_output(time_mu: TInt64, channel: TInt32, addr: TInt32, data: TInt32) -> TNone: raise NotImplementedError("syscall not simulated") diff --git a/artiq/runtime.rs/libksupport/api.rs b/artiq/runtime.rs/libksupport/api.rs index 82448c744..fd069d8c4 100644 --- a/artiq/runtime.rs/libksupport/api.rs +++ b/artiq/runtime.rs/libksupport/api.rs @@ -98,12 +98,12 @@ static mut API: &'static [(&'static str, *const ())] = &[ api!(cache_put = ::cache_put), /* direct syscalls */ - api!(rtio_init), - api!(rtio_get_counter), + api!(rtio_init = ::rtio::init), + api!(rtio_get_counter = ::rtio::get_counter), api!(rtio_log), - api!(rtio_output), - api!(rtio_input_timestamp), - api!(rtio_input_data), + api!(rtio_output = ::rtio::output), + api!(rtio_input_timestamp = ::rtio::input_timestamp), + api!(rtio_input_data = ::rtio::input_data), api!(i2c_init), api!(i2c_start), diff --git a/artiq/runtime.rs/libksupport/lib.rs b/artiq/runtime.rs/libksupport/lib.rs index 7e4d66b59..f46d5efaf 100644 --- a/artiq/runtime.rs/libksupport/lib.rs +++ b/artiq/runtime.rs/libksupport/lib.rs @@ -23,6 +23,34 @@ mod rpc_proto; mod dyld; mod api; +#[allow(improper_ctypes)] +extern { + fn __artiq_raise(exn: *const ::kernel_proto::Exception) -> !; +} + +macro_rules! artiq_raise { + ($name:expr, $message:expr, $param0:expr, $param1:expr, $param2:expr) => ({ + let exn = $crate::kernel_proto::Exception { + name: concat!("0:artiq.coredevice.exceptions.", $name, "\0").as_bytes().as_ptr(), + file: concat!(file!(), "\0").as_bytes().as_ptr(), + line: line!(), + column: column!(), + // https://github.com/rust-lang/rfcs/pull/1719 + function: "(Rust function)\0".as_bytes().as_ptr(), + message: concat!($message, "\0").as_bytes().as_ptr(), + param: [$param0, $param1, $param2], + phantom: ::core::marker::PhantomData + }; + #[allow(unused_unsafe)] + unsafe { $crate::__artiq_raise(&exn as *const _) } + }); + ($name:expr, $message:expr) => ({ + artiq_raise!($name, $message, 0, 0, 0) + }); +} + +mod rtio; + use core::{mem, ptr, slice, str}; use std::io::Cursor; use libc::{c_char, size_t}; @@ -91,12 +119,17 @@ extern fn panic_fmt(args: core::fmt::Arguments, file: &'static str, line: u32) - static mut NOW: u64 = 0; #[no_mangle] -pub extern fn send_to_log(ptr: *const u8, len: usize) { +pub extern fn send_to_core_log(ptr: *const u8, len: usize) { send(&LogSlice(unsafe { str::from_utf8_unchecked(slice::from_raw_parts(ptr, len)) })) } +#[no_mangle] +pub extern fn send_to_rtio_log(timestamp: i64, ptr: *const u8, len: usize) { + rtio::log(timestamp, unsafe { slice::from_raw_parts(ptr, len) }) +} + extern fn abort() -> ! { println!("kernel called abort()"); send(&RunAborted); @@ -151,28 +184,6 @@ extern fn recv_rpc(slot: *mut ()) -> usize { }) } -#[allow(improper_ctypes)] -extern { - fn __artiq_raise(exn: *const ::kernel_proto::Exception) -> !; -} - -macro_rules! artiq_raise { - ($name:expr, $message:expr) => ({ - let exn = Exception { - name: concat!("0:artiq.coredevice.exceptions.", $name, "\0").as_bytes().as_ptr(), - file: concat!(file!(), "\0").as_bytes().as_ptr(), - line: line!(), - column: column!(), - // https://github.com/rust-lang/rfcs/pull/1719 - function: "(Rust function)\0".as_bytes().as_ptr(), - message: concat!($message, "\0").as_bytes().as_ptr(), - param: [0; 3], - phantom: ::core::marker::PhantomData - }; - unsafe { __artiq_raise(&exn as *const _) } - }) -} - #[no_mangle] pub extern fn __artiq_terminate(exception: *const kernel_proto::Exception, backtrace_data: *mut usize, diff --git a/artiq/runtime.rs/libksupport/rtio.rs b/artiq/runtime.rs/libksupport/rtio.rs new file mode 100644 index 000000000..107583da4 --- /dev/null +++ b/artiq/runtime.rs/libksupport/rtio.rs @@ -0,0 +1,148 @@ +use board::csr; + +const RTIO_O_STATUS_FULL: u32 = 1; +const RTIO_O_STATUS_UNDERFLOW: u32 = 2; +const RTIO_O_STATUS_SEQUENCE_ERROR: u32 = 4; +const RTIO_O_STATUS_COLLISION: u32 = 8; +const RTIO_O_STATUS_BUSY: u32 = 16; +const RTIO_I_STATUS_EMPTY: u32 = 1; +const RTIO_I_STATUS_OVERFLOW: u32 = 2; + +pub extern fn init() { + unsafe { + csr::rtio::reset_write(1); + csr::rtio::reset_write(0); + csr::rtio::reset_phy_write(0); + } +} + +pub extern fn get_counter() -> i64 { + unsafe { + csr::rtio::counter_update_write(1); + csr::rtio::counter_read() as i64 + } +} + +#[inline(never)] +unsafe fn process_exceptional_status(timestamp: i64, channel: u32, status: u32) { + if status & RTIO_O_STATUS_FULL != 0 { + while csr::rtio::o_status_read() & RTIO_O_STATUS_FULL != 0 {} + } + if status & RTIO_O_STATUS_UNDERFLOW != 0 { + csr::rtio::o_underflow_reset_write(1); + artiq_raise!("RTIOUnderflow", + "RTIO underflow at {0} mu, channel {1}, slack {2} mu", + timestamp, channel as i64, timestamp - get_counter()) + } + if status & RTIO_O_STATUS_SEQUENCE_ERROR != 0 { + csr::rtio::o_sequence_error_reset_write(1); + artiq_raise!("RTIOSequenceError", + "RTIO sequence error at {0} mu, channel {1}", + timestamp, channel as i64, 0) + } + if status & RTIO_O_STATUS_COLLISION != 0 { + csr::rtio::o_collision_reset_write(1); + artiq_raise!("RTIOCollision", + "RTIO collision at {0} mu, channel {1}", + timestamp, channel as i64, 0) + } + if status & RTIO_O_STATUS_BUSY != 0 { + csr::rtio::o_busy_reset_write(1); + artiq_raise!("RTIOBusy", + "RTIO busy on channel {0}", + channel as i64, 0, 0) + } +} + +pub extern fn output(timestamp: i64, channel: u32, addr: u32, data: u32) { + unsafe { + csr::rtio::chan_sel_write(channel); + csr::rtio::o_timestamp_write(timestamp as u64); + csr::rtio::o_address_write(addr); + csr::rtio::o_data_write(data); + csr::rtio::o_we_write(1); + let status = csr::rtio::o_status_read(); + if status != 0 { + process_exceptional_status(timestamp, channel, status); + } + } +} + +pub extern fn input_timestamp(timeout: i64, channel: u32) -> u64 { + unsafe { + csr::rtio::chan_sel_write(channel); + let mut status; + loop { + status = csr::rtio::i_status_read(); + if status == 0 { break } + + if status & RTIO_I_STATUS_OVERFLOW != 0 { + csr::rtio::i_overflow_reset_write(1); + break + } + if get_counter() >= timeout { + // check empty flag again to prevent race condition. + // now we are sure that the time limit has been exceeded. + let status = csr::rtio::i_status_read(); + if status & RTIO_I_STATUS_EMPTY != 0 { break } + } + // input FIFO is empty - keep waiting + } + + if status & RTIO_I_STATUS_OVERFLOW != 0 { + artiq_raise!("RTIOOverflow", + "RTIO input overflow on channel {0}", + channel as i64, 0, 0); + } + if status & RTIO_I_STATUS_EMPTY != 0 { + return !0 + } + + let timestamp = csr::rtio::i_timestamp_read(); + csr::rtio::i_re_write(1); + timestamp + } +} + +pub extern fn input_data(channel: u32) -> u32 { + unsafe { + csr::rtio::chan_sel_write(channel); + loop { + let status = csr::rtio::i_status_read(); + if status == 0 { break } + + if status & RTIO_I_STATUS_OVERFLOW != 0 { + csr::rtio::i_overflow_reset_write(1); + artiq_raise!("RTIOOverflow", + "RTIO input overflow on channel {0}", + channel as i64, 0, 0); + } + } + + let data = csr::rtio::i_data_read(); + csr::rtio::i_re_write(1); + data + } +} + +pub fn log(timestamp: i64, data: &[u8]) { + unsafe { + csr::rtio::chan_sel_write(csr::CONFIG_RTIO_LOG_CHANNEL); + csr::rtio::o_timestamp_write(timestamp as u64); + + let mut word: u32 = 0; + for i in 0..data.len() { + word <<= 8; + word |= data[i] as u32; + if i % 4 == 0 { + csr::rtio::o_data_write(word); + csr::rtio::o_we_write(1); + word = 0; + } + } + + word <<= 8; + csr::rtio::o_data_write(word); + csr::rtio::o_we_write(1); + } +} diff --git a/artiq/runtime.rs/src/kernel_proto.rs b/artiq/runtime.rs/src/kernel_proto.rs index afaf7c35e..c015b76d2 100644 --- a/artiq/runtime.rs/src/kernel_proto.rs +++ b/artiq/runtime.rs/src/kernel_proto.rs @@ -17,7 +17,7 @@ pub struct Exception<'a> { pub column: u32, pub function: *const u8, pub message: *const u8, - pub param: [u64; 3], + pub param: [i64; 3], pub phantom: PhantomData<&'a str> } diff --git a/artiq/runtime.rs/src/session_proto.rs b/artiq/runtime.rs/src/session_proto.rs index fd48c46b3..e83b955f7 100644 --- a/artiq/runtime.rs/src/session_proto.rs +++ b/artiq/runtime.rs/src/session_proto.rs @@ -30,7 +30,7 @@ pub enum Request { RpcException { name: String, message: String, - param: [u64; 3], + param: [i64; 3], file: String, line: u32, column: u32, @@ -59,9 +59,9 @@ impl Request { 8 => Request::RpcException { name: try!(read_string(reader)), message: try!(read_string(reader)), - param: [try!(read_u64(reader)), - try!(read_u64(reader)), - try!(read_u64(reader))], + param: [try!(read_u64(reader).map(|x| x as i64)), + try!(read_u64(reader).map(|x| x as i64)), + try!(read_u64(reader).map(|x| x as i64))], file: try!(read_string(reader)), line: try!(read_u32(reader)), column: try!(read_u32(reader)), @@ -99,7 +99,7 @@ pub enum Reply<'a> { KernelException { name: &'a str, message: &'a str, - param: [u64; 3], + param: [i64; 3], file: &'a str, line: u32, column: u32, @@ -157,9 +157,9 @@ impl<'a> Reply<'a> { try!(write_u8(writer, 9)); try!(write_string(writer, name)); try!(write_string(writer, message)); - try!(write_u64(writer, param[0])); - try!(write_u64(writer, param[1])); - try!(write_u64(writer, param[2])); + try!(write_u64(writer, param[0] as u64)); + try!(write_u64(writer, param[1] as u64)); + try!(write_u64(writer, param[2] as u64)); try!(write_string(writer, file)); try!(write_u32(writer, line)); try!(write_u32(writer, column)); diff --git a/artiq/runtime/Makefile b/artiq/runtime/Makefile index 797123c2c..36685661c 100644 --- a/artiq/runtime/Makefile +++ b/artiq/runtime/Makefile @@ -4,7 +4,7 @@ include $(MISOC_DIRECTORY)/software/common.mak PYTHON ?= python3.5 OBJECTS := flash_storage.o main.o -OBJECTS_KSUPPORT := ksupport_glue.o artiq_personality.o rtio.o i2c.o +OBJECTS_KSUPPORT := ksupport_glue.o artiq_personality.o i2c.o RUSTOUT_DIRECTORY := cargo/or1k-unknown-none/debug CORE_IO_COMMIT := d40c593f42fafbac1ff3d827f6df96338b5b7d8b diff --git a/artiq/runtime/ksupport_glue.c b/artiq/runtime/ksupport_glue.c index f0d496dc3..6525b4314 100644 --- a/artiq/runtime/ksupport_glue.c +++ b/artiq/runtime/ksupport_glue.c @@ -6,7 +6,8 @@ #include #include -void send_to_log(const char *ptr, size_t length); +void send_to_core_log(const char *ptr, size_t length); +void send_to_rtio_log(long long int timestamp, const char *ptr, size_t length); #define KERNELCPU_EXEC_ADDRESS 0x40800000 #define KERNELCPU_PAYLOAD_ADDRESS 0x40840000 @@ -16,20 +17,18 @@ void send_to_log(const char *ptr, size_t length); /* called by libunwind */ int fprintf(FILE *stream, const char *fmt, ...) { - size_t size; - char *buf; va_list args; va_start(args, fmt); - size = vsnprintf(NULL, 0, fmt, args); - buf = __builtin_alloca(size + 1); + size_t size = vsnprintf(NULL, 0, fmt, args); + char *buf = __builtin_alloca(size + 1); va_end(args); va_start(args, fmt); vsnprintf(buf, size + 1, fmt, args); va_end(args); - send_to_log(buf, size); + send_to_core_log(buf, size); return 0; } @@ -103,19 +102,35 @@ double round(double x) int core_log(const char *fmt, ...); int core_log(const char *fmt, ...) { - size_t size; - char *buf; va_list args; va_start(args, fmt); - size = vsnprintf(NULL, 0, fmt, args); - buf = __builtin_alloca(size + 1); + size_t size = vsnprintf(NULL, 0, fmt, args); + char *buf = __builtin_alloca(size + 1); va_end(args); va_start(args, fmt); vsnprintf(buf, size + 1, fmt, args); va_end(args); - send_to_log(buf, size); + send_to_core_log(buf, size); return 0; } + +/* called by kernel */ +void rtio_log(long long int timestamp, const char *fmt, ...); +void rtio_log(long long int timestamp, const char *fmt, ...) +{ + va_list args; + + va_start(args, fmt); + size_t size = vsnprintf(NULL, 0, fmt, args); + char *buf = __builtin_alloca(size + 1); + va_end(args); + + va_start(args, fmt); + vsnprintf(buf, size + 1, fmt, args); + va_end(args); + + send_to_rtio_log(timestamp, buf, size); +} diff --git a/artiq/runtime/rtio.c b/artiq/runtime/rtio.c deleted file mode 100644 index ac5168224..000000000 --- a/artiq/runtime/rtio.c +++ /dev/null @@ -1,166 +0,0 @@ -#include - -#include "artiq_personality.h" -#include "rtio.h" - -void rtio_init(void) -{ - rtio_reset_write(1); - rtio_reset_write(0); - rtio_reset_phy_write(0); -} - -long long int rtio_get_counter(void) -{ - rtio_counter_update_write(1); - return rtio_counter_read(); -} - -static void rtio_process_exceptional_status( - long long int timestamp, int channel, int status) -{ - if(status & RTIO_O_STATUS_FULL) - while(rtio_o_status_read() & RTIO_O_STATUS_FULL); - if(status & RTIO_O_STATUS_UNDERFLOW) { - rtio_o_underflow_reset_write(1); - artiq_raise_from_c("RTIOUnderflow", - "RTIO underflow at {0} mu, channel {1}, slack {2} mu", - timestamp, channel, timestamp - rtio_get_counter()); - } - if(status & RTIO_O_STATUS_SEQUENCE_ERROR) { - rtio_o_sequence_error_reset_write(1); - artiq_raise_from_c("RTIOSequenceError", - "RTIO sequence error at {0} mu, channel {1}", - timestamp, channel, 0); - } - if(status & RTIO_O_STATUS_COLLISION) { - rtio_o_collision_reset_write(1); - artiq_raise_from_c("RTIOCollision", - "RTIO collision at {0} mu, channel {1}", - timestamp, channel, 0); - } - if(status & RTIO_O_STATUS_BUSY) { - rtio_o_busy_reset_write(1); - artiq_raise_from_c("RTIOBusy", - "RTIO busy on channel {0}", - channel, 0, 0); - } -} - - -void rtio_output(long long int timestamp, int channel, unsigned int addr, - unsigned int data) -{ - int status; - - rtio_chan_sel_write(channel); - rtio_o_timestamp_write(timestamp); -#ifdef CSR_RTIO_O_ADDRESS_ADDR - rtio_o_address_write(addr); -#endif - rtio_o_data_write(data); - rtio_o_we_write(1); - status = rtio_o_status_read(); - if(status) - rtio_process_exceptional_status(timestamp, channel, status); -} - - -long long int rtio_input_timestamp(long long int timeout, int channel) -{ - long long int r; - int status; - - rtio_chan_sel_write(channel); - while((status = rtio_i_status_read())) { - if(status & RTIO_I_STATUS_OVERFLOW) { - rtio_i_overflow_reset_write(1); - break; - } - if(rtio_get_counter() >= timeout) { - /* check empty flag again to prevent race condition. - * now we are sure that the time limit has been exceeded. - */ - status = rtio_i_status_read(); - if(status & RTIO_I_STATUS_EMPTY) - break; - } - /* input FIFO is empty - keep waiting */ - } - - if (status & RTIO_I_STATUS_OVERFLOW) - artiq_raise_from_c("RTIOOverflow", - "RTIO input overflow on channel {0}", - channel, 0, 0); - if (status & RTIO_I_STATUS_EMPTY) - return -1; - - r = rtio_i_timestamp_read(); - rtio_i_re_write(1); - return r; -} - - -unsigned int rtio_input_data(int channel) -{ - unsigned int data; - int status; - - rtio_chan_sel_write(channel); - while((status = rtio_i_status_read())) { - if(status & RTIO_I_STATUS_OVERFLOW) { - rtio_i_overflow_reset_write(1); - artiq_raise_from_c("RTIOOverflow", - "RTIO input overflow on channel {0}", - channel, 0, 0); - } - } - - data = rtio_i_data_read(); - rtio_i_re_write(1); - return data; -} - - -void rtio_log_va(long long int timestamp, const char *fmt, va_list args) -{ -#ifdef CONFIG_RTIO_LOG_CHANNEL - // This executes on the kernel CPU's stack, which is specifically designed - // for allocation of this kind of massive buffers. - int len = vsnprintf(NULL, 0, fmt, args); - char *buf = __builtin_alloca(len + 1); - vsnprintf(buf, len + 1, fmt, args); - - rtio_chan_sel_write(CONFIG_RTIO_LOG_CHANNEL); - rtio_o_timestamp_write(timestamp); - - int i = 0; - unsigned int word = 0; - while(1) { - word <<= 8; - word |= *buf & 0xff; - if(*buf == 0) { - rtio_o_data_write(word); - rtio_o_we_write(1); - break; - } - buf++; - i++; - if(i == 4) { - rtio_o_data_write(word); - rtio_o_we_write(1); - word = 0; - i = 0; - } - } -#endif -} - -void rtio_log(long long int timestamp, const char *fmt, ...) -{ - va_list args; - - va_start(args, fmt); - rtio_log_va(timestamp, fmt, args); - va_end(args); -} diff --git a/artiq/runtime/rtio.h b/artiq/runtime/rtio.h deleted file mode 100644 index aad6460ba..000000000 --- a/artiq/runtime/rtio.h +++ /dev/null @@ -1,33 +0,0 @@ -#ifndef __RTIO_H -#define __RTIO_H - -#include - -#define RTIO_O_STATUS_FULL 1 -#define RTIO_O_STATUS_UNDERFLOW 2 -#define RTIO_O_STATUS_SEQUENCE_ERROR 4 -#define RTIO_O_STATUS_COLLISION 8 -#define RTIO_O_STATUS_BUSY 16 -#define RTIO_I_STATUS_EMPTY 1 -#define RTIO_I_STATUS_OVERFLOW 2 - -void rtio_init(void); -long long int rtio_get_counter(void); -void rtio_log(long long int timestamp, const char *format, ...); -void rtio_log_va(long long int timestamp, const char *format, va_list args); -void rtio_output(long long int timestamp, int channel, unsigned int address, - unsigned int data); - -/* - * Waits at least until timeout and returns the timestamp of the first - * input event on the chanel, -1 if there was no event. - */ -long long int rtio_input_timestamp(long long int timeout, int channel); - -/* - * Assumes that there is or will be an event in the channel and returns only - * its data. - */ -unsigned int rtio_input_data(int channel); - -#endif /* __RTIO_H */