diff --git a/artiq/compiler/module.py b/artiq/compiler/module.py index 355e3072d..0e07a5b60 100644 --- a/artiq/compiler/module.py +++ b/artiq/compiler/module.py @@ -81,13 +81,6 @@ class Module: embedding_map=self.embedding_map) return llvm_ir_generator.process(self.artiq_ir, attribute_writeback=True) - def entry_point(self): - """Return the name of the function that is the entry point of this module.""" - if self.name != "": - return self.name + ".__modinit__" - else: - return "__modinit__" - def __repr__(self): printer = types.TypePrinter() globals = ["%s: %s" % (var, printer.name(self.globals[var])) for var in self.globals] diff --git a/artiq/compiler/targets.py b/artiq/compiler/targets.py index dff28bb2e..aab813737 100644 --- a/artiq/compiler/targets.py +++ b/artiq/compiler/targets.py @@ -161,9 +161,9 @@ class Target: return llmachine.emit_object(llmodule) - def link(self, objects, init_fn): + def link(self, objects): """Link the relocatable objects into a shared library for this target.""" - with RunTool([self.triple + "-ld", "-shared", "--eh-frame-hdr", "-init", init_fn] + + with RunTool([self.triple + "-ld", "-shared", "--eh-frame-hdr"] + ["{{obj{}}}".format(index) for index in range(len(objects))] + ["-o", "{output}"], output=b"", @@ -177,8 +177,7 @@ class Target: return library def compile_and_link(self, modules): - return self.link([self.assemble(self.compile(module)) for module in modules], - init_fn=modules[0].entry_point()) + return self.link([self.assemble(self.compile(module)) for module in modules]) def strip(self, library): with RunTool([self.triple + "-strip", "--strip-debug", "{library}", "-o", "{output}"], diff --git a/artiq/compiler/testbench/perf_embedding.py b/artiq/compiler/testbench/perf_embedding.py index 81c0c02db..87d35ee04 100644 --- a/artiq/compiler/testbench/perf_embedding.py +++ b/artiq/compiler/testbench/perf_embedding.py @@ -47,7 +47,7 @@ def main(): target = OR1KTarget() llvm_ir = target.compile(module) elf_obj = target.assemble(llvm_ir) - elf_shlib = target.link([elf_obj], init_fn=module.entry_point()) + elf_shlib = target.link([elf_obj]) benchmark(lambda: embed(), "ARTIQ embedding") @@ -61,7 +61,7 @@ def main(): benchmark(lambda: target.assemble(llvm_ir), "LLVM machine code emission") - benchmark(lambda: target.link([elf_obj], init_fn=module.entry_point()), + benchmark(lambda: target.link([elf_obj]), "Linking") benchmark(lambda: target.strip(elf_shlib), diff --git a/artiq/runtime.rs/libksupport/Cargo.lock b/artiq/runtime.rs/libksupport/Cargo.lock new file mode 100644 index 000000000..042a9a750 --- /dev/null +++ b/artiq/runtime.rs/libksupport/Cargo.lock @@ -0,0 +1,3 @@ +[root] +name = "ksupport" +version = "0.0.0" diff --git a/artiq/runtime.rs/libksupport/Cargo.toml b/artiq/runtime.rs/libksupport/Cargo.toml new file mode 100644 index 000000000..e5a29ff08 --- /dev/null +++ b/artiq/runtime.rs/libksupport/Cargo.toml @@ -0,0 +1,13 @@ +[package] +authors = ["The ARTIQ Project Developers"] +name = "ksupport" +version = "0.0.0" + +[lib] +name = "ksupport" +path = "lib.rs" +crate-type = ["staticlib"] + +[profile.dev] +panic = 'unwind' +opt-level = 2 diff --git a/artiq/runtime.rs/libksupport/api.rs b/artiq/runtime.rs/libksupport/api.rs new file mode 100644 index 000000000..46570feab --- /dev/null +++ b/artiq/runtime.rs/libksupport/api.rs @@ -0,0 +1,120 @@ +use libc::{c_void, c_char, size_t}; + +macro_rules! api { + ($i:ident) => ({ + extern { static $i: c_void; } + api!($i = &$i as *const _) + }); + ($i:ident, $d:item) => ({ + $d + api!($i = $i) + }); + ($i:ident = $e:expr) => { + (stringify!($i), unsafe { $e as *const () }) + } +} + +pub fn resolve(required: &str) -> usize { + unsafe { + API.iter() + .find(|&&(exported, _)| exported == required) + .map(|&(_, ptr)| ptr as usize) + .unwrap_or(0) + } +} + +#[allow(unused_unsafe)] +static mut API: &'static [(&'static str, *const ())] = &[ + api!(__divsi3), + api!(__modsi3), + api!(__ledf2), + api!(__gedf2), + api!(__unorddf2), + api!(__eqdf2), + api!(__ltdf2), + api!(__nedf2), + api!(__gtdf2), + api!(__negsf2), + api!(__negdf2), + api!(__addsf3), + api!(__subsf3), + api!(__mulsf3), + api!(__divsf3), + api!(__lshrdi3), + api!(__muldi3), + api!(__divdi3), + api!(__ashldi3), + api!(__ashrdi3), + api!(__udivmoddi4), + api!(__floatsisf), + api!(__floatunsisf), + api!(__fixsfsi), + api!(__fixunssfsi), + api!(__adddf3), + api!(__subdf3), + api!(__muldf3), + api!(__divdf3), + api!(__floatsidf), + api!(__floatunsidf), + api!(__floatdidf), + api!(__fixdfsi), + api!(__fixdfdi), + api!(__fixunsdfsi), + api!(__clzsi2), + api!(__ctzsi2), + api!(__udivdi3), + api!(__umoddi3), + api!(__moddi3), + api!(__powidf2), + + /* libc */ + api!(strcmp), + api!(strlen, extern { fn strlen(s: *const c_char) -> size_t; }), + api!(abort = ::abort), + + /* libm */ + api!(sqrt), + api!(lround), + + /* exceptions */ + api!(_Unwind_Resume), + api!(__artiq_personality), + api!(__artiq_raise), + api!(__artiq_reraise), + + /* proxified syscalls */ + api!(core_log), + + api!(now = &::NOW as *const _), + + api!(watchdog_set = ::watchdog_set), + api!(watchdog_clear = ::watchdog_clear), + + api!(send_rpc = ::send_rpc), + api!(recv_rpc = ::recv_rpc), + + api!(cache_get = ::cache_get), + api!(cache_put = ::cache_put), + + /* direct syscalls */ + api!(rtio_init), + api!(rtio_get_counter), + api!(rtio_log), + api!(rtio_output), + api!(rtio_input_timestamp), + api!(rtio_input_data), + +// #if ((defined CONFIG_RTIO_DDS_COUNT) && (CONFIG_RTIO_DDS_COUNT > 0)) + api!(dds_init), + api!(dds_init_sync), + api!(dds_batch_enter), + api!(dds_batch_exit), + api!(dds_set), +// #endif + + api!(i2c_init), + api!(i2c_start), + api!(i2c_stop), + api!(i2c_write), + api!(i2c_read), +]; diff --git a/artiq/runtime.rs/libksupport/dyld.rs b/artiq/runtime.rs/libksupport/dyld.rs new file mode 100644 index 000000000..c25f7daea --- /dev/null +++ b/artiq/runtime.rs/libksupport/dyld.rs @@ -0,0 +1,57 @@ +use core::{ptr, slice, str}; +use core::slice::SliceExt; +use libc::{c_void, c_char, c_int, size_t}; + +#[allow(non_camel_case_types)] +#[repr(C)] +#[derive(Default)] +struct dyld_info { + __opaque: [usize; 7] +} + +extern { + fn dyld_load(shlib: *const c_void, base: usize, + resolve: extern fn(*mut c_void, *const c_char) -> usize, + resolve_data: *mut c_void, + info: *mut dyld_info, error_out: *mut *const c_char) -> c_int; + fn dyld_lookup(symbol: *const c_char, info: *const dyld_info) -> *const c_void; + + fn strlen(ptr: *const c_char) -> size_t; +} + +pub struct Library { + lower: dyld_info +} + +impl Library { + pub unsafe fn load(shlib: &[u8], base: usize, mut resolve: F) + -> Result + where F: Fn(&str) -> usize { + extern fn wrapper(data: *mut c_void, name: *const c_char) -> usize + where F: Fn(&str) -> usize { + unsafe { + let f = data as *mut F; + let name = slice::from_raw_parts(name as *const u8, strlen(name)); + (*f)(str::from_utf8_unchecked(name)) + } + } + + let mut library = Library { lower: dyld_info::default() }; + let mut error: *const c_char = ptr::null(); + if dyld_load(shlib.as_ptr() as *const _, base, + wrapper::, &mut resolve as *mut _ as *mut _, + &mut library.lower, &mut error) == 0 { + let error = slice::from_raw_parts(error as *const u8, strlen(error)); + Err(str::from_utf8_unchecked(error)) + } else { + Ok(library) + } + } + + pub unsafe fn lookup(&self, symbol: &str) -> usize { + assert!(symbol.len() < 32); + let mut buf = [0u8; 32]; + buf[0..symbol.as_bytes().len()].copy_from_slice(symbol.as_bytes()); + dyld_lookup(&buf as *const _ as *const c_char, &self.lower) as usize + } +} diff --git a/artiq/runtime.rs/libksupport/lib.rs b/artiq/runtime.rs/libksupport/lib.rs new file mode 100644 index 000000000..4163482ab --- /dev/null +++ b/artiq/runtime.rs/libksupport/lib.rs @@ -0,0 +1,252 @@ +#![feature(lang_items, needs_panic_runtime, asm, libc, core_slice_ext)] + +#![no_std] +#![needs_panic_runtime] + +extern crate libc; + +#[path = "../src/board.rs"] +mod board; +#[path = "../src/mailbox.rs"] +mod mailbox; +#[path = "../src/kernel_proto.rs"] +mod kernel_proto; + +mod dyld; +mod api; + +use core::{mem, ptr, slice, str}; +use libc::{c_char, size_t}; +use kernel_proto::*; +use dyld::Library; + +fn send(request: &Message) { + unsafe { mailbox::send(request as *const _ as usize) } + while !mailbox::acknowledged() {} +} + +fn recv R>(f: F) -> R { + while mailbox::receive() == 0 {} + let result = f(unsafe { mem::transmute::(mailbox::receive()) }); + mailbox::acknowledge(); + result +} + +macro_rules! recv { + ($p:pat => $e:expr) => { + recv(|request| { + if let $p = request { + $e + } else { + send(&Log(format_args!("unexpected reply: {:?}", request))); + loop {} + } + }) + } +} + +macro_rules! print { + ($($arg:tt)*) => ($crate::send(&Log(format_args!($($arg)*)))); +} + +macro_rules! println { + ($fmt:expr) => (print!(concat!($fmt, "\n"))); + ($fmt:expr, $($arg:tt)*) => (print!(concat!($fmt, "\n"), $($arg)*)); +} + +#[lang = "panic_fmt"] +extern fn panic_fmt(args: core::fmt::Arguments, file: &'static str, line: u32) -> ! { + println!("panic at {}:{}: {}", file, line, args); + send(&RunAborted); + loop {} +} + +static mut NOW: u64 = 0; + +#[no_mangle] +pub extern fn send_to_log(ptr: *const u8, len: usize) { + send(&LogSlice(unsafe { + str::from_utf8_unchecked(slice::from_raw_parts(ptr, len)) + })) +} + +extern fn abort() -> ! { + println!("kernel called abort()"); + send(&RunAborted); + loop {} +} + +extern fn send_rpc(service: u32, tag: *const u8, data: *const *const ()) { + extern { fn strlen(s: *const c_char) -> size_t; } + let tag = unsafe { slice::from_raw_parts(tag, strlen(tag as *const c_char)) }; + + send(&RpcSend { + service: service as u32, + batch: service == 0, + tag: tag, + data: data + }) +} + +extern fn recv_rpc(slot: *mut ()) -> usize { + send(&RpcRecvRequest(slot)); + recv!(&RpcRecvReply(ref result) => { + match result { + &Ok(alloc_size) => alloc_size, + &Err(ref exception) => unsafe { __artiq_raise(exception as *const _) } + } + }) +} + +#[allow(improper_ctypes)] +extern { + fn __artiq_raise(exn: *const ::kernel_proto::Exception) -> !; +} + +macro_rules! artiq_raise { + ($name:expr, $message:expr) => ({ + let exn = Exception { + name: concat!("0:artiq.coredevice.exceptions.", $name, "\0").as_bytes().as_ptr(), + file: concat!(file!(), "\0").as_bytes().as_ptr(), + line: line!(), + column: column!(), + // https://github.com/rust-lang/rfcs/pull/1719 + function: "(Rust function)\0".as_bytes().as_ptr(), + message: concat!($message, "\0").as_bytes().as_ptr(), + param: [0; 3], + phantom: ::core::marker::PhantomData + }; + unsafe { __artiq_raise(&exn as *const _) } + }) +} + +#[no_mangle] +pub extern fn __artiq_terminate(exception: *const kernel_proto::Exception, + backtrace_data: *mut usize, + backtrace_size: usize) -> ! { + let backtrace = unsafe { slice::from_raw_parts_mut(backtrace_data, backtrace_size) }; + let mut cursor = 0; + for index in 0..backtrace.len() { + if backtrace[index] > kernel_proto::KERNELCPU_PAYLOAD_ADDRESS { + backtrace[cursor] = backtrace[index] - kernel_proto::KERNELCPU_PAYLOAD_ADDRESS; + cursor += 1; + } + } + let backtrace = &mut backtrace[0..cursor]; + + send(&NowSave(unsafe { NOW })); + send(&RunException { + exception: unsafe { (*exception).clone() }, + backtrace: backtrace + }); + loop {} +} + +extern fn watchdog_set(ms: i32) -> usize { + // FIXME: fix ms + send(&WatchdogSetRequest { ms: ms as u64 }); + recv!(&WatchdogSetReply { id } => id) +} + +extern fn watchdog_clear(id: usize) { + send(&WatchdogClear { id: id }) +} + +extern fn cache_get(key: *const u8) -> (usize, *const u32) { + extern { fn strlen(s: *const c_char) -> size_t; } + let key = unsafe { slice::from_raw_parts(key, strlen(key as *const c_char)) }; + let key = unsafe { str::from_utf8_unchecked(key) }; + + send(&CacheGetRequest { key: key }); + recv!(&CacheGetReply { value } => (value.len(), value.as_ptr())) +} + +extern fn cache_put(key: *const u8, &(len, ptr): &(usize, *const u32)) { + extern { fn strlen(s: *const c_char) -> size_t; } + let key = unsafe { slice::from_raw_parts(key, strlen(key as *const c_char)) }; + let key = unsafe { str::from_utf8_unchecked(key) }; + + let value = unsafe { slice::from_raw_parts(ptr, len) }; + send(&CachePutRequest { key: key, value: value }); + recv!(&CachePutReply { succeeded } => { + if !succeeded { + artiq_raise!("CacheError", "cannot put into a busy cache row") + } + }) +} + +unsafe fn attribute_writeback(typeinfo: *const ()) { + struct Attr { + offset: usize, + tag: *const u8, + name: *const u8 + } + + struct Type { + attributes: *const *const Attr, + objects: *const *const () + } + + let mut tys = typeinfo as *const *const Type; + while !(*tys).is_null() { + let ty = *tys; + tys = tys.offset(1); + + let mut objects = (*ty).objects; + while !(*objects).is_null() { + let object = *objects; + objects = objects.offset(1); + + let mut attributes = (*ty).attributes; + while !(*attributes).is_null() { + let attribute = *attributes; + attributes = attributes.offset(1); + + if !(*attribute).tag.is_null() { + send_rpc(0, (*attribute).tag, [ + &object as *const _ as *const (), + &(*attribute).name as *const _ as *const (), + (object as usize + (*attribute).offset) as *const () + ].as_ptr()); + } + } + } + } +} + +#[no_mangle] +pub unsafe fn main() { + let library = recv!(&LoadRequest(library) => { + match Library::load(library, kernel_proto::KERNELCPU_PAYLOAD_ADDRESS, api::resolve) { + Err(error) => { + send(&LoadReply(Err(error))); + loop {} + }, + Ok(library) => { + send(&LoadReply(Ok(()))); + library + } + } + }); + + let __bss_start = library.lookup("__bss_start"); + let _end = library.lookup("_end"); + ptr::write_bytes(__bss_start as *mut u8, 0, _end - __bss_start); + + send(&NowInitRequest); + recv!(&NowInitReply(now) => NOW = now); + (mem::transmute::(library.lookup("__modinit__")))(); + send(&NowSave(NOW)); + + attribute_writeback(library.lookup("typeinfo") as *const ()); + + send(&RunFinished); + + loop {} +} + +#[no_mangle] +pub fn exception_handler(vect: u32, _regs: *const u32, pc: u32, ea: u32) { + println!("exception {:?} at PC 0x{:x}, EA 0x{:x}", vect, pc, ea); + send(&RunAborted) +} diff --git a/artiq/runtime.rs/libstd_artiq/lib.rs b/artiq/runtime.rs/libstd_artiq/lib.rs index e2fe3b9d8..568dc8663 100644 --- a/artiq/runtime.rs/libstd_artiq/lib.rs +++ b/artiq/runtime.rs/libstd_artiq/lib.rs @@ -65,9 +65,8 @@ pub fn print_fmt(args: self::core::fmt::Arguments) { #[lang = "panic_fmt"] extern fn panic_fmt(args: self::core::fmt::Arguments, file: &'static str, line: u32) -> ! { - let _ = write!(Console, "panic at {}:{}: ", file, line); - let _ = Console.write_fmt(args); - let _ = write!(Console, "\nwaiting for debugger...\n"); + let _ = write!(Console, "panic at {}:{}: {}\n", file, line, args); + let _ = write!(Console, "waiting for debugger...\n"); unsafe { let _ = readchar(); loop { asm!("l.trap 0") } diff --git a/artiq/runtime.rs/src/kernel.rs b/artiq/runtime.rs/src/kernel.rs index f38e951f0..e50902cf5 100644 --- a/artiq/runtime.rs/src/kernel.rs +++ b/artiq/runtime.rs/src/kernel.rs @@ -2,9 +2,7 @@ use core::ptr; use board::csr; use mailbox; -const KERNELCPU_EXEC_ADDRESS: usize = 0x40400000; -const KERNELCPU_LAST_ADDRESS: usize = 0x4fffffff; -const KSUPPORT_HEADER_SIZE: usize = 0x80; +use kernel_proto::{KERNELCPU_EXEC_ADDRESS, KERNELCPU_LAST_ADDRESS, KSUPPORT_HEADER_SIZE}; pub unsafe fn start() { if csr::kernel_cpu::reset_read() == 0 { @@ -13,15 +11,9 @@ pub unsafe fn start() { stop(); - extern { - static _binary_ksupport_elf_start: u8; - static _binary_ksupport_elf_end: u8; - } - let ksupport_start = &_binary_ksupport_elf_start as *const _; - let ksupport_end = &_binary_ksupport_elf_end as *const _; - ptr::copy_nonoverlapping(ksupport_start, - (KERNELCPU_EXEC_ADDRESS - KSUPPORT_HEADER_SIZE) as *mut u8, - ksupport_end as usize - ksupport_start as usize); + let ksupport_image = include_bytes!(concat!(env!("CARGO_TARGET_DIR"), "/../ksupport.elf")); + let ksupport_addr = (KERNELCPU_EXEC_ADDRESS - KSUPPORT_HEADER_SIZE) as *mut u8; + ptr::copy_nonoverlapping(ksupport_image.as_ptr(), ksupport_addr, ksupport_image.len()); csr::kernel_cpu::reset_write(0); } diff --git a/artiq/runtime.rs/src/kernel_proto.rs b/artiq/runtime.rs/src/kernel_proto.rs index 64a074518..a75875738 100644 --- a/artiq/runtime.rs/src/kernel_proto.rs +++ b/artiq/runtime.rs/src/kernel_proto.rs @@ -1,24 +1,30 @@ -use core::{ptr, mem, slice}; -use std::string::String; -use std::io; -use mailbox; -use kernel; +#![allow(dead_code)] -#[derive(Debug)] +use core::marker::PhantomData; +use core::fmt; + +pub const KERNELCPU_EXEC_ADDRESS: usize = 0x40400000; +pub const KERNELCPU_PAYLOAD_ADDRESS: usize = 0x40440000; +pub const KERNELCPU_LAST_ADDRESS: usize = 0x4fffffff; +pub const KSUPPORT_HEADER_SIZE: usize = 0x80; + +#[repr(C)] +#[derive(Debug, Clone)] pub struct Exception<'a> { - pub name: &'a str, - pub file: &'a str, + pub name: *const u8, + pub file: *const u8, pub line: u32, pub column: u32, - pub function: &'a str, - pub message: &'a str, + pub function: *const u8, + pub message: *const u8, pub param: [u64; 3], + pub phantom: PhantomData<&'a str> } #[derive(Debug)] pub enum Message<'a> { LoadRequest(&'a [u8]), - LoadReply { error: Option<&'a str> }, + LoadReply(Result<(), &'a str>), NowInitRequest, NowInitReply(u64), @@ -29,6 +35,7 @@ pub enum Message<'a> { exception: Exception<'a>, backtrace: &'a [usize] }, + RunAborted, WatchdogSetRequest { ms: u64 }, WatchdogSetReply { id: usize }, @@ -40,386 +47,16 @@ pub enum Message<'a> { tag: &'a [u8], data: *const *const () }, - RpcRecvRequest { - slot: *mut () - }, - RpcRecvReply { - alloc_size: usize, - exception: Option> - }, + RpcRecvRequest(*mut ()), + RpcRecvReply(Result>), CacheGetRequest { key: &'a str }, CacheGetReply { value: &'static [u32] }, CachePutRequest { key: &'a str, value: &'static [u32] }, CachePutReply { succeeded: bool }, - Log(&'a str) + Log(fmt::Arguments<'a>), + LogSlice(&'a str) } pub use self::Message::*; - -impl<'a> Message<'a> { - fn into_lower R>(self, f: F) -> R { - match self { - Message::LoadRequest(library) => { - let msg = c::LoadRequest { - ty: c::Type::LoadRequest, - library: library.as_ptr() as *const _ - }; - f(&msg as *const _ as *const _) - } - - Message::NowInitReply(now) => { - let msg = c::NowInitReply { - ty: c::Type::NowInitReply, - now: now - }; - f(&msg as *const _ as *const _) - } - - Message::WatchdogSetReply { id } => { - let msg = c::WatchdogSetReply { - ty: c::Type::WatchdogSetReply, - id: id as _ - }; - f(&msg as *const _ as *const _) - } - - Message::RpcRecvReply { alloc_size, exception } => { - let exn = exception.map(|exception| { - // FIXME: disgusting - let name = String::from(exception.name) + "\0"; - let file = String::from(exception.file) + "\0"; - let function = String::from(exception.function) + "\0"; - let message = String::from(exception.message) + "\0"; - let exn = c::Exception { - name: name.as_ptr() as *const _, - file: file.as_ptr() as *const _, - line: exception.line, - column: exception.column, - function: function.as_ptr() as *const _, - message: message.as_ptr() as *const _, - param: exception.param, - }; - mem::forget(name); - mem::forget(file); - mem::forget(function); - mem::forget(message); - exn - }); - let msg = c::RpcRecvReply { - ty: c::Type::RpcRecvReply, - alloc_size: alloc_size as _, - exception: exn.as_ref().map_or(ptr::null(), |exn| exn as *const _) - }; - f(&msg as *const _ as *const _) - } - - Message::CacheGetReply { value } => { - let msg = c::CacheGetReply { - ty: c::Type::CacheGetReply, - length: value.len(), - elements: value.as_ptr() - }; - f(&msg as *const _ as *const _) - } - Message::CachePutReply { succeeded } => { - let msg = c::CachePutReply { - ty: c::Type::CachePutReply, - succeeded: succeeded as _ - }; - f(&msg as *const _ as *const _) - } - - other => panic!("Message::into_lower: {:?} unimplemented", other) - } - } - - unsafe fn from_lower(ptr: *const ()) -> Self { - let msg = ptr as *const c::Message; - match (*msg).ty { - c::Type::LoadReply => { - let msg = ptr as *const c::LoadReply; - let error = if (*msg).error.is_null() { - None - } else { - Some(c::from_c_str((*msg).error)) - }; - Message::LoadReply { error: error } - } - - c::Type::NowInitRequest => Message::NowInitRequest, - c::Type::NowSave => { - let msg = ptr as *const c::NowSave; - Message::NowSave((*msg).now) - } - - c::Type::RunFinished => Message::RunFinished, - c::Type::RunException => { - let msg = ptr as *const c::RunException; - let exc = (*msg).exception; - Message::RunException { - exception: Exception { - name: c::from_c_str((*exc).name), - file: c::from_c_str((*exc).file), - line: (*exc).line, - column: (*exc).column, - function: c::from_c_str((*exc).function), - message: c::from_c_str((*exc).message), - param: (*exc).param, - }, - backtrace: slice::from_raw_parts((*msg).backtrace, (*msg).backtrace_size) - } - } - - c::Type::WatchdogSetRequest => { - let msg = ptr as *const c::WatchdogSetRequest; - Message::WatchdogSetRequest { ms: (*msg).ms as u64 } - }, - c::Type::WatchdogClear => { - let msg = ptr as *const c::WatchdogClear; - Message::WatchdogClear { id: (*msg).id as usize } - } - - c::Type::RpcSend | c::Type::RpcBatch => { - let msg = ptr as *const c::RpcSend; - Message::RpcSend { - service: (*msg).service as _, - batch: (*msg).ty == c::Type::RpcBatch, - tag: slice::from_raw_parts((*msg).tag as *const _, - c::strlen((*msg).tag) as usize), - data: (*msg).data as *const _ - } - } - c::Type::RpcRecvRequest => { - let msg = ptr as *const c::RpcRecvRequest; - Message::RpcRecvRequest { slot: (*msg).slot as *mut _ } - } - - c::Type::CacheGetRequest => { - let msg = ptr as *const c::CacheGetRequest; - let key = c::from_c_str((*msg).key); - Message::CacheGetRequest { key: key } - } - c::Type::CachePutRequest => { - let msg = ptr as *const c::CachePutRequest; - let key = c::from_c_str((*msg).key); - let value = slice::from_raw_parts((*msg).elements, (*msg).length); - Message::CachePutRequest { key: key, value: value } - } - - c::Type::Log => { - let msg = ptr as *const c::Log; - Message::Log(c::from_c_str_len((*msg).buf, (*msg).len)) - } - - ref other => panic!("Message::from_lower: {:?} unimplemented", other) - } - } - - pub fn send_and_wait(self, waiter: ::sched::Waiter) -> io::Result<()> { - self.into_lower(|ptr| { - unsafe { mailbox::send(ptr as usize) } - waiter.until(mailbox::acknowledged) - }) - } - - pub fn wait_and_receive(waiter: ::sched::Waiter, f: F) -> io::Result - where F: FnOnce(Message<'a>) -> io::Result { - try!(waiter.until(|| mailbox::receive() != 0)); - if !kernel::validate(mailbox::receive()) { - return Err(io::Error::new(io::ErrorKind::InvalidData, "invalid kernel CPU pointer")) - } - - let msg = unsafe { Self::from_lower(mailbox::receive() as *const ()) }; - Ok(try!(f(msg))) - } - - pub fn acknowledge() { - mailbox::acknowledge() - } -} - -// Low-level representation, compatible with the C code in ksupport -mod c { - use libc::{c_void, c_int, c_char, size_t}; - use core::{str, slice}; - - extern { pub fn strlen(ptr: *const c_char) -> size_t; } - - #[repr(u32)] - #[derive(Debug, PartialEq, Eq)] - #[allow(dead_code)] - pub enum Type { - LoadRequest, - LoadReply, - NowInitRequest, - NowInitReply, - NowSave, - RunFinished, - RunException, - WatchdogSetRequest, - WatchdogSetReply, - WatchdogClear, - RpcSend, - RpcRecvRequest, - RpcRecvReply, - RpcBatch, - CacheGetRequest, - CacheGetReply, - CachePutRequest, - CachePutReply, - Log, - } - - #[repr(C)] - #[derive(Debug)] - pub struct Message { - pub ty: Type - } - - // kernel messages - - #[repr(C)] - #[derive(Debug)] - pub struct LoadRequest { - pub ty: Type, - pub library: *const c_void, - } - - #[repr(C)] - #[derive(Debug)] - pub struct LoadReply { - pub ty: Type, - pub error: *const c_char - } - - #[repr(C)] - #[derive(Debug)] - pub struct NowInitReply { - pub ty: Type, - pub now: u64 - } - - #[repr(C)] - #[derive(Debug)] - pub struct NowSave { - pub ty: Type, - pub now: u64 - } - - #[repr(C)] - #[derive(Debug)] - pub struct RunException { - pub ty: Type, - pub exception: *const Exception, - pub backtrace: *const usize, - pub backtrace_size: size_t - } - - #[repr(C)] - #[derive(Debug)] - pub struct WatchdogSetRequest { - pub ty: Type, - pub ms: c_int - } - - #[repr(C)] - #[derive(Debug)] - pub struct WatchdogSetReply { - pub ty: Type, - pub id: c_int - } - - #[repr(C)] - #[derive(Debug)] - pub struct WatchdogClear { - pub ty: Type, - pub id: c_int - } - - #[repr(C)] - #[derive(Debug)] - pub struct RpcSend { - pub ty: Type, - pub service: c_int, - pub tag: *const c_char, - pub data: *const *const c_void - } - - #[repr(C)] - #[derive(Debug)] - pub struct RpcRecvRequest { - pub ty: Type, - pub slot: *mut c_void - } - - #[repr(C)] - #[derive(Debug)] - pub struct RpcRecvReply { - pub ty: Type, - pub alloc_size: c_int, - pub exception: *const Exception - } - - #[repr(C)] - #[derive(Debug)] - pub struct CacheGetRequest { - pub ty: Type, - pub key: *const c_char - } - - #[repr(C)] - #[derive(Debug)] - pub struct CacheGetReply { - pub ty: Type, - pub length: size_t, - pub elements: *const u32 - } - - #[repr(C)] - #[derive(Debug)] - pub struct CachePutRequest { - pub ty: Type, - pub key: *const c_char, - pub length: size_t, - pub elements: *const u32 - } - - #[repr(C)] - #[derive(Debug)] - pub struct CachePutReply { - pub ty: Type, - pub succeeded: c_int - } - - #[repr(C)] - #[derive(Debug)] - pub struct Log { - pub ty: Type, - pub buf: *const c_char, - pub len: size_t - } - - // Supplementary structures - - #[repr(C)] - #[derive(Debug)] - pub struct Exception { - pub name: *const c_char, // or typeinfo - pub file: *const c_char, - pub line: u32, - pub column: u32, - pub function: *const c_char, - pub message: *const c_char, - pub param: [u64; 3], - } - - pub unsafe fn from_c_str_len<'a>(ptr: *const c_char, len: size_t) -> &'a str { - str::from_utf8_unchecked(slice::from_raw_parts(ptr as *const u8, len)) - } - - pub unsafe fn from_c_str<'a>(ptr: *const c_char) -> &'a str { - from_c_str_len(ptr, strlen(ptr)) - } -} diff --git a/artiq/runtime.rs/src/session.rs b/artiq/runtime.rs/src/session.rs index bbdf30f48..0c05f4a5c 100644 --- a/artiq/runtime.rs/src/session.rs +++ b/artiq/runtime.rs/src/session.rs @@ -1,6 +1,7 @@ use std::prelude::v1::*; use std::{mem, str}; use std::cell::RefCell; +use std::fmt::Write; use std::io::{self, Read}; use {config, rtio_crg, clock, mailbox, kernel}; use logger::BufferLogger; @@ -75,6 +76,15 @@ impl<'a> Session<'a> { KernelState::Running | KernelState::RpcWait => true } } + + fn flush_log_buffer(&mut self) { + if &self.log_buffer[self.log_buffer.len() - 1..] == "\n" { + for line in self.log_buffer.lines() { + info!(target: "kernel", "{}", line); + } + self.log_buffer.clear() + } + } } impl<'a> Drop for Session<'a> { @@ -109,24 +119,44 @@ fn host_write(stream: &mut TcpStream, reply: host::Reply) -> io::Result<()> { reply.write_to(stream) } -fn kern_send<'a>(waiter: Waiter, request: kern::Message<'a>) -> io::Result<()> { - match &request { +fn kern_send(waiter: Waiter, request: &kern::Message) -> io::Result<()> { + match request { &kern::LoadRequest(_) => trace!("comm->kern LoadRequest(...)"), _ => trace!("comm->kern {:?}", request) } - request.send_and_wait(waiter) + unsafe { mailbox::send(request as *const _ as usize) } + waiter.until(mailbox::acknowledged) } +fn kern_recv_notrace(waiter: Waiter, f: F) -> io::Result + where F: FnOnce(&kern::Message) -> io::Result { + try!(waiter.until(|| mailbox::receive() != 0)); + if !kernel::validate(mailbox::receive()) { + return Err(io::Error::new(io::ErrorKind::InvalidData, "invalid kernel CPU pointer")) + } + + f(unsafe { mem::transmute::(mailbox::receive()) }) +} + +fn kern_recv_dotrace(reply: &kern::Message) { + match reply { + &kern::Log(_) => trace!("comm<-kern Log(...)"), + &kern::LogSlice(_) => trace!("comm<-kern LogSlice(...)"), + _ => trace!("comm<-kern {:?}", reply) + } +} + +#[inline(always)] fn kern_recv(waiter: Waiter, f: F) -> io::Result - where F: FnOnce(kern::Message) -> io::Result { - kern::Message::wait_and_receive(waiter, |reply| { - trace!("comm<-kern {:?}", reply); + where F: FnOnce(&kern::Message) -> io::Result { + kern_recv_notrace(waiter, |reply| { + kern_recv_dotrace(reply); f(reply) }) } fn kern_acknowledge() -> io::Result<()> { - kern::Message::acknowledge(); + mailbox::acknowledge(); Ok(()) } @@ -137,15 +167,15 @@ unsafe fn kern_load(waiter: Waiter, session: &mut Session, library: &[u8]) -> io kernel::start(); - try!(kern_send(waiter, kern::LoadRequest(&library))); + try!(kern_send(waiter, &kern::LoadRequest(&library))); kern_recv(waiter, |reply| { match reply { - kern::LoadReply { error: None } => { + &kern::LoadReply(Ok(())) => { session.kernel_state = KernelState::Loaded; Ok(()) } - kern::LoadReply { error: Some(cause) } => - unexpected!("cannot load kernel: {}", cause), + &kern::LoadReply(Err(error)) => + unexpected!("cannot load kernel: {}", error), other => unexpected!("unexpected reply from kernel CPU: {:?}", other) } @@ -224,7 +254,10 @@ fn process_host_message(waiter: Waiter, host::Request::LoadKernel(kernel) => match unsafe { kern_load(waiter, session, &kernel) } { Ok(()) => host_write(stream, host::Reply::LoadCompleted), - Err(_) => host_write(stream, host::Reply::LoadFailed) + Err(_) => { + try!(kern_acknowledge()); + host_write(stream, host::Reply::LoadFailed) + } }, host::Request::RunKernel => @@ -240,23 +273,20 @@ fn process_host_message(waiter: Waiter, let slot = try!(kern_recv(waiter, |reply| { match reply { - kern::RpcRecvRequest { slot } => Ok(slot), - other => - unexpected!("unexpected reply from kernel CPU: {:?}", other) + &kern::RpcRecvRequest(slot) => Ok(slot), + other => unexpected!("unexpected reply from kernel CPU: {:?}", other) } })); try!(rpc::recv_return(stream, &tag, slot, &|size| { - try!(kern_send(waiter, kern::RpcRecvReply { - alloc_size: size, exception: None - })); + try!(kern_send(waiter, &kern::RpcRecvReply(Ok(size)))); kern_recv(waiter, |reply| { match reply { - kern::RpcRecvRequest { slot } => Ok(slot), - _ => unreachable!() + &kern::RpcRecvRequest(slot) => Ok(slot), + other => unexpected!("unexpected reply from kernel CPU: {:?}", other) } }) })); - try!(kern_send(waiter, kern::RpcRecvReply { alloc_size: 0, exception: None })); + try!(kern_send(waiter, &kern::RpcRecvReply(Ok(0)))); session.kernel_state = KernelState::Running; Ok(()) @@ -271,23 +301,30 @@ fn process_host_message(waiter: Waiter, try!(kern_recv(waiter, |reply| { match reply { - kern::RpcRecvRequest { .. } => Ok(()), + &kern::RpcRecvRequest(_) => Ok(()), other => unexpected!("unexpected reply from kernel CPU: {:?}", other) } })); - try!(kern_send(waiter, kern::RpcRecvReply { - alloc_size: 0, - exception: Some(kern::Exception { - name: &name, - message: &message, - param: param, - file: &file, - line: line, - column: column, - function: &function - }) - })); + + // FIXME: gross. + fn into_c_str(s: String) -> *const u8 { + let s = s + "\0"; + let p = s.as_bytes().as_ptr(); + mem::forget(s); + p + } + let exn = kern::Exception { + name: into_c_str(name), + message: into_c_str(message), + param: param, + file: into_c_str(file), + line: line, + column: column, + function: into_c_str(function), + phantom: ::core::marker::PhantomData + }; + try!(kern_send(waiter, &kern::RpcRecvReply(Err(exn)))); session.kernel_state = KernelState::Running; Ok(()) @@ -298,10 +335,10 @@ fn process_host_message(waiter: Waiter, fn process_kern_message(waiter: Waiter, mut stream: Option<&mut TcpStream>, session: &mut Session) -> io::Result { - kern::Message::wait_and_receive(waiter, |request| { - match (&request, session.kernel_state) { - (&kern::LoadReply { .. }, KernelState::Loaded) | - (&kern::RpcRecvRequest { .. }, KernelState::RpcWait) => { + kern_recv_notrace(waiter, |request| { + match (request, session.kernel_state) { + (&kern::LoadReply(_), KernelState::Loaded) | + (&kern::RpcRecvRequest(_), KernelState::RpcWait) => { // We're standing by; ignore the message. return Ok(false) } @@ -312,41 +349,41 @@ fn process_kern_message(waiter: Waiter, } } - trace!("comm<-kern {:?}", request); + kern_recv_dotrace(request); match request { - kern::Log(log) => { - session.log_buffer += log; - try!(kern_acknowledge()); - - if &log[log.len() - 1..] == "\n" { - for line in session.log_buffer.lines() { - info!(target: "kernel", "{}", line); - } - session.log_buffer.clear() - } - Ok(()) + &kern::Log(args) => { + try!(session.log_buffer.write_fmt(args) + .map_err(|_| io_error("cannot append to session log buffer"))); + session.flush_log_buffer(); + kern_acknowledge() } - kern::NowInitRequest => - kern_send(waiter, kern::NowInitReply(session.congress.now)), + &kern::LogSlice(arg) => { + session.log_buffer += arg; + session.flush_log_buffer(); + kern_acknowledge() + } - kern::NowSave(now) => { + &kern::NowInitRequest => + kern_send(waiter, &kern::NowInitReply(session.congress.now)), + + &kern::NowSave(now) => { session.congress.now = now; kern_acknowledge() } - kern::WatchdogSetRequest { ms } => { + &kern::WatchdogSetRequest { ms } => { let id = try!(session.watchdog_set.set_ms(ms) .map_err(|()| io_error("out of watchdogs"))); - kern_send(waiter, kern::WatchdogSetReply { id: id }) + kern_send(waiter, &kern::WatchdogSetReply { id: id }) } - kern::WatchdogClear { id } => { + &kern::WatchdogClear { id } => { session.watchdog_set.clear(id); kern_acknowledge() } - kern::RpcSend { service, batch, tag, data } => { + &kern::RpcSend { service, batch, tag, data } => { match stream { None => unexpected!("unexpected RPC in flash kernel"), Some(ref mut stream) => { @@ -362,19 +399,19 @@ fn process_kern_message(waiter: Waiter, } } - kern::CacheGetRequest { key } => { + &kern::CacheGetRequest { key } => { let value = session.congress.cache.get(key); - kern_send(waiter, kern::CacheGetReply { + kern_send(waiter, &kern::CacheGetReply { value: unsafe { mem::transmute::<*const [u32], &'static [u32]>(value) } }) } - kern::CachePutRequest { key, value } => { + &kern::CachePutRequest { key, value } => { let succeeded = session.congress.cache.put(key, value).is_ok(); - kern_send(waiter, kern::CachePutReply { succeeded: succeeded }) + kern_send(waiter, &kern::CachePutReply { succeeded: succeeded }) } - kern::RunFinished => { + &kern::RunFinished => { kernel::stop(); session.kernel_state = KernelState::Absent; unsafe { session.congress.cache.unborrow() } @@ -386,27 +423,38 @@ fn process_kern_message(waiter: Waiter, } } - kern::RunException { exception: ref exn, backtrace } => { + &kern::RunException { exception: ref exn, backtrace } => { kernel::stop(); session.kernel_state = KernelState::Absent; unsafe { session.congress.cache.unborrow() } + unsafe fn from_c_str<'a>(s: *const u8) -> &'a str { + use ::libc::{c_char, size_t}; + use core::slice; + extern { fn strlen(s: *const c_char) -> size_t; } + let s = slice::from_raw_parts(s, strlen(s as *const c_char)); + str::from_utf8_unchecked(s) + } + let name = unsafe { from_c_str(exn.name) }; + let message = unsafe { from_c_str(exn.message) }; + let file = unsafe { from_c_str(exn.file) }; + let function = unsafe { from_c_str(exn.function) }; match stream { None => { error!("exception in flash kernel"); - error!("{}: {} {:?}", exn.name, exn.message, exn.param); - error!("at {}:{}:{} in {}", exn.file, exn.line, exn.column, exn.function); + error!("{}: {} {:?}", name, message, exn.param); + error!("at {}:{}:{} in {}", file, exn.line, exn.column, function); return Ok(true) }, Some(ref mut stream) => host_write(stream, host::Reply::KernelException { - name: exn.name, - message: exn.message, + name: name, + message: message, param: exn.param, - file: exn.file, + file: file, line: exn.line, column: exn.column, - function: exn.function, + function: function, backtrace: backtrace }) } diff --git a/artiq/runtime/Makefile b/artiq/runtime/Makefile index 2990c9b23..1c37c4ec0 100644 --- a/artiq/runtime/Makefile +++ b/artiq/runtime/Makefile @@ -3,11 +3,13 @@ include $(MISOC_DIRECTORY)/software/common.mak PYTHON ?= python3.5 -OBJECTS := flash_storage.o ksupport_data.o main.o -OBJECTS_KSUPPORT := ksupport.o artiq_personality.o mailbox.o \ - rtio.o dds.o i2c.o +OBJECTS := flash_storage.o main.o +OBJECTS_KSUPPORT := ksupport_glue.o artiq_personality.o rtio.o dds.o i2c.o -CFLAGS += -I$(LIBALLOC_DIRECTORY) \ +RUSTOUT_DIRECTORY := cargo/or1k-unknown-none/debug + +CFLAGS += \ + -I$(LIBALLOC_DIRECTORY) \ -I$(MISOC_DIRECTORY)/software/include/dyld \ -I$(LIBDYLD_DIRECTORY)/include \ -I$(LIBUNWIND_DIRECTORY) \ @@ -16,8 +18,57 @@ CFLAGS += -I$(LIBALLOC_DIRECTORY) \ -I$(LIBLWIP_DIRECTORY) CFLAGS += -DNDEBUG +LDFLAGS += --gc-sections \ + -L../libcompiler-rt \ + -L../libbase \ + -L../libm \ + -L../liballoc \ + -L../libunwind \ + -L../libdyld \ + -L../liblwip + all: runtime.bin runtime.fbi +.PHONY: $(RUSTOUT_DIRECTORY)/libruntime.a +$(RUSTOUT_DIRECTORY)/libruntime.a: ksupport.elf + CARGO_TARGET_DIR=$(realpath ./cargo) \ + cargo rustc --verbose \ + --manifest-path $(realpath $(RUNTIME_DIRECTORY)/../runtime.rs/Cargo.toml) \ + --target=or1k-unknown-none -- \ + $(shell cat $(BUILDINC_DIRECTORY)/generated/rust-cfg) \ + -C target-feature=+mul,+div,+ffl1,+cmov,+addc -C opt-level=s \ + -L../libcompiler-rt + +runtime.elf: $(OBJECTS) $(RUSTOUT_DIRECTORY)/libruntime.a + $(LD) $(LDFLAGS) \ + -T $(RUNTIME_DIRECTORY)/runtime.ld \ + -o $@ \ + $^ \ + -lbase-nofloat -lcompiler-rt -lalloc -llwip + @chmod -x $@ + +.PHONY: $(RUSTOUT_DIRECTORY)/libksupport.a +$(RUSTOUT_DIRECTORY)/libksupport.a: + CARGO_TARGET_DIR=$(realpath ./cargo) \ + cargo rustc --verbose \ + --manifest-path $(realpath $(RUNTIME_DIRECTORY)/../runtime.rs/libksupport/Cargo.toml) \ + --target=or1k-unknown-none -- \ + $(shell cat $(BUILDINC_DIRECTORY)/generated/rust-cfg) \ + -C target-feature=+mul,+div,+ffl1,+cmov,+addc -C opt-level=s \ + -L../libcompiler-rt + +ksupport.elf: $(OBJECTS_KSUPPORT) $(RUSTOUT_DIRECTORY)/libksupport.a + $(LD) $(LDFLAGS) \ + --eh-frame-hdr \ + -T $(RUNTIME_DIRECTORY)/ksupport.ld \ + -o $@ \ + $^ \ + -lbase -lm -lcompiler-rt -ldyld -lunwind + @chmod -x $@ + +%.o: $(RUNTIME_DIRECTORY)/%.c + $(compile) + %.bin: %.elf $(OBJCOPY) -O binary $< $@ @chmod -x $@ @@ -25,57 +76,10 @@ all: runtime.bin runtime.fbi %.fbi: %.bin @echo " MSCIMG " $@ && $(PYTHON) -m misoc.tools.mkmscimg -f -o $@ $< -runtime.elf: $(OBJECTS) libruntime.a - $(LD) $(LDFLAGS) \ - --gc-sections \ - -T $(RUNTIME_DIRECTORY)/runtime.ld \ - -N -o $@ \ - ../libbase/crt0-$(CPU).o \ - $(OBJECTS) \ - -L../libcompiler-rt \ - -L../libbase \ - -L../liballoc \ - -L../liblwip \ - -Lcargo/or1k-unknown-none/debug/ \ - -lruntime -lbase-nofloat -lcompiler-rt -lalloc -llwip - @chmod -x $@ - -ksupport.elf: $(OBJECTS_KSUPPORT) - $(LD) $(LDFLAGS) \ - --eh-frame-hdr \ - -T $(RUNTIME_DIRECTORY)/ksupport.ld \ - -N -o $@ \ - ../libbase/crt0-$(CPU).o \ - $^ \ - -L../libbase \ - -L../libm \ - -L../libcompiler-rt \ - -L../libunwind \ - -L../libdyld \ - -lbase -lm -lcompiler-rt -ldyld -lunwind - @chmod -x $@ - -ksupport_data.o: ksupport.elf - $(LD) -r -b binary -o $@ $< - -libruntime.a: - CARGO_TARGET_DIR="./cargo" \ - cargo rustc --verbose \ - --manifest-path $(RUNTIME_DIRECTORY)/../runtime.rs/Cargo.toml \ - --target=or1k-unknown-none -- \ - $(shell cat $(BUILDINC_DIRECTORY)/generated/rust-cfg) \ - -C target-feature=+mul,+div,+ffl1,+cmov,+addc -C opt-level=s \ - -L../libcompiler-rt - -%.o: $(RUNTIME_DIRECTORY)/%.c - $(compile) - -%.o: $(RUNTIME_DIRECTORY)/%.S - $(assemble) - clean: $(RM) $(OBJECTS) $(OBJECTS_KSUPPORT) $(RM) runtime.elf runtime.bin runtime.fbi .*~ *~ $(RM) ksupport.elf ksupport.bin + $(RM) -rf cargo .PHONY: all clean diff --git a/artiq/runtime/ksupport.c b/artiq/runtime/ksupport.c deleted file mode 100644 index aaa313373..000000000 --- a/artiq/runtime/ksupport.c +++ /dev/null @@ -1,509 +0,0 @@ -#include -#include -#include -#include -#include - -#include -#include -#include -#include - -#include "ksupport.h" -#include "mailbox.h" -#include "messages.h" -#include "artiq_personality.h" -#include "rtio.h" -#include "dds.h" -#include "i2c.h" - -#define KERNELCPU_EXEC_ADDRESS 0x40400000 -#define KERNELCPU_PAYLOAD_ADDRESS 0x40420000 -#define KERNELCPU_LAST_ADDRESS 0x4fffffff -#define KSUPPORT_HEADER_SIZE 0x80 - -long lround(double x); - -void ksupport_abort(void); -static void attribute_writeback(void *); - -int64_t now; - -/* compiler-rt symbols */ -extern void __divsi3, __modsi3, __ledf2, __gedf2, __unorddf2, __eqdf2, __ltdf2, - __nedf2, __gtdf2, __negsf2, __negdf2, __addsf3, __subsf3, __mulsf3, - __divsf3, __lshrdi3, __muldi3, __divdi3, __ashldi3, __ashrdi3, - __udivmoddi4, __floatsisf, __floatunsisf, __fixsfsi, __fixunssfsi, - __adddf3, __subdf3, __muldf3, __divdf3, __floatsidf, __floatunsidf, - __floatdidf, __fixdfsi, __fixdfdi, __fixunsdfsi, __clzsi2, __ctzsi2, - __udivdi3, __umoddi3, __moddi3, __powidf2; - -/* artiq_personality symbols */ -extern void __artiq_personality; - -struct symbol { - const char *name; - void *addr; -}; - -static const struct symbol runtime_exports[] = { - /* compiler-rt */ - {"__divsi3", &__divsi3}, - {"__modsi3", &__modsi3}, - {"__ledf2", &__ledf2}, - {"__gedf2", &__gedf2}, - {"__unorddf2", &__unorddf2}, - {"__eqdf2", &__eqdf2}, - {"__ltdf2", &__ltdf2}, - {"__nedf2", &__nedf2}, - {"__gtdf2", &__gtdf2}, - {"__negsf2", &__negsf2}, - {"__negdf2", &__negdf2}, - {"__addsf3", &__addsf3}, - {"__subsf3", &__subsf3}, - {"__mulsf3", &__mulsf3}, - {"__divsf3", &__divsf3}, - {"__lshrdi3", &__lshrdi3}, - {"__muldi3", &__muldi3}, - {"__divdi3", &__divdi3}, - {"__ashldi3", &__ashldi3}, - {"__ashrdi3", &__ashrdi3}, - {"__udivmoddi4", &__udivmoddi4}, - {"__floatsisf", &__floatsisf}, - {"__floatunsisf", &__floatunsisf}, - {"__fixsfsi", &__fixsfsi}, - {"__fixunssfsi", &__fixunssfsi}, - {"__adddf3", &__adddf3}, - {"__subdf3", &__subdf3}, - {"__muldf3", &__muldf3}, - {"__divdf3", &__divdf3}, - {"__floatsidf", &__floatsidf}, - {"__floatunsidf", &__floatunsidf}, - {"__floatdidf", &__floatdidf}, - {"__fixdfsi", &__fixdfsi}, - {"__fixdfdi", &__fixdfdi}, - {"__fixunsdfsi", &__fixunsdfsi}, - {"__clzsi2", &__clzsi2}, - {"__ctzsi2", &__ctzsi2}, - {"__udivdi3", &__udivdi3}, - {"__umoddi3", &__umoddi3}, - {"__moddi3", &__moddi3}, - {"__powidf2", &__powidf2}, - - /* libm */ - {"sqrt", &sqrt}, - {"lround", &lround}, - - /* exceptions */ - {"_Unwind_Resume", &_Unwind_Resume}, - {"__artiq_personality", &__artiq_personality}, - {"__artiq_raise", &__artiq_raise}, - {"__artiq_reraise", &__artiq_reraise}, - {"strcmp", &strcmp}, - {"strlen", &strlen}, - {"abort", &ksupport_abort}, - - /* proxified syscalls */ - {"core_log", &core_log}, - - {"now", &now}, - - {"watchdog_set", &watchdog_set}, - {"watchdog_clear", &watchdog_clear}, - - {"printf", &core_log}, - {"send_rpc", &send_rpc}, - {"recv_rpc", &recv_rpc}, - - /* direct syscalls */ - {"rtio_init", &rtio_init}, - {"rtio_get_counter", &rtio_get_counter}, - {"rtio_log", &rtio_log}, - {"rtio_output", &rtio_output}, - {"rtio_input_timestamp", &rtio_input_timestamp}, - {"rtio_input_data", &rtio_input_data}, - -#if ((defined CONFIG_RTIO_DDS_COUNT) && (CONFIG_RTIO_DDS_COUNT > 0)) - {"dds_init", &dds_init}, - {"dds_init_sync", &dds_init_sync}, - {"dds_batch_enter", &dds_batch_enter}, - {"dds_batch_exit", &dds_batch_exit}, - {"dds_set", &dds_set}, -#endif - - {"i2c_init", &i2c_init}, - {"i2c_start", &i2c_start}, - {"i2c_stop", &i2c_stop}, - {"i2c_write", &i2c_write}, - {"i2c_read", &i2c_read}, - - {"cache_get", &cache_get}, - {"cache_put", &cache_put}, - - /* end */ - {NULL, NULL} -}; - -long lround(double x) -{ - return x < 0 ? floor(x) : ceil(x); -} - -/* called by libunwind */ -int fprintf(FILE *stream, const char *fmt, ...) -{ - va_list args; - va_start(args, fmt); - - char buf[256]; - int len = vscnprintf(buf, sizeof(buf), fmt, args); - - va_end(args); - - struct msg_log request; - request.type = MESSAGE_TYPE_LOG; - request.buf = buf; - request.len = len; - mailbox_send_and_wait(&request); - - return 0; -} - -/* called by libunwind */ -int dladdr (const void *address, Dl_info *info) -{ - /* we don't try to resolve names */ - return 0; -} - -/* called by libunwind */ -int dl_iterate_phdr (int (*callback)(struct dl_phdr_info *, size_t, void *), void *data) -{ - Elf32_Ehdr *ehdr; - struct dl_phdr_info phdr_info; - int retval; - - ehdr = (Elf32_Ehdr *)(KERNELCPU_EXEC_ADDRESS - KSUPPORT_HEADER_SIZE); - phdr_info = (struct dl_phdr_info){ - .dlpi_addr = 0, /* absolutely linked */ - .dlpi_name = "", - .dlpi_phdr = (Elf32_Phdr*) ((intptr_t)ehdr + ehdr->e_phoff), - .dlpi_phnum = ehdr->e_phnum, - }; - retval = callback(&phdr_info, sizeof(phdr_info), data); - if(retval) - return retval; - - ehdr = (Elf32_Ehdr *)KERNELCPU_PAYLOAD_ADDRESS; - phdr_info = (struct dl_phdr_info){ - .dlpi_addr = KERNELCPU_PAYLOAD_ADDRESS, - .dlpi_name = "", - .dlpi_phdr = (Elf32_Phdr*) ((intptr_t)ehdr + ehdr->e_phoff), - .dlpi_phnum = ehdr->e_phnum, - }; - retval = callback(&phdr_info, sizeof(phdr_info), data); - return retval; -} - -static Elf32_Addr resolve_runtime_export(const char *name) -{ - const struct symbol *sym = runtime_exports; - while(sym->name) { - if(!strcmp(sym->name, name)) - return (Elf32_Addr)sym->addr; - ++sym; - } - return 0; -} - -void exception_handler(unsigned long vect, unsigned long *regs, - unsigned long pc, unsigned long ea); -void exception_handler(unsigned long vect, unsigned long *regs, - unsigned long pc, unsigned long ea) -{ - artiq_raise_from_c("InternalError", - "Hardware exception {0} at PC 0x{1:08x}, EA 0x{2:08x}", - vect, pc, ea); -} - -static void now_init(void) -{ - struct msg_base request; - struct msg_now_init_reply *reply; - - request.type = MESSAGE_TYPE_NOW_INIT_REQUEST; - mailbox_send_and_wait(&request); - - reply = mailbox_wait_and_receive(); - if(reply->type != MESSAGE_TYPE_NOW_INIT_REPLY) { - core_log("Malformed MESSAGE_TYPE_NOW_INIT_REQUEST reply type %d\n", - reply->type); - while(1); - } - now = reply->now; - mailbox_acknowledge(); -} - -static void now_save(void) -{ - struct msg_now_save request; - - request.type = MESSAGE_TYPE_NOW_SAVE; - request.now = now; - mailbox_send_and_wait(&request); -} - -int main(void); -int main(void) -{ - static struct dyld_info library_info; - - struct msg_load_request *request = mailbox_wait_and_receive(); - struct msg_load_reply load_reply = { - .type = MESSAGE_TYPE_LOAD_REPLY, - .error = NULL - }; - - if(!dyld_load(request->library, KERNELCPU_PAYLOAD_ADDRESS, - resolve_runtime_export, &library_info, - &load_reply.error)) { - mailbox_send(&load_reply); - while(1); - } - - void *__bss_start = dyld_lookup("__bss_start", &library_info); - void *_end = dyld_lookup("_end", &library_info); - memset(__bss_start, 0, _end - __bss_start); - - void (*kernel_run)() = library_info.init; - void *typeinfo = dyld_lookup("typeinfo", &library_info); - - mailbox_send_and_wait(&load_reply); - - now_init(); - kernel_run(); - now_save(); - - attribute_writeback(typeinfo); - - struct msg_base finished_reply; - finished_reply.type = MESSAGE_TYPE_FINISHED; - mailbox_send_and_wait(&finished_reply); - - while(1); -} - -/* called from __artiq_personality */ -void __artiq_terminate(struct artiq_exception *artiq_exn, - uintptr_t *backtrace, - size_t backtrace_size) -{ - struct msg_exception msg; - - now_save(); - - uintptr_t *cursor = backtrace; - - // Remove all backtrace items belonging to ksupport and subtract - // shared object base from the addresses. - for(int i = 0; i < backtrace_size; i++) { - if(backtrace[i] > KERNELCPU_PAYLOAD_ADDRESS) { - backtrace[i] -= KERNELCPU_PAYLOAD_ADDRESS; - *cursor++ = backtrace[i]; - } - } - - backtrace_size = cursor - backtrace; - - msg.type = MESSAGE_TYPE_EXCEPTION; - msg.exception = artiq_exn; - msg.backtrace = backtrace; - msg.backtrace_size = backtrace_size; - mailbox_send(&msg); - - while(1); -} - -void ksupport_abort() -{ - artiq_raise_from_c("InternalError", "abort() called; check device log for details", - 0, 0, 0); -} - -int watchdog_set(int ms) -{ - struct msg_watchdog_set_request request; - struct msg_watchdog_set_reply *reply; - int id; - - request.type = MESSAGE_TYPE_WATCHDOG_SET_REQUEST; - request.ms = ms; - mailbox_send_and_wait(&request); - - reply = mailbox_wait_and_receive(); - if(reply->type != MESSAGE_TYPE_WATCHDOG_SET_REPLY) { - core_log("Malformed MESSAGE_TYPE_WATCHDOG_SET_REQUEST reply type %d\n", - reply->type); - while(1); - } - id = reply->id; - mailbox_acknowledge(); - - return id; -} - -void watchdog_clear(int id) -{ - struct msg_watchdog_clear request; - - request.type = MESSAGE_TYPE_WATCHDOG_CLEAR; - request.id = id; - mailbox_send_and_wait(&request); -} - -void send_rpc(int service, const char *tag, void **data) -{ - struct msg_rpc_send request; - - if(service != 0) - request.type = MESSAGE_TYPE_RPC_SEND; - else - request.type = MESSAGE_TYPE_RPC_BATCH; - request.service = service; - request.tag = tag; - request.data = data; - mailbox_send_and_wait(&request); -} - -int recv_rpc(void *slot) -{ - struct msg_rpc_recv_request request; - struct msg_rpc_recv_reply *reply; - - request.type = MESSAGE_TYPE_RPC_RECV_REQUEST; - request.slot = slot; - mailbox_send_and_wait(&request); - - reply = mailbox_wait_and_receive(); - if(reply->type != MESSAGE_TYPE_RPC_RECV_REPLY) { - core_log("Malformed MESSAGE_TYPE_RPC_RECV_REQUEST reply type %d\n", - reply->type); - while(1); - } - - if(reply->exception) { - struct artiq_exception exception; - memcpy(&exception, reply->exception, - sizeof(struct artiq_exception)); - mailbox_acknowledge(); - __artiq_raise(&exception); - } else { - int alloc_size = reply->alloc_size; - mailbox_acknowledge(); - return alloc_size; - } -} - -struct attr_desc { - uint32_t offset; - const char *tag; - const char *name; -}; - -struct type_desc { - struct attr_desc **attributes; - void **objects; -}; - -void attribute_writeback(void *utypes) -{ - struct type_desc **types = (struct type_desc **)utypes; - while(*types) { - struct type_desc *type = *types++; - - size_t attr_count = 0; - for(struct attr_desc **attr = type->attributes; *attr; attr++) - attr_count++; - - void **objects = type->objects; - while(*objects) { - void *object = *objects++; - - struct attr_desc **attrs = type->attributes; - while(*attrs) { - struct attr_desc *attr = *attrs++; - - if(attr->tag) { - uintptr_t value = (uintptr_t)object + attr->offset; - void *args[] = { - &object, - &attr->name, - (void*)value - }; - send_rpc(0, attr->tag, args); - } - } - } - } -} - -struct artiq_list cache_get(const char *key) -{ - struct msg_cache_get_request request; - struct msg_cache_get_reply *reply; - - request.type = MESSAGE_TYPE_CACHE_GET_REQUEST; - request.key = key; - mailbox_send_and_wait(&request); - - reply = mailbox_wait_and_receive(); - if(reply->type != MESSAGE_TYPE_CACHE_GET_REPLY) { - core_log("Malformed MESSAGE_TYPE_CACHE_GET_REQUEST reply type %d\n", - reply->type); - while(1); - } - - return (struct artiq_list) { reply->length, reply->elements }; -} - -void cache_put(const char *key, struct artiq_list value) -{ - struct msg_cache_put_request request; - struct msg_cache_put_reply *reply; - - request.type = MESSAGE_TYPE_CACHE_PUT_REQUEST; - request.key = key; - request.elements = value.elements; - request.length = value.length; - mailbox_send_and_wait(&request); - - reply = mailbox_wait_and_receive(); - if(reply->type != MESSAGE_TYPE_CACHE_PUT_REPLY) { - core_log("Malformed MESSAGE_TYPE_CACHE_PUT_REQUEST reply type %d\n", - reply->type); - while(1); - } - - if(!reply->succeeded) { - artiq_raise_from_c("CacheError", - "cannot put into a busy cache row", - 0, 0, 0); - } -} - -void core_log(const char *fmt, ...) -{ - va_list args; - va_start(args, fmt); - - char buf[256]; - int len = vscnprintf(buf, sizeof(buf), fmt, args); - - va_end(args); - - struct msg_log request; - request.type = MESSAGE_TYPE_LOG; - request.buf = buf; - request.len = len; - mailbox_send_and_wait(&request); -} diff --git a/artiq/runtime/ksupport.h b/artiq/runtime/ksupport.h deleted file mode 100644 index 6624ffcfa..000000000 --- a/artiq/runtime/ksupport.h +++ /dev/null @@ -1,17 +0,0 @@ -#ifndef __KSTARTUP_H -#define __KSTARTUP_H - -struct artiq_list { - int32_t length; - int32_t *elements; -}; - -int watchdog_set(int ms); -void watchdog_clear(int id); -void send_rpc(int service, const char *tag, void **data); -int recv_rpc(void *slot); -struct artiq_list cache_get(const char *key); -void cache_put(const char *key, struct artiq_list value); -void core_log(const char *fmt, ...); - -#endif /* __KSTARTUP_H */ diff --git a/artiq/runtime/ksupport.ld b/artiq/runtime/ksupport.ld index 4985009fe..de2267304 100644 --- a/artiq/runtime/ksupport.ld +++ b/artiq/runtime/ksupport.ld @@ -1,14 +1,15 @@ INCLUDE generated/output_format.ld +STARTUP(crt0-or1k.o) ENTRY(_start) INCLUDE generated/regions.ld /* First 4M of main memory are reserved for runtime * code/data/heap, then comes kernel memory. - * First 128K of kernel memory are for support code. + * First 256K of kernel memory are for support code. */ MEMORY { - ksupport (RWX) : ORIGIN = 0x40400000, LENGTH = 0x20000 + ksupport (RWX) : ORIGIN = 0x40400000, LENGTH = 0x40000 } /* Kernel stack is at the end of main RAM. */ @@ -30,6 +31,16 @@ SECTIONS _etext = .; } :text + /* https://sourceware.org/bugzilla/show_bug.cgi?id=20475 */ + .got : { + _GLOBAL_OFFSET_TABLE_ = .; + *(.got) + } :text + + .got.plt : { + *(.got.plt) + } :text + .rodata : { . = ALIGN(4); @@ -41,12 +52,12 @@ SECTIONS .eh_frame : { - *(.eh_frame) + KEEP(*(.eh_frame)) } :text .eh_frame_hdr : { - *(.eh_frame_hdr) + KEEP(*(.eh_frame_hdr)) } :text :eh_frame .data : @@ -72,4 +83,9 @@ SECTIONS . = ALIGN(4); _ebss = .; } + + /DISCARD/ : + { + *(.debug*) + } } diff --git a/artiq/runtime/ksupport_glue.c b/artiq/runtime/ksupport_glue.c new file mode 100644 index 000000000..9de97b75d --- /dev/null +++ b/artiq/runtime/ksupport_glue.c @@ -0,0 +1,98 @@ +#include +#include +#include +#include + +#include +#include + +void send_to_log(const char *ptr, size_t length); + +#define KERNELCPU_EXEC_ADDRESS 0x40400000 +#define KERNELCPU_PAYLOAD_ADDRESS 0x40440000 +#define KERNELCPU_LAST_ADDRESS 0x4fffffff +#define KSUPPORT_HEADER_SIZE 0x80 + +/* called by libunwind */ +int fprintf(FILE *stream, const char *fmt, ...) +{ + size_t size; + char *buf; + va_list args; + + va_start(args, fmt); + size = vsnprintf(NULL, 0, fmt, args); + buf = __builtin_alloca(size + 1); + va_end(args); + + va_start(args, fmt); + vsnprintf(buf, size + 1, fmt, args); + va_end(args); + + send_to_log(buf, size); + return 0; +} + +/* called by libunwind */ +int dladdr (const void *address, Dl_info *info) +{ + /* we don't try to resolve names */ + return 0; +} + +/* called by libunwind */ +int dl_iterate_phdr (int (*callback)(struct dl_phdr_info *, size_t, void *), void *data) +{ + Elf32_Ehdr *ehdr; + struct dl_phdr_info phdr_info; + int retval; + + ehdr = (Elf32_Ehdr *)(KERNELCPU_EXEC_ADDRESS - KSUPPORT_HEADER_SIZE); + phdr_info = (struct dl_phdr_info){ + .dlpi_addr = 0, /* absolutely linked */ + .dlpi_name = "", + .dlpi_phdr = (Elf32_Phdr*) ((intptr_t)ehdr + ehdr->e_phoff), + .dlpi_phnum = ehdr->e_phnum, + }; + retval = callback(&phdr_info, sizeof(phdr_info), data); + if(retval) + return retval; + + ehdr = (Elf32_Ehdr *)KERNELCPU_PAYLOAD_ADDRESS; + phdr_info = (struct dl_phdr_info){ + .dlpi_addr = KERNELCPU_PAYLOAD_ADDRESS, + .dlpi_name = "", + .dlpi_phdr = (Elf32_Phdr*) ((intptr_t)ehdr + ehdr->e_phoff), + .dlpi_phnum = ehdr->e_phnum, + }; + retval = callback(&phdr_info, sizeof(phdr_info), data); + return retval; +} + +/* called by kernel */ +long lround(double x); +long lround(double x) +{ + return x < 0 ? floor(x) : ceil(x); +} + +/* called by kernel */ +int core_log(const char *fmt, ...); +int core_log(const char *fmt, ...) +{ + size_t size; + char *buf; + va_list args; + + va_start(args, fmt); + size = vsnprintf(NULL, 0, fmt, args); + buf = __builtin_alloca(size + 1); + va_end(args); + + va_start(args, fmt); + vsnprintf(buf, size + 1, fmt, args); + va_end(args); + + send_to_log(buf, size); + return 0; +} diff --git a/artiq/runtime/mailbox.c b/artiq/runtime/mailbox.c deleted file mode 100644 index 1faa6d374..000000000 --- a/artiq/runtime/mailbox.c +++ /dev/null @@ -1,59 +0,0 @@ -#include -#include -#include -#include -#include - -#include "mailbox.h" - -#define KERNELCPU_MAILBOX MMPTR(MAILBOX_BASE) - -static unsigned int last_transmission; - -void mailbox_send(void *ptr) -{ - last_transmission = (unsigned int)ptr; - KERNELCPU_MAILBOX = last_transmission; -} - -int mailbox_acknowledged(void) -{ - unsigned int m; - - m = KERNELCPU_MAILBOX; - return !m || (m != last_transmission); -} - -void mailbox_send_and_wait(void *ptr) -{ - mailbox_send(ptr); - while(!mailbox_acknowledged()); -} - -void *mailbox_receive(void) -{ - unsigned int r; - - r = KERNELCPU_MAILBOX; - if(r == last_transmission) - return NULL; - else { - if(r) { - flush_cpu_dcache(); - } - return (void *)r; - } -} - -void *mailbox_wait_and_receive(void) -{ - void *r; - - while(!(r = mailbox_receive())); - return r; -} - -void mailbox_acknowledge(void) -{ - KERNELCPU_MAILBOX = 0; -} diff --git a/artiq/runtime/mailbox.h b/artiq/runtime/mailbox.h deleted file mode 100644 index f1d40c973..000000000 --- a/artiq/runtime/mailbox.h +++ /dev/null @@ -1,12 +0,0 @@ -#ifndef __MAILBOX_H -#define __MAILBOX_H - -void mailbox_send(void *ptr); -int mailbox_acknowledged(void); -void mailbox_send_and_wait(void *ptr); - -void *mailbox_receive(void); -void *mailbox_wait_and_receive(void); -void mailbox_acknowledge(void); - -#endif /* __MAILBOX_H */ diff --git a/artiq/runtime/messages.h b/artiq/runtime/messages.h deleted file mode 100644 index a19190cfd..000000000 --- a/artiq/runtime/messages.h +++ /dev/null @@ -1,125 +0,0 @@ -#ifndef __MESSAGES_H -#define __MESSAGES_H - -#include -#include -#include - -enum { - MESSAGE_TYPE_LOAD_REQUEST, - MESSAGE_TYPE_LOAD_REPLY, - MESSAGE_TYPE_NOW_INIT_REQUEST, - MESSAGE_TYPE_NOW_INIT_REPLY, - MESSAGE_TYPE_NOW_SAVE, - MESSAGE_TYPE_FINISHED, - MESSAGE_TYPE_EXCEPTION, - MESSAGE_TYPE_WATCHDOG_SET_REQUEST, - MESSAGE_TYPE_WATCHDOG_SET_REPLY, - MESSAGE_TYPE_WATCHDOG_CLEAR, - MESSAGE_TYPE_RPC_SEND, - MESSAGE_TYPE_RPC_RECV_REQUEST, - MESSAGE_TYPE_RPC_RECV_REPLY, - MESSAGE_TYPE_RPC_BATCH, - MESSAGE_TYPE_CACHE_GET_REQUEST, - MESSAGE_TYPE_CACHE_GET_REPLY, - MESSAGE_TYPE_CACHE_PUT_REQUEST, - MESSAGE_TYPE_CACHE_PUT_REPLY, - MESSAGE_TYPE_LOG, -}; - -struct msg_base { - int type; -}; - -/* kernel messages */ - -struct msg_load_request { - int type; - const void *library; -}; - -struct msg_load_reply { - int type; - const char *error; -}; - -struct msg_now_init_reply { - int type; - long long int now; -}; - -struct msg_now_save { - int type; - long long int now; -}; - -struct msg_exception { - int type; - struct artiq_exception *exception; - uintptr_t *backtrace; - size_t backtrace_size; -}; - -struct msg_watchdog_set_request { - int type; - int ms; -}; - -struct msg_watchdog_set_reply { - int type; - int id; -}; - -struct msg_watchdog_clear { - int type; - int id; -}; - -struct msg_rpc_send { - int type; - int service; - const char *tag; - void **data; -}; - -struct msg_rpc_recv_request { - int type; - void *slot; -}; - -struct msg_rpc_recv_reply { - int type; - int alloc_size; - struct artiq_exception *exception; -}; - -struct msg_cache_get_request { - int type; - const char *key; -}; - -struct msg_cache_get_reply { - int type; - size_t length; - int32_t *elements; -}; - -struct msg_cache_put_request { - int type; - const char *key; - size_t length; - int32_t *elements; -}; - -struct msg_cache_put_reply { - int type; - int succeeded; -}; - -struct msg_log { - int type; - const char *buf; - size_t len; -}; - -#endif /* __MESSAGES_H */ diff --git a/artiq/runtime/runtime.ld b/artiq/runtime/runtime.ld index abcc69e91..5d3d00e08 100644 --- a/artiq/runtime/runtime.ld +++ b/artiq/runtime/runtime.ld @@ -1,4 +1,5 @@ INCLUDE generated/output_format.ld +STARTUP(crt0-or1k.o) ENTRY(_start) INCLUDE generated/regions.ld @@ -73,7 +74,9 @@ SECTIONS .heap : { _fheap = .; - . = ORIGIN(runtime) + LENGTH(runtime) - 0x1000; + . = ORIGIN(runtime) + LENGTH(runtime) + /* Leave room for ksupport headers. */ + - 0x1000; _eheap = .; } > runtime