M-Labs
/
artiq
mirror of https://github.com/m-labs/artiq.git
2
0
Fork 2
Code Issues Projects Releases Wiki Activity

Merge branch 'master' into nac3

This commit is contained in:
Sebastien Bourdeauducq 2022-01-26 07:25:20 +08:00
parent c8ec2b0d7f 9d43762695
commit ceceabbaf0
31 changed files with 664 additions and 446 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
RELEASE_NOTES.rst
View File

@ -32,9 +32,6 @@ Highlights:
* Previously detected RTIO async errors are reported to the host after each kernel terminates and a
warning is logged. The warning is additional to the one already printed in the core device log upon
detection of the error.
* HDF5 options can now be passed when creating datasets with ``set_dataset``. This allows
in particular to use transparent compression filters as follows:
``set_dataset(name, value, hdf5_options={"compression": "gzip"})``.
* Removed worker DB warning for writing a dataset that is also in the archive
Breaking changes:
@ -51,10 +48,6 @@ Breaking changes:
calling `ADF5356.init()`.
* DRTIO: Changed message alignment from 32-bits to 64-bits.
* The deprecated ``set_dataset(..., save=...)`` is no longer supported.
* The internal dataset representation was changed to support tracking HDF5 options like e.g.
a compression method. This requires changes to code reading the dataset persistence file
(``dataset_db.pyon``) and to custom applets.
ARTIQ-6
-------
@ -129,7 +122,6 @@ Breaking changes:
* Experiment classes with underscore-prefixed names are now ignored when ``artiq_client``
determines which experiment to submit (consistent with ``artiq_run``).
ARTIQ-5
-------

2
artiq/applets/big_number.py
View File

@ -13,7 +13,7 @@ class NumberWidget(QtWidgets.QLCDNumber):
def data_changed(self, data, mods):
try:
n = float(data[self.dataset_name]["value"])
n = float(data[self.dataset_name][1])
except (KeyError, ValueError, TypeError):
n = "---"
self.display(n)

2
artiq/applets/image.py
View File

@ -13,7 +13,7 @@ class Image(pyqtgraph.ImageView):
def data_changed(self, data, mods):
try:
img = data[self.args.img]["value"]
img = data[self.args.img][1]
except KeyError:
return
self.setImage(img)

4
artiq/applets/plot_hist.py
View File

@ -17,11 +17,11 @@ class HistogramPlot(pyqtgraph.PlotWidget):
def data_changed(self, data, mods, title):
try:
y = data[self.args.y]["value"]
y = data[self.args.y][1]
if self.args.x is None:
x = None
else:
x = data[self.args.x]["value"]
x = data[self.args.x][1]
except KeyError:
return
if x is None:

9
artiq/applets/plot_xy.py
View File

@ -6,7 +6,6 @@ from PyQt5.QtCore import QTimer
import pyqtgraph
from artiq.applets.simple import TitleApplet
from artiq.master.databases import make_dataset as empty_dataset
class XYPlot(pyqtgraph.PlotWidget):
@ -22,14 +21,14 @@ class XYPlot(pyqtgraph.PlotWidget):
def data_changed(self, data, mods, title):
try:
y = data[self.args.y]["value"]
y = data[self.args.y][1]
except KeyError:
return
x = data.get(self.args.x, empty_dataset())["value"]
x = data.get(self.args.x, (False, None))[1]
if x is None:
x = np.arange(len(y))
error = data.get(self.args.error, empty_dataset())["value"]
fit = data.get(self.args.fit, empty_dataset())["value"]
error = data.get(self.args.error, (False, None))[1]
fit = data.get(self.args.fit, (False, None))[1]
if not len(y) or len(y) != len(x):
self.mismatch['X values'] = True

6
artiq/applets/plot_xy_hist.py
View File

@ -126,9 +126,9 @@ class XYHistPlot(QtWidgets.QSplitter):
def data_changed(self, data, mods):
try:
xs = data[self.args.xs]["value"]
histogram_bins = data[self.args.histogram_bins]["value"]
histograms_counts = data[self.args.histograms_counts]["value"]
xs = data[self.args.xs][1]
histogram_bins = data[self.args.histogram_bins][1]
histograms_counts = data[self.args.histograms_counts][1]
except KeyError:
return
if len(xs) != histograms_counts.shape[0]:

3
artiq/applets/simple.py
View File

@ -10,7 +10,6 @@ from sipyco.sync_struct import Subscriber, process_mod
from sipyco import pyon
from sipyco.pipe_ipc import AsyncioChildComm
from artiq.master.databases import make_dataset as empty_dataset
logger = logging.getLogger(__name__)
@ -252,7 +251,7 @@ class TitleApplet(SimpleApplet):
def emit_data_changed(self, data, mod_buffer):
if self.args.title is not None:
title_values = {k.replace(".", "/"): data.get(k, empty_dataset())["value"]
title_values = {k.replace(".", "/"): data.get(k, (False, None))[1]
for k in self.dataset_title}
try:
title = self.args.title.format(**title_values)

4
artiq/browser/datasets.py
View File

@ -104,8 +104,8 @@ class DatasetsDock(QtWidgets.QDockWidget):
idx = self.table_model_filter.mapToSource(idx[0])
key = self.table_model.index_to_key(idx)
if key is not None:
dataset = self.table_model.backing_store[key]
asyncio.ensure_future(self._upload_dataset(key, dataset["value"]))
persist, value = self.table_model.backing_store[key]
asyncio.ensure_future(self._upload_dataset(key, value))
def save_state(self):
return bytes(self.table.header().saveState())

73
artiq/coredevice/comm_kernel.py
View File

@ -571,29 +571,33 @@ class CommKernel:
self._write_header(Request.RPCException)
# Note: instead of sending strings, we send object ID
# This is to avoid the need of allocatio on the device side
# This is a special case: this only applies to exceptions
if hasattr(exn, "artiq_core_exception"):
exn = exn.artiq_core_exception
self._write_string(exn.name)
self._write_string(self._truncate_message(exn.message))
self._write_int32(embedding_map.store_str(exn.name))
self._write_int32(embedding_map.store_str(self._truncate_message(exn.message)))
for index in range(3):
self._write_int64(exn.param[index])
filename, line, column, function = exn.traceback[-1]
self._write_string(filename)
self._write_int32(embedding_map.store_str(filename))
self._write_int32(line)
self._write_int32(column)
self._write_string(function)
self._write_int32(embedding_map.store_str(function))
else:
exn_type = type(exn)
if exn_type in (ZeroDivisionError, ValueError, IndexError, RuntimeError) or \
hasattr(exn, "artiq_builtin"):
self._write_string("0:{}".format(exn_type.__name__))
name = "0:{}".format(exn_type.__name__)
else:
exn_id = embedding_map.store_object(exn_type)
self._write_string("{}:{}.{}".format(exn_id,
name = "{}:{}.{}".format(exn_id,
exn_type.__module__,
exn_type.__qualname__))
self._write_string(self._truncate_message(str(exn)))
exn_type.__qualname__)
self._write_int32(embedding_map.store_str(name))
self._write_int32(embedding_map.store_str(self._truncate_message(str(exn))))
for index in range(3):
self._write_int64(0)
@ -604,10 +608,10 @@ class CommKernel:
((filename, line, function, _), ) = tb
else:
assert False
self._write_string(filename)
self._write_int32(embedding_map.store_str(filename))
self._write_int32(line)
self._write_int32(-1) # column not known
self._write_string(function)
self._write_int32(embedding_map.store_str(function))
self._flush()
else:
logger.debug("rpc service: %d %r %r = %r",
@ -619,28 +623,59 @@ class CommKernel:
self._flush()
def _serve_exception(self, embedding_map, symbolizer, demangler):
name = self._read_string()
message = self._read_string()
exception_count = self._read_int32()
nested_exceptions = []
def read_exception_string():
# note: if length == -1, the following int32 is the object key
length = self._read_int32()
if length == -1:
return embedding_map.retrieve_str(self._read_int32())
else:
return self._read(length).decode("utf-8")
for _ in range(exception_count):
name = embedding_map.retrieve_str(self._read_int32())
message = read_exception_string()
params = [self._read_int64() for _ in range(3)]
filename = self._read_string()
filename = read_exception_string()
line = self._read_int32()
column = self._read_int32()
function = self._read_string()
function = read_exception_string()
nested_exceptions.append([name, message, params,
filename, line, column, function])
demangled_names = demangler([ex[6] for ex in nested_exceptions])
for i in range(exception_count):
nested_exceptions[i][6] = demangled_names[i]
exception_info = []
for _ in range(exception_count):
sp = self._read_int32()
initial_backtrace = self._read_int32()
current_backtrace = self._read_int32()
exception_info.append((sp, initial_backtrace, current_backtrace))
backtrace = []
stack_pointers = []
for _ in range(self._read_int32()):
backtrace.append(self._read_int32())
stack_pointers.append(self._read_int32())
backtrace = [self._read_int32() for _ in range(self._read_int32())]
self._process_async_error()
traceback = list(reversed(symbolizer(backtrace))) + \
[(filename, line, column, *demangler([function]), None)]
core_exn = exceptions.CoreException(name, message, params, traceback)
traceback = list(symbolizer(backtrace))
core_exn = exceptions.CoreException(nested_exceptions, exception_info,
traceback, stack_pointers)
if core_exn.id == 0:
python_exn_type = getattr(exceptions, core_exn.name.split('.')[-1])
else:
python_exn_type = embedding_map.retrieve_object(core_exn.id)
python_exn = python_exn_type(message.format(*params))
python_exn = python_exn_type(
nested_exceptions[-1][1].format(*nested_exceptions[0][2]))
python_exn.artiq_core_exception = core_exn
raise python_exn

76
artiq/coredevice/exceptions.py
View File

@ -16,50 +16,94 @@ AssertionError = builtins.AssertionError
class CoreException:
"""Information about an exception raised or passed through the core device."""
def __init__(self, exceptions, exception_info, traceback, stack_pointers):
self.exceptions = exceptions
self.exception_info = exception_info
self.traceback = list(traceback)
self.stack_pointers = stack_pointers
def __init__(self, name, message, params, traceback):
first_exception = exceptions[0]
name = first_exception[0]
if ':' in name:
exn_id, self.name = name.split(':', 2)
self.id = int(exn_id)
else:
self.id, self.name = 0, name
self.message, self.params = message, params
self.traceback = list(traceback)
self.message = first_exception[1]
self.params = first_exception[2]
def __str__(self):
lines = []
lines.append("Core Device Traceback (most recent call last):")
last_address = 0
for (filename, line, column, function, address) in self.traceback:
def append_backtrace(self, record, inlined=False):
filename, line, column, function, address = record
stub_globals = {"__name__": filename, "__loader__": source_loader}
source_line = linecache.getline(filename, line, stub_globals)
indentation = re.search(r"^\s*", source_line).end()
if address is None:
formatted_address = ""
elif address == last_address:
elif inlined:
formatted_address = " (inlined)"
else:
formatted_address = " (RA=+0x{:x})".format(address)
last_address = address
filename = filename.replace(artiq_dir, "<artiq>")
lines = []
if column == -1:
lines.append(" {}".format(source_line.strip() if source_line else "<unknown>"))
lines.append(" File \"{file}\", line {line}, in {function}{address}".
format(file=filename, line=line, function=function,
address=formatted_address))
lines.append(" {}".format(source_line.strip() if source_line else "<unknown>"))
else:
lines.append(" {}^".format(" " * (column - indentation)))
lines.append(" {}".format(source_line.strip() if source_line else "<unknown>"))
lines.append(" File \"{file}\", line {line}, column {column},"
" in {function}{address}".
format(file=filename, line=line, column=column + 1,
function=function, address=formatted_address))
lines.append(" {}".format(source_line.strip() if source_line else "<unknown>"))
lines.append(" {}^".format(" " * (column - indentation)))
return lines
lines.append("{}({}): {}".format(self.name, self.id,
self.message.format(*self.params)))
return "\n".join(lines)
def single_traceback(self, exception_index):
# note that we insert in reversed order
lines = []
last_sp = 0
start_backtrace_index = self.exception_info[exception_index][1]
zipped = list(zip(self.traceback[start_backtrace_index:],
self.stack_pointers[start_backtrace_index:]))
exception = self.exceptions[exception_index]
name = exception[0]
message = exception[1]
params = exception[2]
if ':' in name:
exn_id, name = name.split(':', 2)
exn_id = int(exn_id)
else:
exn_id = 0
lines.append("{}({}): {}".format(name, exn_id, message.format(*params)))
zipped.append(((exception[3], exception[4], exception[5], exception[6],
None, []), None))
for ((filename, line, column, function, address, inlined), sp) in zipped:
# backtrace of nested exceptions may be discontinuous
# but the stack pointer must increase monotonically
if sp is not None and sp <= last_sp:
continue
last_sp = sp
for record in reversed(inlined):
lines += self.append_backtrace(record, True)
lines += self.append_backtrace((filename, line, column, function,
address))
lines.append("Traceback (most recent call first):")
return "\n".join(reversed(lines))
def __str__(self):
tracebacks = [self.single_traceback(i) for i in range(len(self.exceptions))]
traceback_str = ('\n\nDuring handling of the above exception, ' +
'another exception occurred:\n\n').join(tracebacks)
return 'Core Device Traceback:\n' +\
traceback_str +\
'\n\nEnd of Code Device Traceback\n'
class InternalError(Exception):

16
artiq/dashboard/datasets.py
View File

@ -147,15 +147,15 @@ class Creator(QtWidgets.QDialog):
class Model(DictSyncTreeSepModel):
def __init__(self, init):
DictSyncTreeSepModel.__init__(
self, ".", ["Dataset", "Persistent", "Value"], init
)
DictSyncTreeSepModel.__init__(self, ".",
["Dataset", "Persistent", "Value"],
init)
def convert(self, k, v, column):
if column == 1:
return "Y" if v["persist"] else "N"
return "Y" if v[0] else "N"
elif column == 2:
return short_format(v["value"])
return short_format(v[1])
else:
raise ValueError
@ -223,8 +223,8 @@ class DatasetsDock(QtWidgets.QDockWidget):
idx = self.table_model_filter.mapToSource(idx[0])
key = self.table_model.index_to_key(idx)
if key is not None:
dataset = self.table_model.backing_store[key]
t = type(dataset["value"])
persist, value = self.table_model.backing_store[key]
t = type(value)
if np.issubdtype(t, np.number):
dialog_cls = NumberEditor
elif np.issubdtype(t, np.bool_):
@ -235,7 +235,7 @@ class DatasetsDock(QtWidgets.QDockWidget):
logger.error("Cannot edit dataset %s: "
"type %s is not supported", key, t)
return
dialog_cls(self, self.dataset_ctl, key, dataset["value"]).open()
dialog_cls(self, self.dataset_ctl, key, value).open()
def delete_clicked(self):
idx = self.table.selectedIndexes()

1
artiq/firmware/Cargo.lock generated
View File

@ -253,6 +253,7 @@ dependencies = [
"byteorder",
"cslice",
"dyld",
"eh",
"failure",
"failure_derive",
"io",

3
artiq/firmware/ksupport/api.rs
View File

@ -117,7 +117,8 @@ static mut API: &'static [(&'static str, *const ())] = &[
api!(_Unwind_Resume = ::unwind::_Unwind_Resume),
api!(__artiq_personality = ::eh_artiq::personality),
api!(__artiq_raise = ::eh_artiq::raise),
api!(__artiq_reraise = ::eh_artiq::reraise),
api!(__artiq_resume = ::eh_artiq::resume),
api!(__artiq_end_catch = ::eh_artiq::end_catch),
/* proxified syscalls */
api!(core_log),

354
artiq/firmware/ksupport/eh_artiq.rs
View File

@ -10,12 +10,15 @@
// except according to those terms.
#![allow(non_camel_case_types)]
use core::{ptr, mem};
use cslice::CSlice;
use core::mem;
use cslice::AsCSlice;
use unwind as uw;
use libc::{c_int, c_void};
use eh::dwarf::{self, EHAction, EHContext};
use eh::{self, dwarf::{self, EHAction, EHContext}};
pub type Exception<'a> = eh::eh_artiq::Exception<'a>;
pub type StackPointerBacktrace = eh::eh_artiq::StackPointerBacktrace;
type _Unwind_Stop_Fn = extern "C" fn(version: c_int,
actions: uw::_Unwind_Action,
@ -30,40 +33,74 @@ extern {
stop_parameter: *mut c_void) -> uw::_Unwind_Reason_Code;
}
#[repr(C)]
#[derive(Clone, Copy)]
pub struct Exception<'a> {
pub name: CSlice<'a, u8>,
pub file: CSlice<'a, u8>,
pub line: u32,
pub column: u32,
pub function: CSlice<'a, u8>,
pub message: CSlice<'a, u8>,
pub param: [i64; 3]
}
pub static mut PAYLOAD_ADDRESS: usize = 0;
const EXCEPTION_CLASS: uw::_Unwind_Exception_Class = 0x4d_4c_42_53_41_52_54_51; /* 'MLBSARTQ' */
const MAX_INFLIGHT_EXCEPTIONS: usize = 10;
const MAX_BACKTRACE_SIZE: usize = 128;
#[repr(C)]
struct ExceptionInfo {
uw_exception: uw::_Unwind_Exception,
exception: Option<Exception<'static>>,
handled: bool,
backtrace: [usize; MAX_BACKTRACE_SIZE],
backtrace_size: usize
struct ExceptionBuffer {
// we need n _Unwind_Exception, because each will have their own private data
uw_exceptions: [uw::_Unwind_Exception; MAX_INFLIGHT_EXCEPTIONS],
exceptions: [Option<Exception<'static>>; MAX_INFLIGHT_EXCEPTIONS + 1],
exception_stack: [isize; MAX_INFLIGHT_EXCEPTIONS + 1],
// nested exceptions will share the backtrace buffer, treated as a tree
// backtrace contains a tuple of IP and SP
backtrace: [(usize, usize); MAX_BACKTRACE_SIZE],
backtrace_size: usize,
// stack pointers are stored to reconstruct backtrace for each exception
stack_pointers: [StackPointerBacktrace; MAX_INFLIGHT_EXCEPTIONS + 1],
// current allocated nested exceptions
exception_count: usize,
}
static mut EXCEPTION_BUFFER: ExceptionBuffer = ExceptionBuffer {
uw_exceptions: [uw::_Unwind_Exception {
exception_class: EXCEPTION_CLASS,
exception_cleanup: cleanup,
private: [0; uw::unwinder_private_data_size],
}; MAX_INFLIGHT_EXCEPTIONS],
exceptions: [None; MAX_INFLIGHT_EXCEPTIONS + 1],
exception_stack: [-1; MAX_INFLIGHT_EXCEPTIONS + 1],
backtrace: [(0, 0); MAX_BACKTRACE_SIZE],
backtrace_size: 0,
stack_pointers: [StackPointerBacktrace {
stack_pointer: 0,
initial_backtrace_size: 0,
current_backtrace_size: 0
}; MAX_INFLIGHT_EXCEPTIONS + 1],
exception_count: 0
};
pub unsafe extern fn reset_exception_buffer(payload_addr: usize) {
EXCEPTION_BUFFER.uw_exceptions = [uw::_Unwind_Exception {
exception_class: EXCEPTION_CLASS,
exception_cleanup: cleanup,
private: [0; uw::unwinder_private_data_size],
}; MAX_INFLIGHT_EXCEPTIONS];
EXCEPTION_BUFFER.exceptions = [None; MAX_INFLIGHT_EXCEPTIONS + 1];
EXCEPTION_BUFFER.exception_stack = [-1; MAX_INFLIGHT_EXCEPTIONS + 1];
EXCEPTION_BUFFER.backtrace_size = 0;
EXCEPTION_BUFFER.exception_count = 0;
PAYLOAD_ADDRESS = payload_addr;
}
#[cfg(target_arch = "x86_64")]
const UNWIND_DATA_REG: (i32, i32) = (0, 1); // RAX, RDX
#[cfg(target_arch = "x86_64")]
// actually this is not the SP, but frame pointer
// but it serves its purpose, and getting SP will somehow cause segfault...
const UNW_FP_REG: c_int = 12;
#[cfg(any(target_arch = "riscv32"))]
const UNWIND_DATA_REG: (i32, i32) = (10, 11); // X10, X11
#[cfg(any(target_arch = "riscv32"))]
const UNW_FP_REG: c_int = 2;
#[export_name="__artiq_personality"]
pub extern fn personality(version: c_int,
actions: uw::_Unwind_Action,
_actions: uw::_Unwind_Action,
uw_exception_class: uw::_Unwind_Exception_Class,
uw_exception: *mut uw::_Unwind_Exception,
context: *mut uw::_Unwind_Context)
@ -85,32 +122,20 @@ pub extern fn personality(version: c_int,
get_data_start: &|| uw::_Unwind_GetDataRelBase(context),
};
let exception_info = &mut *(uw_exception as *mut ExceptionInfo);
let exception = &exception_info.exception.unwrap();
let index = EXCEPTION_BUFFER.exception_stack[EXCEPTION_BUFFER.exception_count - 1];
assert!(index != -1);
let exception = EXCEPTION_BUFFER.exceptions[index as usize].as_ref().unwrap();
let id = exception.id;
let name_ptr = exception.name.as_ptr();
let len = exception.name.len();
let eh_action = match dwarf::find_eh_action(lsda, &eh_context, name_ptr, len) {
let eh_action = match dwarf::find_eh_action(lsda, &eh_context, id) {
Ok(action) => action,
Err(_) => return uw::_URC_FATAL_PHASE1_ERROR,
};
if actions as u32 & uw::_UA_SEARCH_PHASE as u32 != 0 {
match eh_action {
EHAction::None |
EHAction::Cleanup(_) => return uw::_URC_CONTINUE_UNWIND,
EHAction::Catch(_) => return uw::_URC_HANDLER_FOUND,
EHAction::Terminate => return uw::_URC_FATAL_PHASE1_ERROR,
}
} else {
match eh_action {
EHAction::None => return uw::_URC_CONTINUE_UNWIND,
EHAction::Cleanup(lpad) |
EHAction::Catch(lpad) => {
if actions as u32 & uw::_UA_HANDLER_FRAME as u32 != 0 {
exception_info.handled = true
}
// Pass a pair of the unwinder exception and ARTIQ exception
// (which immediately follows).
uw::_Unwind_SetGR(context, UNWIND_DATA_REG.0,
@ -124,94 +149,209 @@ pub extern fn personality(version: c_int,
}
}
}
#[export_name="__artiq_raise"]
#[unwind(allowed)]
pub unsafe extern fn raise(exception: *const Exception) -> ! {
let count = EXCEPTION_BUFFER.exception_count;
let stack = &mut EXCEPTION_BUFFER.exception_stack;
let diff = exception as isize - EXCEPTION_BUFFER.exceptions.as_ptr() as isize;
if 0 <= diff && diff <= (mem::size_of::<Option<Exception>>() * MAX_INFLIGHT_EXCEPTIONS) as isize {
let index = diff / (mem::size_of::<Option<Exception>>() as isize);
let mut found = false;
for i in 0..=MAX_INFLIGHT_EXCEPTIONS + 1 {
if found {
if stack[i] == -1 {
stack[i - 1] = index;
assert!(i == count);
break;
} else {
stack[i - 1] = stack[i];
}
} else {
if stack[i] == index {
found = true;
}
}
}
assert!(found);
let _result = _Unwind_ForcedUnwind(&mut EXCEPTION_BUFFER.uw_exceptions[stack[count - 1] as usize],
stop_fn, core::ptr::null_mut());
} else {
if count < MAX_INFLIGHT_EXCEPTIONS {
let exception = &*exception;
for (i, slot) in EXCEPTION_BUFFER.exceptions.iter_mut().enumerate() {
// we should always be able to find a slot
if slot.is_none() {
*slot = Some(
*mem::transmute::<*const Exception, *const Exception<'static>>
(exception));
EXCEPTION_BUFFER.exception_stack[count] = i as isize;
EXCEPTION_BUFFER.uw_exceptions[i].private =
[0; uw::unwinder_private_data_size];
EXCEPTION_BUFFER.stack_pointers[i] = StackPointerBacktrace {
stack_pointer: 0,
initial_backtrace_size: EXCEPTION_BUFFER.backtrace_size,
current_backtrace_size: 0,
};
EXCEPTION_BUFFER.exception_count += 1;
let _result = _Unwind_ForcedUnwind(&mut EXCEPTION_BUFFER.uw_exceptions[i],
stop_fn, core::ptr::null_mut());
}
}
} else {
// TODO: better reporting?
let exception = Exception {
id: get_exception_id("RuntimeError"),
file: file!().as_c_slice(),
line: line!(),
column: column!(),
// https://github.com/rust-lang/rfcs/pull/1719
function: "__artiq_raise".as_c_slice(),
message: "too many nested exceptions".as_c_slice(),
param: [0, 0, 0]
};
EXCEPTION_BUFFER.exceptions[MAX_INFLIGHT_EXCEPTIONS] = Some(mem::transmute(exception));
EXCEPTION_BUFFER.stack_pointers[MAX_INFLIGHT_EXCEPTIONS] = Default::default();
EXCEPTION_BUFFER.exception_count += 1;
uncaught_exception()
}
}
unreachable!();
}
#[export_name="__artiq_resume"]
#[unwind(allowed)]
pub unsafe extern fn resume() -> ! {
assert!(EXCEPTION_BUFFER.exception_count != 0);
let i = EXCEPTION_BUFFER.exception_stack[EXCEPTION_BUFFER.exception_count - 1];
assert!(i != -1);
let _result = _Unwind_ForcedUnwind(&mut EXCEPTION_BUFFER.uw_exceptions[i as usize],
stop_fn, core::ptr::null_mut());
unreachable!()
}
#[export_name="__artiq_end_catch"]
#[unwind(allowed)]
pub unsafe extern fn end_catch() {
let mut count = EXCEPTION_BUFFER.exception_count;
assert!(count != 0);
// we remove all exceptions with SP <= current exception SP
// i.e. the outer exception escapes the finally block
let index = EXCEPTION_BUFFER.exception_stack[count - 1] as usize;
EXCEPTION_BUFFER.exception_stack[count - 1] = -1;
EXCEPTION_BUFFER.exceptions[index] = None;
let outer_sp = EXCEPTION_BUFFER.stack_pointers
[index].stack_pointer;
count -= 1;
for i in (0..count).rev() {
let index = EXCEPTION_BUFFER.exception_stack[i];
assert!(index != -1);
let index = index as usize;
let sp = EXCEPTION_BUFFER.stack_pointers[index].stack_pointer;
if sp >= outer_sp {
break;
}
EXCEPTION_BUFFER.exceptions[index] = None;
EXCEPTION_BUFFER.exception_stack[i] = -1;
count -= 1;
}
EXCEPTION_BUFFER.exception_count = count;
EXCEPTION_BUFFER.backtrace_size = if count > 0 {
let index = EXCEPTION_BUFFER.exception_stack[count - 1];
assert!(index != -1);
EXCEPTION_BUFFER.stack_pointers[index as usize].current_backtrace_size
} else {
0
};
}
extern fn cleanup(_unwind_code: uw::_Unwind_Reason_Code,
uw_exception: *mut uw::_Unwind_Exception) {
_uw_exception: *mut uw::_Unwind_Exception) {
unimplemented!()
}
fn uncaught_exception() -> ! {
unsafe {
let exception_info = &mut *(uw_exception as *mut ExceptionInfo);
exception_info.exception = None;
// dump way to reorder the stack
for i in 0..EXCEPTION_BUFFER.exception_count {
if EXCEPTION_BUFFER.exception_stack[i] != i as isize {
// find the correct index
let index = EXCEPTION_BUFFER.exception_stack
.iter()
.position(|v| *v == i as isize).unwrap();
let a = EXCEPTION_BUFFER.exception_stack[index];
let b = EXCEPTION_BUFFER.exception_stack[i];
assert!(a != -1 && b != -1);
core::mem::swap(&mut EXCEPTION_BUFFER.exception_stack[index],
&mut EXCEPTION_BUFFER.exception_stack[i]);
core::mem::swap(&mut EXCEPTION_BUFFER.exceptions[a as usize],
&mut EXCEPTION_BUFFER.exceptions[b as usize]);
core::mem::swap(&mut EXCEPTION_BUFFER.stack_pointers[a as usize],
&mut EXCEPTION_BUFFER.stack_pointers[b as usize]);
}
}
}
unsafe {
::terminate(
EXCEPTION_BUFFER.exceptions[..EXCEPTION_BUFFER.exception_count].as_ref(),
EXCEPTION_BUFFER.stack_pointers[..EXCEPTION_BUFFER.exception_count].as_ref(),
EXCEPTION_BUFFER.backtrace[..EXCEPTION_BUFFER.backtrace_size].as_mut())
}
}
extern fn uncaught_exception(_version: c_int,
// stop function which would be executed when we unwind each frame
extern fn stop_fn(_version: c_int,
actions: uw::_Unwind_Action,
_uw_exception_class: uw::_Unwind_Exception_Class,
uw_exception: *mut uw::_Unwind_Exception,
_uw_exception: *mut uw::_Unwind_Exception,
context: *mut uw::_Unwind_Context,
_stop_parameter: *mut c_void)
-> uw::_Unwind_Reason_Code {
_stop_parameter: *mut c_void) -> uw::_Unwind_Reason_Code {
unsafe {
let exception_info = &mut *(uw_exception as *mut ExceptionInfo);
if exception_info.backtrace_size < exception_info.backtrace.len() {
let backtrace_size = EXCEPTION_BUFFER.backtrace_size;
if backtrace_size < MAX_BACKTRACE_SIZE {
let ip = uw::_Unwind_GetIP(context);
exception_info.backtrace[exception_info.backtrace_size] = ip;
exception_info.backtrace_size += 1;
let fp = uw::_Unwind_GetGR(context, UNW_FP_REG);
if PAYLOAD_ADDRESS == 0 || ip > PAYLOAD_ADDRESS {
let ip = ip - PAYLOAD_ADDRESS;
EXCEPTION_BUFFER.backtrace[backtrace_size] = (ip, fp);
EXCEPTION_BUFFER.backtrace_size += 1;
let last_index = EXCEPTION_BUFFER.exception_stack[EXCEPTION_BUFFER.exception_count - 1];
assert!(last_index != -1);
let sp_info = &mut EXCEPTION_BUFFER.stack_pointers[last_index as usize];
sp_info.stack_pointer = fp;
sp_info.current_backtrace_size = backtrace_size + 1;
}
}
if actions as u32 & uw::_UA_END_OF_STACK as u32 != 0 {
::terminate(&exception_info.exception.unwrap(),
exception_info.backtrace[..exception_info.backtrace_size].as_mut())
uncaught_exception()
} else {
uw::_URC_NO_REASON
}
}
}
// We can unfortunately not use mem::zeroed in a static, so Option<> is used as a workaround.
// See https://github.com/rust-lang/rust/issues/39498.
static mut INFLIGHT: ExceptionInfo = ExceptionInfo {
uw_exception: uw::_Unwind_Exception {
exception_class: EXCEPTION_CLASS,
exception_cleanup: cleanup,
private: [0; uw::unwinder_private_data_size],
},
exception: None,
handled: true,
backtrace: [0; MAX_BACKTRACE_SIZE],
backtrace_size: 0
};
static EXCEPTION_ID_LOOKUP: [(&str, u32); 10] = [
("RuntimeError", 0),
("RTIOUnderflow", 1),
("RTIOOverflow", 2),
("RTIODestinationUnreachable", 3),
("DMAError", 4),
("I2CError", 5),
("CacheError", 6),
("SPIError", 7),
("ZeroDivisionError", 8),
("IndexError", 9)
];
#[export_name="__artiq_raise"]
#[unwind(allowed)]
pub unsafe extern fn raise(exception: *const Exception) -> ! {
// Zing! The Exception<'a> to Exception<'static> transmute is not really sound in case
// the exception is ever captured. Fortunately, they currently aren't, and we save
// on the hassle of having to allocate exceptions somewhere except on stack.
INFLIGHT.exception = Some(mem::transmute::<Exception, Exception<'static>>(*exception));
INFLIGHT.handled = false;
let result = uw::_Unwind_RaiseException(&mut INFLIGHT.uw_exception);
assert!(result == uw::_URC_END_OF_STACK);
INFLIGHT.backtrace_size = 0;
let _result = _Unwind_ForcedUnwind(&mut INFLIGHT.uw_exception,
uncaught_exception, ptr::null_mut());
unreachable!()
}
#[export_name="__artiq_reraise"]
#[unwind(allowed)]
pub unsafe extern fn reraise() -> ! {
use cslice::AsCSlice;
if INFLIGHT.handled {
match INFLIGHT.exception {
Some(ref exception) => raise(exception),
None => raise(&Exception {
name: "0:artiq.coredevice.exceptions.RuntimeError".as_c_slice(),
file: file!().as_c_slice(),
line: line!(),
column: column!(),
// https://github.com/rust-lang/rfcs/pull/1719
function: "__artiq_reraise".as_c_slice(),
message: "No active exception to reraise".as_c_slice(),
param: [0, 0, 0]
})
}
} else {
uw::_Unwind_Resume(&mut INFLIGHT.uw_exception)
pub fn get_exception_id(name: &str) -> u32 {
for (n, id) in EXCEPTION_ID_LOOKUP.iter() {
if *n == name {
return *id
}
}
unimplemented!("unallocated internal exception id")
}

40
artiq/firmware/ksupport/lib.rs
View File

@ -1,5 +1,5 @@
#![feature(lang_items, llvm_asm, panic_unwind, libc, unwind_attributes,
panic_info_message, nll)]
panic_info_message, nll, const_in_array_repeat_expressions)]
#![no_std]
extern crate libc;
@ -80,8 +80,9 @@ macro_rules! println {
macro_rules! raise {
($name:expr, $message:expr, $param0:expr, $param1:expr, $param2:expr) => ({
use cslice::AsCSlice;
let name_id = $crate::eh_artiq::get_exception_id($name);
let exn = $crate::eh_artiq::Exception {
name: concat!("0:artiq.coredevice.exceptions.", $name).as_c_slice(),
id: name_id,
file: file!().as_c_slice(),
line: line!(),
column: column!(),
@ -164,12 +165,12 @@ extern fn rpc_recv(slot: *mut ()) -> usize {
&Err(ref exception) =>
unsafe {
eh_artiq::raise(&eh_artiq::Exception {
name: exception.name.as_bytes().as_c_slice(),
file: exception.file.as_bytes().as_c_slice(),
id: exception.id,
file: exception.file,
line: exception.line,
column: exception.column,
function: exception.function.as_bytes().as_c_slice(),
message: exception.message.as_bytes().as_c_slice(),
function: exception.function,
message: exception.message,
param: exception.param
})
}
@ -177,27 +178,13 @@ extern fn rpc_recv(slot: *mut ()) -> usize {
})
}
fn terminate(exception: &eh_artiq::Exception, backtrace: &mut [usize]) -> ! {
let mut cursor = 0;
for index in 0..backtrace.len() {
if backtrace[index] > kernel_proto::KERNELCPU_PAYLOAD_ADDRESS {
backtrace[cursor] = backtrace[index] - kernel_proto::KERNELCPU_PAYLOAD_ADDRESS;
cursor += 1;
}
}
let backtrace = &mut backtrace.as_mut()[0..cursor];
fn terminate(exceptions: &'static [Option<eh_artiq::Exception<'static>>],
stack_pointers: &'static [eh_artiq::StackPointerBacktrace],
backtrace: &mut [(usize, usize)]) -> ! {
send(&RunException {
exception: kernel_proto::Exception {
name: str::from_utf8(exception.name.as_ref()).unwrap(),
file: str::from_utf8(exception.file.as_ref()).unwrap(),
line: exception.line,
column: exception.column,
function: str::from_utf8(exception.function.as_ref()).unwrap(),
message: str::from_utf8(exception.message.as_ref()).unwrap(),
param: exception.param,
},
backtrace: backtrace
exceptions,
stack_pointers,
backtrace
});
loop {}
}
@ -472,6 +459,7 @@ unsafe fn attribute_writeback(typeinfo: *const ()) {
#[no_mangle]
pub unsafe fn main() {
eh_artiq::reset_exception_buffer(KERNELCPU_PAYLOAD_ADDRESS);
let image = slice::from_raw_parts_mut(kernel_proto::KERNELCPU_PAYLOAD_ADDRESS as *mut u8,
kernel_proto::KERNELCPU_LAST_ADDRESS -
kernel_proto::KERNELCPU_PAYLOAD_ADDRESS);

17
artiq/firmware/libeh/dwarf.rs
View File

@ -202,8 +202,7 @@ unsafe fn get_ttype_entry(
pub unsafe fn find_eh_action(
lsda: *const u8,
context: &EHContext<'_>,
name: *const u8,
len: usize,
id: u32,
) -> Result<EHAction, ()> {
if lsda.is_null() {
return Ok(EHAction::None);
@ -275,20 +274,10 @@ pub unsafe fn find_eh_action(
return Ok(EHAction::Catch(lpad));
}
// this seems to be target dependent
let clause_ptr = *(catch_type as *const CSlice<u8>);
let clause_name_ptr = (clause_ptr).as_ptr();
let clause_name_len = (clause_ptr).len();
if clause_name_len == len {
if (clause_name_ptr == core::ptr::null() ||
clause_name_ptr == name ||
// somehow their name pointers might differ, but the content is the
// same
core::slice::from_raw_parts(clause_name_ptr, clause_name_len) ==
core::slice::from_raw_parts(name, len))
{
let clause_id = *(catch_type as *const u32);
if clause_id == id {
return Ok(EHAction::Catch(lpad));
}
}
} else if ar_filter < 0 {
// FIXME: how to handle this?
break;

47
artiq/firmware/libeh/eh_artiq.rs Normal file
View File

@ -0,0 +1,47 @@
// ARTIQ Exception struct declaration
use cslice::CSlice;
// Note: CSlice within an exception may not be actual cslice, they may be strings that exist only
// in the host. If the length == usize:MAX, the pointer is actually a string key in the host.
#[repr(C)]
#[derive(Copy, Clone)]
pub struct Exception<'a> {
pub id: u32,
pub file: CSlice<'a, u8>,
pub line: u32,
pub column: u32,
pub function: CSlice<'a, u8>,
pub message: CSlice<'a, u8>,
pub param: [i64; 3]
}
fn str_err(_: core::str::Utf8Error) -> core::fmt::Error {
core::fmt::Error
}
fn exception_str<'a>(s: &'a CSlice<'a, u8>) -> Result<&'a str, core::str::Utf8Error> {
if s.len() == usize::MAX {
Ok("<host string>")
} else {
core::str::from_utf8(s.as_ref())
}
}
impl<'a> core::fmt::Debug for Exception<'a> {
fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result {
write!(f, "Exception {} from {} in {}:{}:{}, message: {}",
self.id,
exception_str(&self.function).map_err(str_err)?,
exception_str(&self.file).map_err(str_err)?,
self.line, self.column,
exception_str(&self.message).map_err(str_err)?)
}
}
#[derive(Copy, Clone, Debug, Default)]
pub struct StackPointerBacktrace {
pub stack_pointer: usize,
pub initial_backtrace_size: usize,
pub current_backtrace_size: usize,
}

2
artiq/firmware/libeh/eh_rust.rs
View File

@ -87,5 +87,5 @@ unsafe fn find_eh_action(context: *mut uw::_Unwind_Context) -> Result<EHAction,
get_text_start: &|| uw::_Unwind_GetTextRelBase(context),
get_data_start: &|| uw::_Unwind_GetDataRelBase(context),
};
dwarf::find_eh_action(lsda, &eh_context, core::ptr::null(), 0)
dwarf::find_eh_action(lsda, &eh_context, 0)
}

1
artiq/firmware/libeh/lib.rs
View File

@ -7,3 +7,4 @@ extern crate libc;
pub mod dwarf;
pub mod eh_rust;
pub mod eh_artiq;

1
artiq/firmware/libproto_artiq/Cargo.toml
View File

@ -15,6 +15,7 @@ cslice = { version = "0.3" }
log = { version = "0.4", default-features = false, optional = true }
io = { path = "../libio", features = ["byteorder"] }
dyld = { path = "../libdyld" }
eh = { path = "../libeh" }
[features]
alloc = ["io/alloc"]

18
artiq/firmware/libproto_artiq/kernel_proto.rs
View File

@ -6,17 +6,6 @@ pub const KERNELCPU_PAYLOAD_ADDRESS: usize = 0x45060000;
pub const KERNELCPU_LAST_ADDRESS: usize = 0x4fffffff;
pub const KSUPPORT_HEADER_SIZE: usize = 0x80;
#[derive(Debug, Clone)]
pub struct Exception<'a> {
pub name: &'a str,
pub file: &'a str,
pub line: u32,
pub column: u32,
pub function: &'a str,
pub message: &'a str,
pub param: [i64; 3]
}
#[derive(Debug)]
pub enum Message<'a> {
LoadRequest(&'a [u8]),
@ -47,8 +36,9 @@ pub enum Message<'a> {
RunFinished,
RunException {
exception: Exception<'a>,
backtrace: &'a [usize]
exceptions: &'a [Option<eh::eh_artiq::Exception<'a>>],
stack_pointers: &'a [eh::eh_artiq::StackPointerBacktrace],
backtrace: &'a [(usize, usize)]
},
RunAborted,
@ -59,7 +49,7 @@ pub enum Message<'a> {
data: *const *const ()
},
RpcRecvRequest(*mut ()),
RpcRecvReply(Result<usize, Exception<'a>>),
RpcRecvReply(Result<usize, eh::eh_artiq::Exception<'a>>),
RpcFlush,
CacheGetRequest { key: &'a str },

1
artiq/firmware/libproto_artiq/lib.rs
View File

@ -13,6 +13,7 @@ extern crate log;
extern crate byteorder;
extern crate io;
extern crate dyld;
extern crate eh;
// Internal protocols.
pub mod kernel_proto;

81
artiq/firmware/libproto_artiq/session_proto.rs
View File

@ -1,5 +1,7 @@
use core::str::Utf8Error;
use alloc::{vec::Vec, string::String};
use alloc::vec::Vec;
use eh::eh_artiq::{Exception, StackPointerBacktrace};
use cslice::CSlice;
use io::{Read, ProtoRead, Write, ProtoWrite, Error as IoError, ReadStringError};
@ -70,13 +72,13 @@ pub enum Request {
RpcReply { tag: Vec<u8> },
RpcException {
name: String,
message: String,
id: u32,
message: u32,
param: [i64; 3],
file: String,
file: u32,
line: u32,
column: u32,
function: String,
function: u32,
},
}
@ -95,14 +97,9 @@ pub enum Reply<'a> {
},
KernelStartupFailed,
KernelException {
name: &'a str,
message: &'a str,
param: [i64; 3],
file: &'a str,
line: u32,
column: u32,
function: &'a str,
backtrace: &'a [usize],
exceptions: &'a [Option<Exception<'a>>],
stack_pointers: &'a [StackPointerBacktrace],
backtrace: &'a [(usize, usize)],
async_errors: u8
},
@ -126,15 +123,15 @@ impl Request {
tag: reader.read_bytes()?
},
8 => Request::RpcException {
name: reader.read_string()?,
message: reader.read_string()?,
id: reader.read_u32()?,
message: reader.read_u32()?,
param: [reader.read_u64()? as i64,
reader.read_u64()? as i64,
reader.read_u64()? as i64],
file: reader.read_string()?,
file: reader.read_u32()?,
line: reader.read_u32()?,
column: reader.read_u32()?,
function: reader.read_string()?
function: reader.read_u32()?
},
ty => return Err(Error::UnknownPacket(ty))
@ -142,6 +139,18 @@ impl Request {
}
}
fn write_exception_string<'a, W>(writer: &mut W, s: &CSlice<'a, u8>) -> Result<(), IoError<W::WriteError>>
where W: Write + ?Sized
{
if s.len() == usize::MAX {
writer.write_u32(u32::MAX)?;
writer.write_u32(s.as_ptr() as u32)?;
} else {
writer.write_string(core::str::from_utf8(s.as_ref()).unwrap())?;
}
Ok(())
}
impl<'a> Reply<'a> {
pub fn write_to<W>(&self, writer: &mut W) -> Result<(), IoError<W::WriteError>>
where W: Write + ?Sized
@ -171,22 +180,36 @@ impl<'a> Reply<'a> {
writer.write_u8(8)?;
},
Reply::KernelException {
name, message, param, file, line, column, function, backtrace,
exceptions,
stack_pointers,
backtrace,
async_errors
} => {
writer.write_u8(9)?;
writer.write_string(name)?;
writer.write_string(message)?;
writer.write_u64(param[0] as u64)?;
writer.write_u64(param[1] as u64)?;
writer.write_u64(param[2] as u64)?;
writer.write_string(file)?;
writer.write_u32(line)?;
writer.write_u32(column)?;
writer.write_string(function)?;
writer.write_u32(exceptions.len() as u32)?;
for exception in exceptions.iter() {
let exception = exception.as_ref().unwrap();
writer.write_u32(exception.id as u32)?;
write_exception_string(writer, &exception.message)?;
writer.write_u64(exception.param[0] as u64)?;
writer.write_u64(exception.param[1] as u64)?;
writer.write_u64(exception.param[2] as u64)?;
write_exception_string(writer, &exception.file)?;
writer.write_u32(exception.line)?;
writer.write_u32(exception.column)?;
write_exception_string(writer, &exception.function)?;
}
for sp in stack_pointers.iter() {
writer.write_u32(sp.stack_pointer as u32)?;
writer.write_u32(sp.initial_backtrace_size as u32)?;
writer.write_u32(sp.current_backtrace_size as u32)?;
}
writer.write_u32(backtrace.len() as u32)?;
for &addr in backtrace {
writer.write_u32(addr as u32)?
for &(addr, sp) in backtrace {
writer.write_u32(addr as u32)?;
writer.write_u32(sp as u32)?;
}
writer.write_u8(async_errors)?;
},

32
artiq/firmware/runtime/session.rs
View File

@ -1,6 +1,7 @@
use core::{mem, str, cell::{Cell, RefCell}, fmt::Write as FmtWrite};
use alloc::{vec::Vec, string::String};
use byteorder::{ByteOrder, NativeEndian};
use cslice::CSlice;
use io::{Read, Write, Error as IoError};
use board_misoc::{ident, cache, config};
@ -291,7 +292,7 @@ fn process_host_message(io: &Io,
}
host::Request::RpcException {
name, message, param, file, line, column, function
id, message, param, file, line, column, function
} => {
if session.kernel_state != KernelState::RpcWait {
unexpected!("unsolicited RPC reply")
@ -305,16 +306,18 @@ fn process_host_message(io: &Io,
}
})?;
let exn = kern::Exception {
name: name.as_ref(),
message: message.as_ref(),
unsafe {
let exn = eh::eh_artiq::Exception {
id: id,
message: CSlice::new(message as *const u8, usize::MAX),
param: param,
file: file.as_ref(),
file: CSlice::new(file as *const u8, usize::MAX),
line: line,
column: column,
function: function.as_ref()
function: CSlice::new(function as *const u8, usize::MAX),
};
kern_send(io, &kern::RpcRecvReply(Err(exn)))?;
}
session.kernel_state = KernelState::Running
}
@ -438,7 +441,8 @@ fn process_kern_message(io: &Io, aux_mutex: &Mutex,
}
}
&kern::RunException {
exception: kern::Exception { name, message, param, file, line, column, function },
exceptions,
stack_pointers,
backtrace
} => {
unsafe { kernel::stop() }
@ -448,19 +452,15 @@ fn process_kern_message(io: &Io, aux_mutex: &Mutex,
match stream {
None => {
error!("exception in flash kernel");
error!("{}: {} {:?}", name, message, param);
error!("at {}:{}:{} in {}", file, line, column, function);
for exception in exceptions {
error!("{:?}", exception.unwrap());
}
return Ok(true)
},
Some(ref mut stream) => {
host_write(stream, host::Reply::KernelException {
name: name,
message: message,
param: param,
file: file,
line: line,
column: column,
function: function,
exceptions: exceptions,
stack_pointers: stack_pointers,
backtrace: backtrace,
async_errors: unsafe { get_async_errors() }
}).map_err(|e| e.into())

13
artiq/language/environment.py
View File

@ -330,8 +330,7 @@ class HasEnvironment:
@rpc(flags={"async"})
def set_dataset(self, key, value,
broadcast=False, persist=False, archive=True,
hdf5_options=None):
broadcast=False, persist=False, archive=True):
"""Sets the contents and handling modes of a dataset.
Datasets must be scalars (``bool``, ``int``, ``float`` or NumPy scalar)
@ -343,16 +342,8 @@ class HasEnvironment:
broadcast.
:param archive: the data is saved into the local storage of the current
run (archived as a HDF5 file).
:param hdf5_options: dict of keyword arguments to pass to
:meth:`h5py.Group.create_dataset`. For example, pass ``{"compression": "gzip"}``
to enable transparent zlib compression of this dataset in the HDF5 archive.
See the `h5py documentation <https://docs.h5py.org/en/stable/high/group.html#h5py.Group.create_dataset>`_
for a list of valid options.
"""
self.__dataset_mgr.set(
key, value, broadcast, persist, archive, hdf5_options
)
self.__dataset_mgr.set(key, value, broadcast, persist, archive)
@rpc(flags={"async"})
def mutate_dataset(self, key, index, value):

39
artiq/master/databases.py
View File

@ -35,15 +35,6 @@ class DeviceDB:
return desc
def make_dataset(*, persist=False, value=None, hdf5_options=None):
"PYON-serializable representation of a dataset in the DatasetDB"
return {
"persist": persist,
"value": value,
"hdf5_options": hdf5_options or {},
}
class DatasetDB(TaskObject):
def __init__(self, persist_file, autosave_period=30):
self.persist_file = persist_file
@ -53,23 +44,10 @@ class DatasetDB(TaskObject):
file_data = pyon.load_file(self.persist_file)
except FileNotFoundError:
file_data = dict()
self.data = Notifier(
{
k: make_dataset(
persist=True,
value=v["value"],
hdf5_options=v["hdf5_options"]
)
for k, v in file_data.items()
}
)
self.data = Notifier({k: (True, v) for k, v in file_data.items()})
def save(self):
data = {
k: d
for k, d in self.data.raw_view.items()
if d["persist"]
}
data = {k: v[1] for k, v in self.data.raw_view.items() if v[0]}
pyon.store_file(self.persist_file, data)
async def _do(self):
@ -81,23 +59,20 @@ class DatasetDB(TaskObject):
self.save()
def get(self, key):
return self.data.raw_view[key]
return self.data.raw_view[key][1]
def update(self, mod):
process_mod(self.data, mod)
# convenience functions (update() can be used instead)
def set(self, key, value, persist=None, hdf5_options=None):
def set(self, key, value, persist=None):
if persist is None:
if key in self.data.raw_view:
persist = self.data.raw_view[key]["persist"]
persist = self.data.raw_view[key][0]
else:
persist = False
self.data[key] = make_dataset(
persist=persist,
value=value,
hdf5_options=hdf5_options,
)
self.data[key] = (persist, value)
def delete(self, key):
del self.data[key]
#

36
artiq/master/worker_db.py
View File

@ -8,12 +8,9 @@ from operator import setitem
import importlib
import logging
import numpy as np
from sipyco.sync_struct import Notifier
from sipyco.pc_rpc import AutoTarget, Client, BestEffortClient
from artiq.master.databases import make_dataset
logger = logging.getLogger(__name__)
@ -118,26 +115,17 @@ class DatasetManager:
self.ddb = ddb
self._broadcaster.publish = ddb.update
def set(self, key, value, broadcast=False, persist=False, archive=True,
hdf5_options=None):
def set(self, key, value, broadcast=False, persist=False, archive=True):
if persist:
broadcast = True
if broadcast:
self._broadcaster[key] = make_dataset(
persist=persist,
value=value,
hdf5_options=hdf5_options,
)
self._broadcaster[key] = persist, value
elif key in self._broadcaster.raw_view:
del self._broadcaster[key]
if archive:
self.local[key] = make_dataset(
persist=persist,
value=value,
hdf5_options=hdf5_options,
)
self.local[key] = value
elif key in self.local:
del self.local[key]
@ -145,11 +133,11 @@ class DatasetManager:
target = self.local.get(key, None)
if key in self._broadcaster.raw_view:
if target is not None:
assert target["value"] is self._broadcaster.raw_view[key]["value"]
return self._broadcaster[key]["value"]
assert target is self._broadcaster.raw_view[key][1]
return self._broadcaster[key][1]
if target is None:
raise KeyError("Cannot mutate nonexistent dataset '{}'".format(key))
return target["value"]
return target
def mutate(self, key, index, value):
target = self._get_mutation_target(key)
@ -165,15 +153,15 @@ class DatasetManager:
def get(self, key, archive=False):
if key in self.local:
return self.local[key]["value"]
return self.local[key]
dataset = self.ddb.get(key)
data = self.ddb.get(key)
if archive:
if key in self.archive:
logger.warning("Dataset '%s' is already in archive, "
"overwriting", key, stack_info=True)
self.archive[key] = dataset
return dataset["value"]
self.archive[key] = data
return data
def write_hdf5(self, f):
datasets_group = f.create_group("datasets")
@ -189,7 +177,7 @@ def _write(group, k, v):
# Add context to exception message when the user writes a dataset that is
# not representable in HDF5.
try:
group.create_dataset(k, data=v["value"], **v["hdf5_options"])
group[k] = v
except TypeError as e:
raise TypeError("Error writing dataset '{}' of type '{}': {}".format(
k, type(v["value"]), e))
k, type(v), e))

63
artiq/test/coredevice/test_portability.py
View File

@ -125,6 +125,57 @@ class _Pulses(EnvExperiment):
class _MyException(Exception):
pass
class _NestedFinally(EnvExperiment):
def build(self, trace):
self.setattr_device("core")
self.trace = trace
def _trace(self, i):
self.trace.append(i)
@kernel
def run(self):
try:
try:
raise ValueError
finally:
try:
raise IndexError()
except ValueError:
self._trace(0)
except:
self._trace(1)
finally:
self._trace(2)
class _NestedExceptions(EnvExperiment):
def build(self, trace):
self.setattr_device("core")
self.trace = trace
def _trace(self, i):
self.trace.append(i)
@kernel
def run(self):
try:
try:
raise ValueError
except _MyException:
self._trace(0)
raise
finally:
try:
raise IndexError()
except ValueError:
self._trace(1)
raise
except IndexError:
self._trace(2)
except:
self._trace(3)
finally:
self._trace(4)
class _Exceptions(EnvExperiment):
def build(self, trace):
@ -253,6 +304,18 @@ class HostVsDeviceCase(ExperimentCase):
_run_on_host(_Exceptions, trace=t_host)
self.assertEqual(t_device, t_host)
def test_nested_finally(self):
t_device, t_host = [], []
self.execute(_NestedFinally, trace=t_device)
_run_on_host(_NestedFinally, trace=t_host)
self.assertEqual(t_device, t_host)
def test_nested_exceptions(self):
t_device, t_host = [], []
self.execute(_NestedExceptions, trace=t_device)
_run_on_host(_NestedExceptions, trace=t_host)
self.assertEqual(t_device, t_host)
def test_rpc_exceptions(self):
for f in self.execute, _run_on_host:
with self.assertRaises(_MyException):

50
artiq/test/test_datasets.py
View File

@ -3,9 +3,6 @@
import copy
import unittest
import h5py
import numpy as np
from sipyco.sync_struct import process_mod
from artiq.experiment import EnvExperiment
@ -17,7 +14,7 @@ class MockDatasetDB:
self.data = dict()
def get(self, key):
return self.data[key]["value"]
return self.data[key][1]
def update(self, mod):
# Copy mod before applying to avoid sharing references to objects
@ -85,9 +82,9 @@ class ExperimentDatasetCase(unittest.TestCase):
def test_append_broadcast(self):
self.exp.set(KEY, [], broadcast=True)
self.exp.append(KEY, 0)
self.assertEqual(self.dataset_db.data[KEY]["value"], [0])
self.assertEqual(self.dataset_db.data[KEY][1], [0])
self.exp.append(KEY, 1)
self.assertEqual(self.dataset_db.data[KEY]["value"], [0, 1])
self.assertEqual(self.dataset_db.data[KEY][1], [0, 1])
def test_append_array(self):
for broadcast in (True, False):
@ -106,44 +103,3 @@ class ExperimentDatasetCase(unittest.TestCase):
with self.assertRaises(KeyError):
self.exp.append(KEY, 0)
def test_write_hdf5_options(self):
data = np.random.randint(0, 1024, 1024)
self.exp.set(
KEY,
data,
hdf5_options=dict(
compression="gzip",
compression_opts=6,
shuffle=True,
fletcher32=True
),
)
with h5py.File("test.h5", "a", "core", backing_store=False) as f:
self.dataset_mgr.write_hdf5(f)
self.assertTrue(np.array_equal(f["datasets"][KEY][()], data))
self.assertEqual(f["datasets"][KEY].compression, "gzip")
self.assertEqual(f["datasets"][KEY].compression_opts, 6)
self.assertTrue(f["datasets"][KEY].shuffle)
self.assertTrue(f["datasets"][KEY].fletcher32)
def test_write_hdf5_no_options(self):
data = np.random.randint(0, 1024, 1024)
self.exp.set(KEY, data)
with h5py.File("test.h5", "a", "core", backing_store=False) as f:
self.dataset_mgr.write_hdf5(f)
self.assertTrue(np.array_equal(f["datasets"][KEY][()], data))
self.assertIsNone(f["datasets"][KEY].compression)
def test_write_hdf5_invalid_type(self):
class CustomType:
def __init__(self, x):
self.x = x
self.exp.set(KEY, CustomType(42))
with h5py.File("test.h5", "w", "core", backing_store=False) as f:
with self.assertRaisesRegex(TypeError, "CustomType"):
self.dataset_mgr.write_hdf5(f)

10
artiq/test/test_scheduler.py
View File

@ -6,7 +6,6 @@ from time import time, sleep
from artiq.experiment import *
from artiq.master.scheduler import Scheduler
from artiq.master.databases import make_dataset
class EmptyExperiment(EnvExperiment):
@ -288,13 +287,8 @@ class SchedulerCase(unittest.TestCase):
nonlocal termination_ok
self.assertEqual(
mod,
{
"action": "setitem",
"key": "termination_ok",
"value": make_dataset(value=True),
"path": []
}
)
{"action": "setitem", "key": "termination_ok",
"value": (False, True), "path": []})
termination_ok = True
handlers = {
"update_dataset": check_termination

2
flake.nix
View File

@ -189,7 +189,7 @@
cargoDeps = rustPlatform.fetchCargoTarball {
name = "artiq-firmware-cargo-deps";
src = "${self}/artiq/firmware";
sha256 = "sha256-Lf6M4M/jdRiO5MsWSoqtOSNfRIhbze+qvg4kaiiBWW4=";
sha256 = "sha256-YyycMsDzR+JRcMZJd6A/CRi2J9nKmaWY/KXUnAQaZ00=";
};
nativeBuildInputs = [
(pkgs.python3.withPackages(ps: [ migen misoc artiq ]))