From 965dd89b61ceca3867269205bb00f23d3ef240ab Mon Sep 17 00:00:00 2001 From: Sebastien Bourdeauducq Date: Sat, 21 Feb 2015 14:28:18 -0700 Subject: [PATCH] master: reorganize result output code --- artiq/frontend/artiq_master.py | 2 +- artiq/master/results.py | 72 ++++++++++++++++++++++++++++ artiq/master/rt_results.py | 24 ---------- artiq/master/{db.py => worker_db.py} | 41 ++-------------- artiq/master/worker_impl.py | 13 ++--- 5 files changed, 79 insertions(+), 73 deletions(-) create mode 100644 artiq/master/results.py delete mode 100644 artiq/master/rt_results.py rename artiq/master/{db.py => worker_db.py} (67%) diff --git a/artiq/frontend/artiq_master.py b/artiq/frontend/artiq_master.py index 2b37717e2..d9fe9410b 100755 --- a/artiq/frontend/artiq_master.py +++ b/artiq/frontend/artiq_master.py @@ -9,7 +9,7 @@ from artiq.protocols.pc_rpc import Server from artiq.protocols.sync_struct import Publisher from artiq.protocols.file_db import FlatFileDB, SimpleHistory from artiq.master.scheduler import Scheduler -from artiq.master.rt_results import RTResults +from artiq.master.results import RTResults from artiq.master.repository import Repository from artiq.tools import verbosity_args, init_logger diff --git a/artiq/master/results.py b/artiq/master/results.py new file mode 100644 index 000000000..6235ad003 --- /dev/null +++ b/artiq/master/results.py @@ -0,0 +1,72 @@ +import os +import time + +import numpy +import h5py + +from artiq.protocols.sync_struct import Notifier, process_mod + + +def get_hdf5_output(start_time, rid, name): + dirname = os.path.join("results", + time.strftime("%y-%m-%d", start_time), + time.strftime("%H-%M", start_time)) + filename = "{:09}-{}.h5".format(rid, name) + os.makedirs(dirname, exist_ok=True) + return h5py.File(os.path.join(dirname, filename), "w") + + +_type_to_hdf5 = { + int: h5py.h5t.STD_I64BE, + float: h5py.h5t.IEEE_F64BE +} + +def result_dict_to_hdf5(f, rd): + for name, data in rd.items(): + if isinstance(data, list): + el_ty = type(data[0]) + for d in data: + if type(d) != el_ty: + raise TypeError("All list elements must have the same" + " type for HDF5 output") + try: + el_ty_h5 = _type_to_hdf5[el_ty] + except KeyError: + raise TypeError("List element type {} is not supported for" + " HDF5 output".format(el_ty)) + dataset = f.create_dataset(name, (len(data), ), el_ty_h5) + dataset[:] = data + elif isinstance(data, numpy.ndarray): + f.create_dataset(name, data=data) + else: + ty = type(data) + try: + ty_h5 = _type_to_hdf5[ty] + except KeyError: + raise TypeError("Type {} is not supported for HDF5 output" + .format(ty)) + dataset = f.create_dataset(name, (), ty_h5) + dataset[()] = data + + +class RTResults: + def __init__(self): + self.groups = Notifier(dict()) + self.current_group = "default" + + def init(self, description): + data = dict() + for rtr in description.keys(): + if isinstance(rtr, tuple): + for e in rtr: + data[e] = [] + else: + data[rtr] = [] + self.groups[self.current_group] = { + "description": description, + "data": data + } + + def update(self, mod): + target = self.groups[self.current_group]["data"] + process_mod(target, mod) diff --git a/artiq/master/rt_results.py b/artiq/master/rt_results.py deleted file mode 100644 index f046c6273..000000000 --- a/artiq/master/rt_results.py +++ /dev/null @@ -1,24 +0,0 @@ -from artiq.protocols.sync_struct import Notifier, process_mod - - -class RTResults: - def __init__(self): - self.groups = Notifier(dict()) - self.current_group = "default" - - def init(self, description): - data = dict() - for rtr in description.keys(): - if isinstance(rtr, tuple): - for e in rtr: - data[e] = [] - else: - data[rtr] = [] - self.groups[self.current_group] = { - "description": description, - "data": data - } - - def update(self, mod): - target = self.groups[self.current_group]["data"] - process_mod(target, mod) diff --git a/artiq/master/db.py b/artiq/master/worker_db.py similarity index 67% rename from artiq/master/db.py rename to artiq/master/worker_db.py index 2cad8b6f3..3b10463f1 100644 --- a/artiq/master/db.py +++ b/artiq/master/worker_db.py @@ -1,44 +1,9 @@ from collections import OrderedDict import importlib -import numpy -import h5py - from artiq.protocols.sync_struct import Notifier from artiq.protocols.pc_rpc import Client, BestEffortClient - - -_type_to_hdf5 = { - int: h5py.h5t.STD_I64BE, - float: h5py.h5t.IEEE_F64BE -} - -def _result_dict_to_hdf5(f, rd): - for name, data in rd.items(): - if isinstance(data, list): - el_ty = type(data[0]) - for d in data: - if type(d) != el_ty: - raise TypeError("All list elements must have the same" - " type for HDF5 output") - try: - el_ty_h5 = _type_to_hdf5[el_ty] - except KeyError: - raise TypeError("List element type {} is not supported for" - " HDF5 output".format(el_ty)) - dataset = f.create_dataset(name, (len(data), ), el_ty_h5) - dataset[:] = data - elif isinstance(data, numpy.ndarray): - f.create_dataset(name, data=data) - else: - ty = type(data) - try: - ty_h5 = _type_to_hdf5[ty] - except KeyError: - raise TypeError("Type {} is not supported for HDF5 output" - .format(ty)) - dataset = f.create_dataset(name, (), ty_h5) - dataset[()] = data +from artiq.master.results import result_dict_to_hdf5 class ResultDB: @@ -68,8 +33,8 @@ class ResultDB: self.data[name] = value def write_hdf5(self, f): - _result_dict_to_hdf5(f, self.realtime_data.read) - _result_dict_to_hdf5(f, self.data.read) + result_dict_to_hdf5(f, self.realtime_data.read) + result_dict_to_hdf5(f, self.data.read) def _create_device(desc, dbh): diff --git a/artiq/master/worker_impl.py b/artiq/master/worker_impl.py index 52784f67c..018819a98 100644 --- a/artiq/master/worker_impl.py +++ b/artiq/master/worker_impl.py @@ -1,14 +1,12 @@ import sys -import os import time from inspect import isclass import traceback -import h5py - from artiq.protocols import pyon from artiq.tools import file_import -from artiq.master.db import DBHub, ResultDB +from artiq.master.worker_db import DBHub, ResultDB +from artiq.master.results import get_hdf5_output def get_object(): @@ -118,12 +116,7 @@ def run(rid, run_params): finally: dbh.close() - dirname = os.path.join("results", - time.strftime("%y-%m-%d", start_time), - time.strftime("%H-%M", start_time)) - filename = "{:09}-{}.h5".format(rid, unit.__name__) - os.makedirs(dirname, exist_ok=True) - f = h5py.File(os.path.join(dirname, filename), "w") + f = get_hdf5_output(start_time, rid, unit.__name__) try: rdb.write_hdf5(f) finally: