forked from M-Labs/artiq
set_dataset: pass HDF5 options as a dict, not as loose kwargs
Signed-off-by: Etienne Wodey <wodey@iqo.uni-hannover.de>
This commit is contained in:
parent
12ef907f34
commit
8bedf278f0
@ -15,7 +15,7 @@ Highlights:
|
|||||||
- Exposes upconverter calibration and enabling/disabling of upconverter LO & RF outputs.
|
- Exposes upconverter calibration and enabling/disabling of upconverter LO & RF outputs.
|
||||||
* HDF5 options can now be passed when creating datasets with ``set_dataset``. This allows
|
* HDF5 options can now be passed when creating datasets with ``set_dataset``. This allows
|
||||||
in particular to use transparent compression filters as follows:
|
in particular to use transparent compression filters as follows:
|
||||||
``set_dataset(name, value, compression="gzip")``.
|
``set_dataset(name, value, hdf5_options={"compression": "gzip"})``.
|
||||||
|
|
||||||
|
|
||||||
Breaking changes:
|
Breaking changes:
|
||||||
|
@ -332,7 +332,7 @@ class HasEnvironment:
|
|||||||
@rpc(flags={"async"})
|
@rpc(flags={"async"})
|
||||||
def set_dataset(self, key, value,
|
def set_dataset(self, key, value,
|
||||||
broadcast=False, persist=False, archive=True, save=None,
|
broadcast=False, persist=False, archive=True, save=None,
|
||||||
**hdf5_options):
|
hdf5_options=None):
|
||||||
"""Sets the contents and handling modes of a dataset.
|
"""Sets the contents and handling modes of a dataset.
|
||||||
|
|
||||||
Datasets must be scalars (``bool``, ``int``, ``float`` or NumPy scalar)
|
Datasets must be scalars (``bool``, ``int``, ``float`` or NumPy scalar)
|
||||||
@ -345,8 +345,8 @@ class HasEnvironment:
|
|||||||
:param archive: the data is saved into the local storage of the current
|
:param archive: the data is saved into the local storage of the current
|
||||||
run (archived as a HDF5 file).
|
run (archived as a HDF5 file).
|
||||||
:param save: deprecated.
|
:param save: deprecated.
|
||||||
:param hdf5_options: additional keyword arguments are passed to
|
:param hdf5_options: dict of keyword arguments to pass to
|
||||||
:meth:`h5py.Group.create_dataset`. For example, pass ``compression="gzip"``
|
:meth:`h5py.Group.create_dataset`. For example, pass ``{"compression": "gzip"}``
|
||||||
to enable transparent zlib compression of this dataset in the HDF5 archive.
|
to enable transparent zlib compression of this dataset in the HDF5 archive.
|
||||||
See the `h5py documentation <https://docs.h5py.org/en/stable/high/group.html#h5py.Group.create_dataset>`_
|
See the `h5py documentation <https://docs.h5py.org/en/stable/high/group.html#h5py.Group.create_dataset>`_
|
||||||
for a list of valid options.
|
for a list of valid options.
|
||||||
|
@ -87,7 +87,7 @@ class DatasetDB(TaskObject):
|
|||||||
process_mod(self.data, mod)
|
process_mod(self.data, mod)
|
||||||
|
|
||||||
# convenience functions (update() can be used instead)
|
# convenience functions (update() can be used instead)
|
||||||
def set(self, key, value, persist=None, **hdf5_options):
|
def set(self, key, value, persist=None, hdf5_options=None):
|
||||||
if persist is None:
|
if persist is None:
|
||||||
if key in self.data.raw_view:
|
if key in self.data.raw_view:
|
||||||
persist = self.data.raw_view[key]["persist"]
|
persist = self.data.raw_view[key]["persist"]
|
||||||
|
@ -21,8 +21,8 @@ class TestDatasetDB(unittest.TestCase):
|
|||||||
self.ddb = DatasetDB(self.persist_file.name)
|
self.ddb = DatasetDB(self.persist_file.name)
|
||||||
|
|
||||||
self.ddb.set(KEY1, DATA, persist=True)
|
self.ddb.set(KEY1, DATA, persist=True)
|
||||||
self.ddb.set(KEY2, DATA, persist=True, compression=COMP)
|
self.ddb.set(KEY2, DATA, persist=True, hdf5_options=dict(compression=COMP))
|
||||||
self.ddb.set(KEY3, DATA, shuffle=True)
|
self.ddb.set(KEY3, DATA, hdf5_options=dict(shuffle=True))
|
||||||
|
|
||||||
self.save_ddb_to_disk()
|
self.save_ddb_to_disk()
|
||||||
|
|
||||||
|
@ -108,9 +108,16 @@ class ExperimentDatasetCase(unittest.TestCase):
|
|||||||
|
|
||||||
def test_write_hdf5_options(self):
|
def test_write_hdf5_options(self):
|
||||||
data = np.random.randint(0, 1024, 1024)
|
data = np.random.randint(0, 1024, 1024)
|
||||||
self.exp.set(KEY, data,
|
self.exp.set(
|
||||||
compression="gzip", compression_opts=6,
|
KEY,
|
||||||
shuffle=True, fletcher32=True)
|
data,
|
||||||
|
hdf5_options=dict(
|
||||||
|
compression="gzip",
|
||||||
|
compression_opts=6,
|
||||||
|
shuffle=True,
|
||||||
|
fletcher32=True
|
||||||
|
),
|
||||||
|
)
|
||||||
|
|
||||||
with h5py.File("test.h5", "a", "core", backing_store=False) as f:
|
with h5py.File("test.h5", "a", "core", backing_store=False) as f:
|
||||||
self.dataset_mgr.write_hdf5(f)
|
self.dataset_mgr.write_hdf5(f)
|
||||||
|
Loading…
Reference in New Issue
Block a user