set_dataset: pass HDF5 options as a dict, not as loose kwargs

Signed-off-by: Etienne Wodey <wodey@iqo.uni-hannover.de>
This commit is contained in:
Etienne Wodey 2021-06-17 16:43:05 +02:00
parent 12ef907f34
commit 8bedf278f0
5 changed files with 17 additions and 10 deletions

View File

@ -15,7 +15,7 @@ Highlights:
- Exposes upconverter calibration and enabling/disabling of upconverter LO & RF outputs. - Exposes upconverter calibration and enabling/disabling of upconverter LO & RF outputs.
* HDF5 options can now be passed when creating datasets with ``set_dataset``. This allows * HDF5 options can now be passed when creating datasets with ``set_dataset``. This allows
in particular to use transparent compression filters as follows: in particular to use transparent compression filters as follows:
``set_dataset(name, value, compression="gzip")``. ``set_dataset(name, value, hdf5_options={"compression": "gzip"})``.
Breaking changes: Breaking changes:

View File

@ -332,7 +332,7 @@ class HasEnvironment:
@rpc(flags={"async"}) @rpc(flags={"async"})
def set_dataset(self, key, value, def set_dataset(self, key, value,
broadcast=False, persist=False, archive=True, save=None, broadcast=False, persist=False, archive=True, save=None,
**hdf5_options): hdf5_options=None):
"""Sets the contents and handling modes of a dataset. """Sets the contents and handling modes of a dataset.
Datasets must be scalars (``bool``, ``int``, ``float`` or NumPy scalar) Datasets must be scalars (``bool``, ``int``, ``float`` or NumPy scalar)
@ -345,8 +345,8 @@ class HasEnvironment:
:param archive: the data is saved into the local storage of the current :param archive: the data is saved into the local storage of the current
run (archived as a HDF5 file). run (archived as a HDF5 file).
:param save: deprecated. :param save: deprecated.
:param hdf5_options: additional keyword arguments are passed to :param hdf5_options: dict of keyword arguments to pass to
:meth:`h5py.Group.create_dataset`. For example, pass ``compression="gzip"`` :meth:`h5py.Group.create_dataset`. For example, pass ``{"compression": "gzip"}``
to enable transparent zlib compression of this dataset in the HDF5 archive. to enable transparent zlib compression of this dataset in the HDF5 archive.
See the `h5py documentation <https://docs.h5py.org/en/stable/high/group.html#h5py.Group.create_dataset>`_ See the `h5py documentation <https://docs.h5py.org/en/stable/high/group.html#h5py.Group.create_dataset>`_
for a list of valid options. for a list of valid options.

View File

@ -87,7 +87,7 @@ class DatasetDB(TaskObject):
process_mod(self.data, mod) process_mod(self.data, mod)
# convenience functions (update() can be used instead) # convenience functions (update() can be used instead)
def set(self, key, value, persist=None, **hdf5_options): def set(self, key, value, persist=None, hdf5_options=None):
if persist is None: if persist is None:
if key in self.data.raw_view: if key in self.data.raw_view:
persist = self.data.raw_view[key]["persist"] persist = self.data.raw_view[key]["persist"]

View File

@ -21,8 +21,8 @@ class TestDatasetDB(unittest.TestCase):
self.ddb = DatasetDB(self.persist_file.name) self.ddb = DatasetDB(self.persist_file.name)
self.ddb.set(KEY1, DATA, persist=True) self.ddb.set(KEY1, DATA, persist=True)
self.ddb.set(KEY2, DATA, persist=True, compression=COMP) self.ddb.set(KEY2, DATA, persist=True, hdf5_options=dict(compression=COMP))
self.ddb.set(KEY3, DATA, shuffle=True) self.ddb.set(KEY3, DATA, hdf5_options=dict(shuffle=True))
self.save_ddb_to_disk() self.save_ddb_to_disk()

View File

@ -108,9 +108,16 @@ class ExperimentDatasetCase(unittest.TestCase):
def test_write_hdf5_options(self): def test_write_hdf5_options(self):
data = np.random.randint(0, 1024, 1024) data = np.random.randint(0, 1024, 1024)
self.exp.set(KEY, data, self.exp.set(
compression="gzip", compression_opts=6, KEY,
shuffle=True, fletcher32=True) data,
hdf5_options=dict(
compression="gzip",
compression_opts=6,
shuffle=True,
fletcher32=True
),
)
with h5py.File("test.h5", "a", "core", backing_store=False) as f: with h5py.File("test.h5", "a", "core", backing_store=False) as f:
self.dataset_mgr.write_hdf5(f) self.dataset_mgr.write_hdf5(f)