occheung
/
artiq
forked from M-Labs/artiq
1
0
Fork 0
Code Pull Requests Activity

databases: read and save metadata in lmdb

This commit is contained in:
Simon Renblad 2023-07-04 17:45:03 +08:00 committed by Sébastien Bourdeauducq
parent bfbe13e51b
commit e710d4badd
1 changed files with 17 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

22
artiq/master/databases.py
View File

@ -45,8 +45,9 @@ class DatasetDB(TaskObject):
self.lmdb = lmdb.open(persist_file, subdir=False, map_size=2**30) self.lmdb = lmdb.open(persist_file, subdir=False, map_size=2**30)
data = dict() data = dict()
with self.lmdb.begin() as txn: with self.lmdb.begin() as txn:
for key, value in txn.cursor(): for key, value_and_metadata in txn.cursor():
data[key.decode()] = (True, pyon.decode(value.decode())) value, metadata = pyon.decode(value_and_metadata.decode())
data[key.decode()] = (True, value, metadata)
self.data = Notifier(data) self.data = Notifier(data)
self.pending_keys = set() self.pending_keys = set()
@ -59,7 +60,10 @@ class DatasetDB(TaskObject):
if key not in self.data.raw_view or not self.data.raw_view[key][0]: if key not in self.data.raw_view or not self.data.raw_view[key][0]:
txn.delete(key.encode()) txn.delete(key.encode())
else: else:
txn.put(key.encode(), pyon.encode(self.data.raw_view[key][1]).encode()) value_and_metadata = (self.data.raw_view[key][1],
self.data.raw_view[key][2])
txn.put(key.encode(),
pyon.encode(value_and_metadata).encode())
self.pending_keys.clear() self.pending_keys.clear()
async def _do(self): async def _do(self):
@ -73,6 +77,9 @@ class DatasetDB(TaskObject):
def get(self, key): def get(self, key):
return self.data.raw_view[key][1] return self.data.raw_view[key][1]
def get_metadata(self, key):
return self.data.raw_view[key][2]
def update(self, mod): def update(self, mod):
if mod["path"]: if mod["path"]:
key = mod["path"][0] key = mod["path"][0]
@ -83,13 +90,18 @@ class DatasetDB(TaskObject):
process_mod(self.data, mod) process_mod(self.data, mod)
# convenience functions (update() can be used instead) # convenience functions (update() can be used instead)
def set(self, key, value, persist=None): def set(self, key, value, persist=None, metadata=None):
if persist is None: if persist is None:
if key in self.data.raw_view: if key in self.data.raw_view:
persist = self.data.raw_view[key][0] persist = self.data.raw_view[key][0]
else: else:
persist = False persist = False
self.data[key] = (persist, value) if metadata is None:
if key in self.data.raw_view:
metadata = self.data.raw_view[key][2]
else:
metadata = {}
self.data[key] = (persist, value, metadata)
self.pending_keys.add(key) self.pending_keys.add(key)
def delete(self, key): def delete(self, key):