master: cache last RID. Closes #234

This commit is contained in:
Sebastien Bourdeauducq 2016-02-15 18:20:50 +01:00
parent 649f3b9094
commit 72a993afe0
4 changed files with 78 additions and 48 deletions

6
.gitignore vendored
View File

@ -19,6 +19,7 @@ __pycache__/
/artiq/test/results
/artiq/test/h5types.h5
/examples/master/results
/examples/master/last_rid.pyon
/examples/master/dataset_db.pyon
/examples/sim/results
/examples/sim/dataset_db.pyon
@ -26,8 +27,9 @@ __pycache__/
# recommended location for testbed
/run
# alternatively, when testing ad-hoc experiments at the root:
/device_db.pyon
/dataset_db.pyon
/results
/last_rid.pyon
/dataset_db.pyon
/device_db.pyon
/h5types.h5
/test*.py

View File

@ -12,7 +12,7 @@ from artiq.protocols.logging import Server as LoggingServer
from artiq.master.log import log_args, init_log
from artiq.master.databases import DeviceDB, DatasetDB
from artiq.master.scheduler import Scheduler
from artiq.master.worker_db import get_last_rid
from artiq.master.worker_db import RIDCounter
from artiq.master.experiments import FilesystemBackend, GitBackend, ExperimentDB
@ -73,7 +73,7 @@ def main():
"get_dataset": dataset_db.get,
"update_dataset": dataset_db.update
}
scheduler = Scheduler(get_last_rid() + 1, worker_handlers, experiment_db)
scheduler = Scheduler(RIDCounter(), worker_handlers, experiment_db)
worker_handlers.update({
"scheduler_submit": scheduler.submit,
"scheduler_delete": scheduler.delete,

View File

@ -116,16 +116,6 @@ class Run:
write_results = _mk_worker_method("write_results")
class RIDCounter:
def __init__(self, next_rid):
self._next_rid = next_rid
def get(self):
rid = self._next_rid
self._next_rid += 1
return rid
class RunPool:
def __init__(self, ridc, worker_handlers, notifier, experiment_db):
self.runs = dict()
@ -387,7 +377,7 @@ class Deleter(TaskObject):
class Scheduler:
def __init__(self, next_rid, worker_handlers, experiment_db):
def __init__(self, ridc, worker_handlers, experiment_db):
self.notifier = Notifier(dict())
self._pipelines = dict()
@ -395,7 +385,7 @@ class Scheduler:
self._experiment_db = experiment_db
self._terminated = False
self._ridc = RIDCounter(next_rid)
self._ridc = ridc
self._deleter = Deleter(self._pipelines)
def start(self):

View File

@ -2,6 +2,7 @@ from collections import OrderedDict
import importlib
import logging
import os
import tempfile
import time
import re
@ -15,6 +16,75 @@ from artiq.protocols.pc_rpc import AutoTarget, Client, BestEffortClient
logger = logging.getLogger(__name__)
class RIDCounter:
def __init__(self, cache_filename="last_rid.pyon", results_dir="results"):
self.cache_filename = cache_filename
self.results_dir = results_dir
self._next_rid = self._last_rid() + 1
logger.debug("Next RID is %d", self._next_rid)
def get(self):
rid = self._next_rid
self._next_rid += 1
self._update_cache(rid)
return rid
def _last_rid(self):
try:
rid = self._last_rid_from_cache()
except FileNotFoundError:
logger.debug("Last RID cache not found, scanning results")
rid = self._last_rid_from_results()
self._update_cache(rid)
return rid
else:
logger.debug("Using last RID from cache")
return rid
def _update_cache(self, rid):
contents = str(rid) + "\n"
directory = os.path.abspath(os.path.dirname(self.cache_filename))
with tempfile.NamedTemporaryFile("w", dir=directory, delete=False) as f:
f.write(contents)
tmpname = f.name
os.replace(tmpname, self.cache_filename)
def _last_rid_from_cache(self):
with open(self.cache_filename, "r") as f:
return int(f.read())
def _last_rid_from_results(self):
r = -1
try:
day_folders = os.listdir(self.results_dir)
except:
return r
day_folders = filter(lambda x: re.fullmatch('\d\d\d\d-\d\d-\d\d', x),
day_folders)
for df in day_folders:
day_path = os.path.join(self.results_dir, df)
try:
minute_folders = os.listdir(day_path)
except:
continue
minute_folders = filter(lambda x: re.fullmatch('\d\d-\d\d', x),
minute_folders)
for mf in minute_folders:
minute_path = os.path.join(day_path, mf)
try:
h5files = os.listdir(minute_path)
except:
continue
for x in h5files:
m = re.fullmatch('(\d\d\d\d\d\d\d\d\d)-.*\.h5', x)
if m is None:
continue
rid = int(m.group(1))
if rid > r:
r = rid
return r
def _create_device(desc, device_mgr):
ty = desc["type"]
if ty == "local":
@ -87,38 +157,6 @@ def get_hdf5_output(start_time, rid, name):
return h5py.File(os.path.join(dirname, filename), "w")
def get_last_rid():
r = -1
try:
day_folders = os.listdir("results")
except:
return r
day_folders = filter(lambda x: re.fullmatch('\d\d\d\d-\d\d-\d\d', x),
day_folders)
for df in day_folders:
day_path = os.path.join("results", df)
try:
minute_folders = os.listdir(day_path)
except:
continue
minute_folders = filter(lambda x: re.fullmatch('\d\d-\d\d', x),
minute_folders)
for mf in minute_folders:
minute_path = os.path.join(day_path, mf)
try:
h5files = os.listdir(minute_path)
except:
continue
for x in h5files:
m = re.fullmatch('(\d\d\d\d\d\d\d\d\d)-.*\.h5', x)
if m is None:
continue
rid = int(m.group(1))
if rid > r:
r = rid
return r
_type_to_hdf5 = {
int: h5py.h5t.STD_I64BE,
float: h5py.h5t.IEEE_F64BE,