scheduler: resolve git references into revisions on submission (#2296)

Signed-off-by: Charles Baynham <c.baynham@imperial.ac.uk>
This commit is contained in:
Charles Baynham 2024-01-10 08:05:26 +00:00 committed by GitHub
parent b215df2d25
commit 669edf17c5
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
4 changed files with 39 additions and 12 deletions

View File

@ -45,6 +45,7 @@ Highlights:
* Full Python 3.10 support. * Full Python 3.10 support.
* MSYS2 packaging for Windows, which replaces Conda. Conda packages are still available to * MSYS2 packaging for Windows, which replaces Conda. Conda packages are still available to
support legacy installations, but may be removed in a future release. support legacy installations, but may be removed in a future release.
* Experiments can now be submitted with revisions set to a branch / tag name instead of only git hashes.
Breaking changes: Breaking changes:

View File

@ -349,9 +349,10 @@ class _ExperimentDock(QtWidgets.QMdiSubWindow):
repo_rev = QtWidgets.QLineEdit() repo_rev = QtWidgets.QLineEdit()
repo_rev.setPlaceholderText("current") repo_rev.setPlaceholderText("current")
repo_rev.setClearButtonEnabled(True) repo_rev.setClearButtonEnabled(True)
repo_rev_label = QtWidgets.QLabel("Revision:") repo_rev_label = QtWidgets.QLabel("Rev / ref:")
repo_rev_label.setToolTip("Experiment repository revision " repo_rev_label.setToolTip("Experiment repository revision "
"(commit ID) to use") "(commit ID) or reference (branch "
"or tag) to use")
self.layout.addWidget(repo_rev_label, 3, 2) self.layout.addWidget(repo_rev_label, 3, 2)
self.layout.addWidget(repo_rev, 3, 3) self.layout.addWidget(repo_rev, 3, 3)
@ -739,8 +740,13 @@ class ExperimentManager:
del self.open_experiments[expurl] del self.open_experiments[expurl]
async def _submit_task(self, expurl, *args): async def _submit_task(self, expurl, *args):
rid = await self.schedule_ctl.submit(*args) try:
logger.info("Submitted '%s', RID is %d", expurl, rid) rid = await self.schedule_ctl.submit(*args)
except KeyError:
expid = args[1]
logger.error("Submission failed - revision \"%s\" was not found", expid["repo_rev"])
else:
logger.info("Submitted '%s', RID is %d", expurl, rid)
def submit(self, expurl): def submit(self, expurl):
file, class_name, _ = self.resolve_expurl(expurl) file, class_name, _ = self.resolve_expurl(expurl)

View File

@ -111,7 +111,7 @@ class ExperimentDB:
try: try:
if new_cur_rev is None: if new_cur_rev is None:
new_cur_rev = self.repo_backend.get_head_rev() new_cur_rev = self.repo_backend.get_head_rev()
wd, _ = self.repo_backend.request_rev(new_cur_rev) wd, _, _ = self.repo_backend.request_rev(new_cur_rev)
self.repo_backend.release_rev(self.cur_rev) self.repo_backend.release_rev(self.cur_rev)
self.cur_rev = new_cur_rev self.cur_rev = new_cur_rev
self.status["cur_rev"] = new_cur_rev self.status["cur_rev"] = new_cur_rev
@ -132,7 +132,7 @@ class ExperimentDB:
if use_repository: if use_repository:
if revision is None: if revision is None:
revision = self.cur_rev revision = self.cur_rev
wd, _ = self.repo_backend.request_rev(revision) wd, _, revision = self.repo_backend.request_rev(revision)
filename = os.path.join(wd, filename) filename = os.path.join(wd, filename)
worker = Worker(self.worker_handlers) worker = Worker(self.worker_handlers)
try: try:
@ -169,7 +169,7 @@ class FilesystemBackend:
return "N/A" return "N/A"
def request_rev(self, rev): def request_rev(self, rev):
return self.root, None return self.root, None, "N/A"
def release_rev(self, rev): def release_rev(self, rev):
pass pass
@ -200,14 +200,26 @@ class GitBackend:
def get_head_rev(self): def get_head_rev(self):
return str(self.git.head.target) return str(self.git.head.target)
def _get_pinned_rev(self, rev):
"""
Resolve a git reference (e.g. "HEAD", "master", "abcdef123456...") into
a git hash
"""
commit, _ = self.git.resolve_refish(rev)
logger.debug('Resolved git ref "%s" into "%s"', rev, commit.hex)
return commit.hex
def request_rev(self, rev): def request_rev(self, rev):
rev = self._get_pinned_rev(rev)
if rev in self.checkouts: if rev in self.checkouts:
co = self.checkouts[rev] co = self.checkouts[rev]
co.ref_count += 1 co.ref_count += 1
else: else:
co = _GitCheckout(self.git, rev) co = _GitCheckout(self.git, rev)
self.checkouts[rev] = co self.checkouts[rev] = co
return co.path, co.message return co.path, co.message, rev
def release_rev(self, rev): def release_rev(self, rev):
co = self.checkouts[rev] co = self.checkouts[rev]

View File

@ -132,15 +132,23 @@ class RunPool:
writer.writerow([rid, start_time, expid["file"]]) writer.writerow([rid, start_time, expid["file"]])
def submit(self, expid, priority, due_date, flush, pipeline_name): def submit(self, expid, priority, due_date, flush, pipeline_name):
"""
Submits an experiment to be run by this pool
If expid has the attribute `repo_rev`, treat it as a git revision or
reference and resolve into a unique git hash before submission
"""
# mutates expid to insert head repository revision if None and # mutates expid to insert head repository revision if None and
# replaces relative path with the absolute one. # replaces relative path with the absolute one.
# called through scheduler. # called through scheduler.
rid = self.ridc.get() rid = self.ridc.get()
if "repo_rev" in expid: if "repo_rev" in expid:
if expid["repo_rev"] is None: repo_rev_or_ref = expid["repo_rev"] or self.experiment_db.cur_rev
expid["repo_rev"] = self.experiment_db.cur_rev wd, repo_msg, repo_rev = self.experiment_db.repo_backend.request_rev(repo_rev_or_ref)
wd, repo_msg = self.experiment_db.repo_backend.request_rev(
expid["repo_rev"]) # Mutate expid's repo_rev to that returned from request_rev, in case
# a branch was passed instead of a hash
expid["repo_rev"] = repo_rev
else: else:
if "file" in expid: if "file" in expid:
expid["file"] = os.path.abspath(expid["file"]) expid["file"] = os.path.abspath(expid["file"])