From 669edf17c5a60ee76d0555600dab80889bfcad79 Mon Sep 17 00:00:00 2001 From: Charles Baynham Date: Wed, 10 Jan 2024 08:05:26 +0000 Subject: [PATCH] scheduler: resolve git references into revisions on submission (#2296) Signed-off-by: Charles Baynham --- RELEASE_NOTES.rst | 1 + artiq/dashboard/experiments.py | 14 ++++++++++---- artiq/master/experiments.py | 20 ++++++++++++++++---- artiq/master/scheduler.py | 16 ++++++++++++---- 4 files changed, 39 insertions(+), 12 deletions(-) diff --git a/RELEASE_NOTES.rst b/RELEASE_NOTES.rst index 15ce87197..40fec5f56 100644 --- a/RELEASE_NOTES.rst +++ b/RELEASE_NOTES.rst @@ -45,6 +45,7 @@ Highlights: * Full Python 3.10 support. * MSYS2 packaging for Windows, which replaces Conda. Conda packages are still available to support legacy installations, but may be removed in a future release. +* Experiments can now be submitted with revisions set to a branch / tag name instead of only git hashes. Breaking changes: diff --git a/artiq/dashboard/experiments.py b/artiq/dashboard/experiments.py index 8526fdf32..a7db9047b 100644 --- a/artiq/dashboard/experiments.py +++ b/artiq/dashboard/experiments.py @@ -349,9 +349,10 @@ class _ExperimentDock(QtWidgets.QMdiSubWindow): repo_rev = QtWidgets.QLineEdit() repo_rev.setPlaceholderText("current") repo_rev.setClearButtonEnabled(True) - repo_rev_label = QtWidgets.QLabel("Revision:") + repo_rev_label = QtWidgets.QLabel("Rev / ref:") repo_rev_label.setToolTip("Experiment repository revision " - "(commit ID) to use") + "(commit ID) or reference (branch " + "or tag) to use") self.layout.addWidget(repo_rev_label, 3, 2) self.layout.addWidget(repo_rev, 3, 3) @@ -739,8 +740,13 @@ class ExperimentManager: del self.open_experiments[expurl] async def _submit_task(self, expurl, *args): - rid = await self.schedule_ctl.submit(*args) - logger.info("Submitted '%s', RID is %d", expurl, rid) + try: + rid = await self.schedule_ctl.submit(*args) + except KeyError: + expid = args[1] + logger.error("Submission failed - revision \"%s\" was not found", expid["repo_rev"]) + else: + logger.info("Submitted '%s', RID is %d", expurl, rid) def submit(self, expurl): file, class_name, _ = self.resolve_expurl(expurl) diff --git a/artiq/master/experiments.py b/artiq/master/experiments.py index df29f2a6b..935872fb0 100644 --- a/artiq/master/experiments.py +++ b/artiq/master/experiments.py @@ -111,7 +111,7 @@ class ExperimentDB: try: if new_cur_rev is None: new_cur_rev = self.repo_backend.get_head_rev() - wd, _ = self.repo_backend.request_rev(new_cur_rev) + wd, _, _ = self.repo_backend.request_rev(new_cur_rev) self.repo_backend.release_rev(self.cur_rev) self.cur_rev = new_cur_rev self.status["cur_rev"] = new_cur_rev @@ -132,7 +132,7 @@ class ExperimentDB: if use_repository: if revision is None: revision = self.cur_rev - wd, _ = self.repo_backend.request_rev(revision) + wd, _, revision = self.repo_backend.request_rev(revision) filename = os.path.join(wd, filename) worker = Worker(self.worker_handlers) try: @@ -169,7 +169,7 @@ class FilesystemBackend: return "N/A" def request_rev(self, rev): - return self.root, None + return self.root, None, "N/A" def release_rev(self, rev): pass @@ -200,14 +200,26 @@ class GitBackend: def get_head_rev(self): return str(self.git.head.target) + def _get_pinned_rev(self, rev): + """ + Resolve a git reference (e.g. "HEAD", "master", "abcdef123456...") into + a git hash + """ + commit, _ = self.git.resolve_refish(rev) + + logger.debug('Resolved git ref "%s" into "%s"', rev, commit.hex) + + return commit.hex + def request_rev(self, rev): + rev = self._get_pinned_rev(rev) if rev in self.checkouts: co = self.checkouts[rev] co.ref_count += 1 else: co = _GitCheckout(self.git, rev) self.checkouts[rev] = co - return co.path, co.message + return co.path, co.message, rev def release_rev(self, rev): co = self.checkouts[rev] diff --git a/artiq/master/scheduler.py b/artiq/master/scheduler.py index bc264964e..451e5c84c 100644 --- a/artiq/master/scheduler.py +++ b/artiq/master/scheduler.py @@ -132,15 +132,23 @@ class RunPool: writer.writerow([rid, start_time, expid["file"]]) def submit(self, expid, priority, due_date, flush, pipeline_name): + """ + Submits an experiment to be run by this pool + + If expid has the attribute `repo_rev`, treat it as a git revision or + reference and resolve into a unique git hash before submission + """ # mutates expid to insert head repository revision if None and # replaces relative path with the absolute one. # called through scheduler. rid = self.ridc.get() if "repo_rev" in expid: - if expid["repo_rev"] is None: - expid["repo_rev"] = self.experiment_db.cur_rev - wd, repo_msg = self.experiment_db.repo_backend.request_rev( - expid["repo_rev"]) + repo_rev_or_ref = expid["repo_rev"] or self.experiment_db.cur_rev + wd, repo_msg, repo_rev = self.experiment_db.repo_backend.request_rev(repo_rev_or_ref) + + # Mutate expid's repo_rev to that returned from request_rev, in case + # a branch was passed instead of a hash + expid["repo_rev"] = repo_rev else: if "file" in expid: expid["file"] = os.path.abspath(expid["file"])