master: better repo scan error handling

artiq/master/repository.py

@@ -6,6 +6,7 @@ import logging
 
 from artiq.protocols.sync_struct import Notifier
 from artiq.master.worker import Worker
+from artiq.tools import exc_to_warning
 
 
 logger = logging.getLogger(__name__)
@@ -72,19 +73,20 @@ class Repository:
         if self._scanning:
             return
         self._scanning = True
+        try:
+            if new_cur_rev is None:
+                new_cur_rev = self.backend.get_head_rev()
+            wd, _ = self.backend.request_rev(new_cur_rev)
+            self.backend.release_rev(self.cur_rev)
+            self.cur_rev = new_cur_rev
+            new_explist = yield from _scan_experiments(wd, self.log_fn)
 
-        if new_cur_rev is None:
-            new_cur_rev = self.backend.get_head_rev()
-        wd, _ = self.backend.request_rev(new_cur_rev)
-        self.backend.release_rev(self.cur_rev)
-        self.cur_rev = new_cur_rev
-        new_explist = yield from _scan_experiments(wd, self.log_fn)
-
-        _sync_explist(self.explist, new_explist)
-        self._scanning = False
+            _sync_explist(self.explist, new_explist)
+        finally:
+            self._scanning = False
 
     def scan_async(self, new_cur_rev=None):
-        asyncio.async(self.scan(new_cur_rev))
+        asyncio.async(exc_to_warning(self.scan(new_cur_rev)))
 
 
 class FilesystemBackend:

artiq/tools.py

@@ -11,6 +11,9 @@ from artiq.language.environment import is_experiment
 from artiq.protocols import pyon
 
 
+logger = logging.getLogger(__name__)
+
+
 def parse_arguments(arguments):
     d = {}
     for argument in arguments:
@@ -75,6 +78,15 @@ def init_logger(args):
     logging.basicConfig(level=logging.WARNING + args.quiet*10 - args.verbose*10)
 
 
+@asyncio.coroutine
+def exc_to_warning(coro):
+    try:
+        yield from coro
+    except:
+        logger.warning("asyncio coroutine terminated with exception",
+                       exc_info=True)
+
+
 @asyncio.coroutine
 def asyncio_process_wait_timeout(process, timeout):
     # In Python < 3.5, asyncio.wait_for(process.wait(), ...