master: better repo scan error handling

artiq/master/repository.py

@@ -6,6 +6,7 @@ import logging
 
 from artiq.protocols.sync_struct import Notifier
 from artiq.master.worker import Worker
+from artiq.tools import exc_to_warning
 
 
 logger = logging.getLogger(__name__)
@@ -72,7 +73,7 @@ class Repository:
         if self._scanning:
             return
         self._scanning = True
-
+        try:
             if new_cur_rev is None:
                 new_cur_rev = self.backend.get_head_rev()
             wd, _ = self.backend.request_rev(new_cur_rev)
@@ -81,10 +82,11 @@ class Repository:
             new_explist = yield from _scan_experiments(wd, self.log_fn)
 
             _sync_explist(self.explist, new_explist)
+        finally:
             self._scanning = False
 
     def scan_async(self, new_cur_rev=None):
-        asyncio.async(self.scan(new_cur_rev))
+        asyncio.async(exc_to_warning(self.scan(new_cur_rev)))
 
 
 class FilesystemBackend:

artiq/tools.py

@@ -11,6 +11,9 @@ from artiq.language.environment import is_experiment
 from artiq.protocols import pyon
 
 
+logger = logging.getLogger(__name__)
+
+
 def parse_arguments(arguments):
     d = {}
     for argument in arguments:
@@ -75,6 +78,15 @@ def init_logger(args):
     logging.basicConfig(level=logging.WARNING + args.quiet*10 - args.verbose*10)
 
 
+@asyncio.coroutine
+def exc_to_warning(coro):
+    try:
+        yield from coro
+    except:
+        logger.warning("asyncio coroutine terminated with exception",
+                       exc_info=True)
+
+
 @asyncio.coroutine
 def asyncio_process_wait_timeout(process, timeout):
     # In Python < 3.5, asyncio.wait_for(process.wait(), ...