srenblad
/
artiq
forked from M-Labs/artiq
1
0
Fork 0
Code Pull Requests Activity

ctlmgr: exponential backoff

This commit is contained in:
Sebastien Bourdeauducq 2015-08-09 18:28:56 +08:00
parent 3f68d0ba8f
commit a21049d779
1 changed files with 10 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
artiq/frontend/artiq_ctlmgr.py
View File

@ -40,12 +40,14 @@ class Controller:
self.name = name self.name = name
self.command = ddb_entry["command"] self.command = ddb_entry["command"]
self.retry_timer = ddb_entry.get("retry_timer", 5) self.retry_timer = ddb_entry.get("retry_timer", 5)
self.retry_timer_backoff = ddb_entry.get("retry_timer_backoff", 1.1)
self.host = ddb_entry["host"] self.host = ddb_entry["host"]
self.port = ddb_entry["port"] self.port = ddb_entry["port"]
self.ping_timer = ddb_entry.get("ping_timer", 30) self.ping_timer = ddb_entry.get("ping_timer", 30)
self.ping_timeout = ddb_entry.get("ping_timeout", 30) self.ping_timeout = ddb_entry.get("ping_timeout", 30)
self.retry_timer_cur = self.retry_timer
self.process = None self.process = None
self.launch_task = asyncio.Task(self.launcher()) self.launch_task = asyncio.Task(self.launcher())
@ -69,8 +71,11 @@ class Controller:
@asyncio.coroutine @asyncio.coroutine
def _ping(self): def _ping(self):
try: try:
return (yield from asyncio.wait_for( ok = yield from asyncio.wait_for(self._ping_notimeout(),
self._ping_notimeout(), self.ping_timeout)) self.ping_timeout)
if ok:
self.retry_timer_cur = self.retry_timer
return ok
except: except:
return False return False
@ -102,8 +107,9 @@ class Controller:
logger.warning("Controller %s failed to start", self.name) logger.warning("Controller %s failed to start", self.name)
else: else:
logger.warning("Controller %s exited", self.name) logger.warning("Controller %s exited", self.name)
logger.warning("Restarting in %.1f seconds", self.retry_timer) logger.warning("Restarting in %.1f seconds", self.retry_timer_cur)
yield from asyncio.sleep(self.retry_timer) yield from asyncio.sleep(self.retry_timer_cur)
self.retry_timer_cur *= self.retry_timer_backoff
except asyncio.CancelledError: except asyncio.CancelledError:
yield from self._terminate() yield from self._terminate()