occheung/artiq
1
0
Fork 0
forked from M-Labs/artiq
Code Pull Requests Activity

ctlmgr: graceful controller termination

Browse Source
This commit is contained in:
Sebastien Bourdeauducq 2015-08-09 20:41:11 +08:00
parent 4b195663f6
commit 54b11a392a
1 changed files with 21 additions and 11 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

32
artiq/frontend/artiq_ctlmgr.py
View File

@ -4,7 +4,6 @@ import asyncio
import argparse
import os
import logging
import signal
import shlex
import socket
@ -46,6 +45,7 @@ class Controller:
self.port = ddb_entry["port"]
self.ping_timer = ddb_entry.get("ping_timer", 30)
self.ping_timeout = ddb_entry.get("ping_timeout", 30)
self.term_timeout = ddb_entry.get("term_timeout", 30)
self.retry_timer_cur = self.retry_timer
self.process = None
@ -57,21 +57,21 @@ class Controller:
yield from asyncio.wait_for(self.launch_task, None)
@asyncio.coroutine
def _ping_notimeout(self):
def _call_controller(self, method):
remote = AsyncioClient()
yield from remote.connect_rpc(self.host, self.port, None)
try:
targets, _ = remote.get_rpc_id()
remote.select_rpc_target(targets[0])
ok = yield from remote.ping()
r = yield from getattr(remote, method)()
finally:
remote.close_rpc()
return ok
return r
@asyncio.coroutine
def _ping(self):
try:
ok = yield from asyncio.wait_for(self._ping_notimeout(),
ok = yield from asyncio.wait_for(self._call_controller("ping"),
self.ping_timeout)
if ok:
self.retry_timer_cur = self.retry_timer
@ -92,6 +92,8 @@ class Controller:
logger.warning("Controller %s ping failed", self.name)
yield from self._terminate()
return
else:
break
@asyncio.coroutine
def launcher(self):
@ -117,14 +119,22 @@ class Controller:
def _terminate(self):
logger.info("Terminating controller %s", self.name)
if self.process is not None and self.process.returncode is None:
self.process.send_signal(signal.SIGTERM)
logger.debug("Signal sent")
try:
yield from asyncio_process_wait_timeout(self.process, 5.0)
except asyncio.TimeoutError:
logger.warning("Controller %s did not respond to SIGTERM",
yield from asyncio.wait_for(self._call_controller("terminate"),
self.term_timeout)
except:
logger.warning("Controller %s did not respond to terminate "
"command, killing", self.name)
self.process.kill()
try:
yield from asyncio_process_wait_timeout(self.process,
self.term_timeout)
except:
logger.warning("Controller %s failed to exit, killing",
self.name)
self.process.send_signal(signal.SIGKILL)
self.process.kill()
yield from self.process.wait()
logger.debug("Controller %s terminated", self.name)
def get_ip_addresses(host):