mirror of
https://github.com/m-labs/artiq.git
synced 2024-12-25 03:08:27 +08:00
ctlmgr: graceful controller termination
This commit is contained in:
parent
4b195663f6
commit
54b11a392a
@ -4,7 +4,6 @@ import asyncio
|
||||
import argparse
|
||||
import os
|
||||
import logging
|
||||
import signal
|
||||
import shlex
|
||||
import socket
|
||||
|
||||
@ -46,6 +45,7 @@ class Controller:
|
||||
self.port = ddb_entry["port"]
|
||||
self.ping_timer = ddb_entry.get("ping_timer", 30)
|
||||
self.ping_timeout = ddb_entry.get("ping_timeout", 30)
|
||||
self.term_timeout = ddb_entry.get("term_timeout", 30)
|
||||
|
||||
self.retry_timer_cur = self.retry_timer
|
||||
self.process = None
|
||||
@ -57,21 +57,21 @@ class Controller:
|
||||
yield from asyncio.wait_for(self.launch_task, None)
|
||||
|
||||
@asyncio.coroutine
|
||||
def _ping_notimeout(self):
|
||||
def _call_controller(self, method):
|
||||
remote = AsyncioClient()
|
||||
yield from remote.connect_rpc(self.host, self.port, None)
|
||||
try:
|
||||
targets, _ = remote.get_rpc_id()
|
||||
remote.select_rpc_target(targets[0])
|
||||
ok = yield from remote.ping()
|
||||
r = yield from getattr(remote, method)()
|
||||
finally:
|
||||
remote.close_rpc()
|
||||
return ok
|
||||
return r
|
||||
|
||||
@asyncio.coroutine
|
||||
def _ping(self):
|
||||
try:
|
||||
ok = yield from asyncio.wait_for(self._ping_notimeout(),
|
||||
ok = yield from asyncio.wait_for(self._call_controller("ping"),
|
||||
self.ping_timeout)
|
||||
if ok:
|
||||
self.retry_timer_cur = self.retry_timer
|
||||
@ -92,6 +92,8 @@ class Controller:
|
||||
logger.warning("Controller %s ping failed", self.name)
|
||||
yield from self._terminate()
|
||||
return
|
||||
else:
|
||||
break
|
||||
|
||||
@asyncio.coroutine
|
||||
def launcher(self):
|
||||
@ -117,14 +119,22 @@ class Controller:
|
||||
def _terminate(self):
|
||||
logger.info("Terminating controller %s", self.name)
|
||||
if self.process is not None and self.process.returncode is None:
|
||||
self.process.send_signal(signal.SIGTERM)
|
||||
logger.debug("Signal sent")
|
||||
try:
|
||||
yield from asyncio_process_wait_timeout(self.process, 5.0)
|
||||
except asyncio.TimeoutError:
|
||||
logger.warning("Controller %s did not respond to SIGTERM",
|
||||
yield from asyncio.wait_for(self._call_controller("terminate"),
|
||||
self.term_timeout)
|
||||
except:
|
||||
logger.warning("Controller %s did not respond to terminate "
|
||||
"command, killing", self.name)
|
||||
self.process.kill()
|
||||
try:
|
||||
yield from asyncio_process_wait_timeout(self.process,
|
||||
self.term_timeout)
|
||||
except:
|
||||
logger.warning("Controller %s failed to exit, killing",
|
||||
self.name)
|
||||
self.process.send_signal(signal.SIGKILL)
|
||||
self.process.kill()
|
||||
yield from self.process.wait()
|
||||
logger.debug("Controller %s terminated", self.name)
|
||||
|
||||
|
||||
def get_ip_addresses(host):
|
||||
|
Loading…
Reference in New Issue
Block a user