forked from M-Labs/artiq
1
0
Fork 0

ctlmgr: support immediate controller retry

This commit is contained in:
Sebastien Bourdeauducq 2015-08-11 23:22:36 +08:00
parent 80805407bf
commit a6ab066c87
2 changed files with 72 additions and 43 deletions

View File

@ -1,6 +1,7 @@
#!/usr/bin/env python3
import asyncio
import atexit
import argparse
import os
import logging
@ -8,9 +9,9 @@ import shlex
import socket
from artiq.protocols.sync_struct import Subscriber
from artiq.protocols.pc_rpc import AsyncioClient
from artiq.protocols.pc_rpc import AsyncioClient, Server
from artiq.tools import verbosity_args, init_logger
from artiq.tools import asyncio_process_wait_timeout
from artiq.tools import TaskObject, asyncio_process_wait_timeout, Condition
logger = logging.getLogger(__name__)
@ -29,8 +30,11 @@ def get_argparser():
"--retry-master", default=5.0, type=float,
help="retry timer for reconnecting to master")
parser.add_argument(
"--retry-command", default=5.0, type=float,
help="retry timer for restarting a controller command")
"--bind", default="::1",
help="hostname or IP address to bind to")
parser.add_argument(
"--bind-port", default=3249, type=int,
help="TCP port to listen to for control (default: %(default)d)")
return parser
@ -48,6 +52,7 @@ class Controller:
self.term_timeout = ddb_entry.get("term_timeout", 30)
self.retry_timer_cur = self.retry_timer
self.retry_now = Condition()
self.process = None
self.launch_task = asyncio.Task(self.launcher())
@ -109,8 +114,13 @@ class Controller:
logger.warning("Controller %s failed to start", self.name)
else:
logger.warning("Controller %s exited", self.name)
logger.warning("Restarting in %.1f seconds", self.retry_timer_cur)
yield from asyncio.sleep(self.retry_timer_cur)
logger.warning("Restarting in %.1f seconds",
self.retry_timer_cur)
try:
yield from asyncio.wait_for(self.retry_now.wait(),
self.retry_timer_cur)
except asyncio.TimeoutError:
pass
self.retry_timer_cur *= self.retry_timer_backoff
except asyncio.CancelledError:
yield from self._terminate()
@ -208,18 +218,26 @@ class ControllerDB:
return self.current_controllers
@asyncio.coroutine
def ctlmgr(server, port, retry_master):
controller_db = ControllerDB()
class ControllerManager(TaskObject):
def __init__(self, server, port, retry_master):
self.server = server
self.port = port
self.retry_master = retry_master
self.controller_db = ControllerDB()
@asyncio.coroutine
def _do(self):
try:
subscriber = Subscriber("devices", controller_db.sync_struct_init)
subscriber = Subscriber("devices",
self.controller_db.sync_struct_init)
while True:
try:
def set_host_filter():
s = subscriber.writer.get_extra_info("socket")
localhost = s.getsockname()[0]
controller_db.set_host_filter(localhost)
yield from subscriber.connect(server, port, set_host_filter)
self.controller_db.set_host_filter(localhost)
yield from subscriber.connect(self.server, self.port,
set_host_filter)
try:
yield from asyncio.wait_for(subscriber.receive_task, None)
finally:
@ -230,12 +248,17 @@ def ctlmgr(server, port, retry_master):
e.__class__.__name__, str(e))
else:
logger.warning("Connection to master lost")
logger.warning("Retrying in %.1f seconds", retry_master)
yield from asyncio.sleep(retry_master)
logger.warning("Retrying in %.1f seconds", self.retry_master)
yield from asyncio.sleep(self.retry_master)
except asyncio.CancelledError:
pass
finally:
yield from controller_db.current_controllers.shutdown()
yield from self.controller_db.current_controllers.shutdown()
def retry_now(self, k):
"""If a controller is disabled and pending retry, perform that retry
now."""
self.controller_db.current_controllers.active[k].retry_now.notify()
def main():
@ -247,18 +270,22 @@ def main():
asyncio.set_event_loop(loop)
else:
loop = asyncio.get_event_loop()
atexit.register(lambda: loop.close())
try:
task = asyncio.Task(ctlmgr(
args.server, args.port, args.retry_master))
try:
loop.run_forever()
finally:
task.cancel()
loop.run_until_complete(asyncio.wait_for(task, None))
ctlmgr = ControllerManager(args.server, args.port, args.retry_master)
ctlmgr.start()
atexit.register(lambda: loop.run_until_complete(ctlmgr.stop()))
class CtlMgrRPC:
retry_now = ctlmgr.retry_now
rpc_target = CtlMgrRPC()
rpc_server = Server({"ctlmgr": rpc_target}, builtin_terminate=True)
loop.run_until_complete(rpc_server.start(args.bind, args.bind_port))
atexit.register(lambda: loop.run_until_complete(rpc_server.stop()))
loop.run_until_complete(rpc_server.wait_terminate())
finally:
loop.close()
if __name__ == "__main__":
main()

View File

@ -8,6 +8,8 @@ Default network ports
+--------------------------+--------------+
| Core device (mon/inj) | 3250 (UDP) |
+--------------------------+--------------+
| Controller manager | 3249 |
+--------------------------+--------------+
| Master (notifications) | 3250 |
+--------------------------+--------------+
| Master (control) | 3251 |