Merge branch 'master' into testbench-controllers

* master:
  gui/experiments: float/bring into focus already open docks when opening experiments
  gui: reduce size of console dock
  protocols/logging,pc_rpc: do not print errors on Windows when clients disconnect
  gui: reduce size of schedule dock
  worker: Windows VMs are slow, increase send_timeout
  protocol/sync_struct: Windows also raises ConnectionAbortedError on disconnection
  gui: reduce size of log dock
  gui: reduce size of experiment dock
  protocols/logging/LogParser: handle Windows CRLF
  pyon: handle \r
  test/pipe_ipc: re-enable
  protocols/asyncio_server: minor cleanup
  protocols/pipe_ipc: Windows support
  Revert "Revert "test/pipe_ipc: temporarily skip test""
  Revert "try debugging weird unittest failure"
  try debugging weird unittest failure
  conda: restrict binutils-or1k-linux dependency to linux.
  transforms.iodelay_estimator: make diagnostics much more clear.
  Fix typo.
This commit is contained in:
Robert Jördens 2016-01-27 14:26:35 -07:00
commit 99f788965e
23 changed files with 180 additions and 49 deletions

View File

@ -24,11 +24,11 @@ class IODelayEstimator(algorithm.Visitor):
self.current_goto = None self.current_goto = None
self.current_return = None self.current_return = None
def evaluate(self, node, abort): def evaluate(self, node, abort, context):
if isinstance(node, asttyped.NumT): if isinstance(node, asttyped.NumT):
return iodelay.Const(node.n) return iodelay.Const(node.n)
elif isinstance(node, asttyped.CoerceT): elif isinstance(node, asttyped.CoerceT):
return self.evaluate(node.value, abort) return self.evaluate(node.value, abort, context)
elif isinstance(node, asttyped.NameT): elif isinstance(node, asttyped.NameT):
if self.current_args is None: if self.current_args is None:
note = diagnostic.Diagnostic("note", note = diagnostic.Diagnostic("note",
@ -48,8 +48,8 @@ class IODelayEstimator(algorithm.Visitor):
] ]
abort(notes) abort(notes)
elif isinstance(node, asttyped.BinOpT): elif isinstance(node, asttyped.BinOpT):
lhs = self.evaluate(node.left, abort) lhs = self.evaluate(node.left, abort, context)
rhs = self.evaluate(node.right, abort) rhs = self.evaluate(node.right, abort, context)
if isinstance(node.op, ast.Add): if isinstance(node.op, ast.Add):
return lhs + rhs return lhs + rhs
elif isinstance(node.op, ast.Sub): elif isinstance(node.op, ast.Sub):
@ -62,12 +62,14 @@ class IODelayEstimator(algorithm.Visitor):
return lhs // rhs return lhs // rhs
else: else:
note = diagnostic.Diagnostic("note", note = diagnostic.Diagnostic("note",
"this operator is not supported", {}, "this operator is not supported {context}",
{"context": context},
node.op.loc) node.op.loc)
abort([note]) abort([note])
else: else:
note = diagnostic.Diagnostic("note", note = diagnostic.Diagnostic("note",
"this expression is not supported", {}, "this expression is not supported {context}",
{"context": context},
node.loc) node.loc)
abort([note]) abort([note])
@ -143,14 +145,14 @@ class IODelayEstimator(algorithm.Visitor):
def visit_LambdaT(self, node): def visit_LambdaT(self, node):
self.visit_function(node.args, node.body, node.type.find(), node.loc) self.visit_function(node.args, node.body, node.type.find(), node.loc)
def get_iterable_length(self, node): def get_iterable_length(self, node, context):
def abort(notes): def abort(notes):
self.abort("for statement cannot be interleaved because " self.abort("for statement cannot be interleaved because "
"trip count is indeterminate", "iteration count is indeterminate",
node.loc, notes) node.loc, notes)
def evaluate(node): def evaluate(node):
return self.evaluate(node, abort) return self.evaluate(node, abort, context)
if isinstance(node, asttyped.CallT) and types.is_builtin(node.func.type, "range"): if isinstance(node, asttyped.CallT) and types.is_builtin(node.func.type, "range"):
range_min, range_max, range_step = iodelay.Const(0), None, iodelay.Const(1) range_min, range_max, range_step = iodelay.Const(0), None, iodelay.Const(1)
@ -177,10 +179,11 @@ class IODelayEstimator(algorithm.Visitor):
self.current_delay = old_delay self.current_delay = old_delay
else: else:
if self.current_goto is not None: if self.current_goto is not None:
self.abort("loop trip count is indeterminate because of control flow", self.abort("loop iteration count is indeterminate because of control flow",
self.current_goto.loc) self.current_goto.loc)
node.trip_count = self.get_iterable_length(node.iter).fold() context = "in an iterable used in a for loop that is being interleaved"
node.trip_count = self.get_iterable_length(node.iter, context).fold()
node.trip_interval = self.current_delay.fold() node.trip_interval = self.current_delay.fold()
self.current_delay = old_delay + node.trip_interval * node.trip_count self.current_delay = old_delay + node.trip_interval * node.trip_count
self.current_goto = old_goto self.current_goto = old_goto
@ -231,6 +234,10 @@ class IODelayEstimator(algorithm.Visitor):
# Interleave failures inside `with` statements are hard failures, # Interleave failures inside `with` statements are hard failures,
# since there's no chance that the code will never actually execute # since there's no chance that the code will never actually execute
# inside a `with` statement after all. # inside a `with` statement after all.
note = diagnostic.Diagnostic("note",
"while interleaving this 'with parallel:' statement", {},
node.loc)
error.cause.notes += [note]
self.engine.process(error.cause) self.engine.process(error.cause)
flow_stmt = None flow_stmt = None
@ -258,15 +265,17 @@ class IODelayEstimator(algorithm.Visitor):
def visit_CallT(self, node): def visit_CallT(self, node):
typ = node.func.type.find() typ = node.func.type.find()
def abort(notes): def abort(notes):
self.abort("this call cannot be interleaved because " self.abort("call cannot be interleaved because "
"an argument cannot be statically evaluated", "an argument cannot be statically evaluated",
node.loc, notes) node.loc, notes)
if types.is_builtin(typ, "delay"): if types.is_builtin(typ, "delay"):
value = self.evaluate(node.args[0], abort=abort) value = self.evaluate(node.args[0], abort=abort,
context="as an argument for delay()")
call_delay = iodelay.SToMU(value, ref_period=self.ref_period) call_delay = iodelay.SToMU(value, ref_period=self.ref_period)
elif types.is_builtin(typ, "delay_mu"): elif types.is_builtin(typ, "delay_mu"):
value = self.evaluate(node.args[0], abort=abort) value = self.evaluate(node.args[0], abort=abort,
context="as an argument for delay_mu()")
call_delay = value call_delay = value
elif not types.is_builtin(typ): elif not types.is_builtin(typ):
if types.is_function(typ): if types.is_function(typ):
@ -297,7 +306,12 @@ class IODelayEstimator(algorithm.Visitor):
args[arg_name] = arg_node args[arg_name] = arg_node
free_vars = delay.duration.free_vars() free_vars = delay.duration.free_vars()
node.arg_exprs = { arg: self.evaluate(args[arg], abort=abort) for arg in free_vars } node.arg_exprs = {
arg: self.evaluate(args[arg], abort=abort,
context="in the expression for argument '{}' "
"that affects I/O delay".format(arg))
for arg in free_vars
}
call_delay = delay.duration.fold(node.arg_exprs) call_delay = delay.duration.fold(node.arg_exprs)
else: else:
assert False assert False

View File

@ -19,7 +19,7 @@ from artiq.coredevice import exceptions
def _render_diagnostic(diagnostic, colored): def _render_diagnostic(diagnostic, colored):
def shorten_path(path): def shorten_path(path):
return path.replace(artiq_dir, "<artiq>") return path.replace(artiq_dir, "<artiq>")
lines = [shorten_path(path) for path in diagnostic.render(colored)] lines = [shorten_path(path) for path in diagnostic.render(colored=colored)]
return "\n".join(lines) return "\n".join(lines)
class _DiagnosticEngine(diagnostic.Engine): class _DiagnosticEngine(diagnostic.Engine):

View File

@ -17,7 +17,7 @@ The following functions are available:
class ConsoleDock(dockarea.Dock): class ConsoleDock(dockarea.Dock):
def __init__(self, dataset_sub, dataset_ctl): def __init__(self, dataset_sub, dataset_ctl):
dockarea.Dock.__init__(self, "Console") dockarea.Dock.__init__(self, "Console")
self.setMinimumSize(QtCore.QSize(850, 300)) self.setMinimumSize(QtCore.QSize(720, 300))
self.dataset_sub = dataset_sub self.dataset_sub = dataset_sub
self.dataset_ctl = dataset_ctl self.dataset_ctl = dataset_ctl
ns = { ns = {

View File

@ -240,7 +240,7 @@ class _ArgumentEditor(QtGui.QTreeWidget):
class _ExperimentDock(dockarea.Dock): class _ExperimentDock(dockarea.Dock):
def __init__(self, manager, expurl): def __init__(self, manager, expurl):
dockarea.Dock.__init__(self, "Exp: " + expurl, closable=True) dockarea.Dock.__init__(self, "Exp: " + expurl, closable=True)
self.setMinimumSize(QtCore.QSize(1100, 700)) self.setMinimumSize(QtCore.QSize(740, 470))
self.layout.setSpacing(5) self.layout.setSpacing(5)
self.layout.setContentsMargins(5, 5, 5, 5) self.layout.setContentsMargins(5, 5, 5, 5)
@ -488,7 +488,9 @@ class ExperimentManager:
def open_experiment(self, expurl): def open_experiment(self, expurl):
if expurl in self.open_experiments: if expurl in self.open_experiments:
return self.open_experiments[expurl] dock = self.open_experiments[expurl]
self.dock_area.floatDock(dock)
return dock
dock = _ExperimentDock(self, expurl) dock = _ExperimentDock(self, expurl)
self.open_experiments[expurl] = dock self.open_experiments[expurl] = dock
self.dock_area.floatDock(dock) self.dock_area.floatDock(dock)

View File

@ -147,7 +147,7 @@ class _LogFilterProxyModel(QSortFilterProxyModel):
class _LogDock(dockarea.Dock): class _LogDock(dockarea.Dock):
def __init__(self, manager, name, log_sub): def __init__(self, manager, name, log_sub):
dockarea.Dock.__init__(self, name, label="Log") dockarea.Dock.__init__(self, name, label="Log")
self.setMinimumSize(QtCore.QSize(850, 450)) self.setMinimumSize(QtCore.QSize(720, 250))
grid = LayoutWidget() grid = LayoutWidget()
self.addWidget(grid) self.addWidget(grid)

View File

@ -58,7 +58,7 @@ class Model(DictSyncModel):
class ScheduleDock(dockarea.Dock): class ScheduleDock(dockarea.Dock):
def __init__(self, status_bar, schedule_ctl, schedule_sub): def __init__(self, status_bar, schedule_ctl, schedule_sub):
dockarea.Dock.__init__(self, "Schedule") dockarea.Dock.__init__(self, "Schedule")
self.setMinimumSize(QtCore.QSize(850, 300)) self.setMinimumSize(QtCore.QSize(740, 200))
self.status_bar = status_bar self.status_bar = status_bar
self.schedule_ctl = schedule_ctl self.schedule_ctl = schedule_ctl

View File

@ -42,7 +42,7 @@ def log_worker_exception():
class Worker: class Worker:
def __init__(self, handlers=dict(), send_timeout=0.5): def __init__(self, handlers=dict(), send_timeout=1.0):
self.handlers = handlers self.handlers = handlers
self.send_timeout = send_timeout self.send_timeout = send_timeout

View File

@ -7,7 +7,6 @@ class AsyncioServer:
Users of this class must derive from it and define the Users of this class must derive from it and define the
``_handle_connection_cr`` method and coroutine. ``_handle_connection_cr`` method and coroutine.
""" """
def __init__(self): def __init__(self):
self._client_tasks = set() self._client_tasks = set()
@ -23,15 +22,12 @@ class AsyncioServer:
:param host: Bind address of the server (see ``asyncio.start_server`` :param host: Bind address of the server (see ``asyncio.start_server``
from the Python standard library). from the Python standard library).
:param port: TCP port to bind to. :param port: TCP port to bind to.
""" """
self.server = await asyncio.start_server(self._handle_connection, self.server = await asyncio.start_server(self._handle_connection,
host, port) host, port)
async def stop(self): async def stop(self):
"""Stops the server. """Stops the server."""
"""
wait_for = copy(self._client_tasks) wait_for = copy(self._client_tasks)
for task in self._client_tasks: for task in self._client_tasks:
task.cancel() task.cancel()
@ -48,6 +44,6 @@ class AsyncioServer:
self._client_tasks.remove(task) self._client_tasks.remove(task)
def _handle_connection(self, reader, writer): def _handle_connection(self, reader, writer):
task = asyncio.Task(self._handle_connection_cr(reader, writer)) task = asyncio.ensure_future(self._handle_connection_cr(reader, writer))
self._client_tasks.add(task) self._client_tasks.add(task)
task.add_done_callback(self._client_done) task.add_done_callback(self._client_done)

View File

@ -78,7 +78,7 @@ class LogParser:
entry = (await stream.readline()) entry = (await stream.readline())
if not entry: if not entry:
break break
self.line_input(entry[:-1].decode()) self.line_input(entry.decode().rstrip("\r\n"))
except: except:
logger.debug("exception in log forwarding", exc_info=True) logger.debug("exception in log forwarding", exc_info=True)
break break
@ -125,6 +125,9 @@ class Server(AsyncioServer):
return return
source, remainder = linesplit source, remainder = linesplit
parser.line_input(remainder) parser.line_input(remainder)
except (ConnectionResetError, ConnectionAbortedError, BrokenPipeError):
# May happens on Windows when client disconnects
pass
finally: finally:
writer.close() writer.close()

View File

@ -502,6 +502,9 @@ class Server(_AsyncioServer):
"message": traceback.format_exc()} "message": traceback.format_exc()}
line = pyon.encode(obj) + "\n" line = pyon.encode(obj) + "\n"
writer.write(line.encode()) writer.write(line.encode())
except (ConnectionResetError, ConnectionAbortedError, BrokenPipeError):
# May happens on Windows when client disconnects
pass
finally: finally:
writer.close() writer.close()

View File

@ -3,6 +3,9 @@ import asyncio
from asyncio.streams import FlowControlMixin from asyncio.streams import FlowControlMixin
__all__ = ["AsyncioParentComm", "AsyncioChildComm", "ChildComm"]
class _BaseIO: class _BaseIO:
def write(self, data): def write(self, data):
self.writer.write(data) self.writer.write(data)
@ -92,10 +95,78 @@ if os.name != "nt":
else: # windows else: # windows
class AsyncioParentComm(_BaseIO): import itertools
_pipe_count = itertools.count()
class AsyncioParentComm:
"""Requires ProactorEventLoop"""
def __init__(self):
# We cannot use anonymous pipes on Windows, because we do not know
# in advance if the child process wants a handle open in overlapped
# mode or not.
self.address = "\\\\.\\pipe\\artiq-{}-{}".format(os.getpid(),
next(_pipe_count))
self.server = None
self.ready = asyncio.Event()
self.write_buffer = b""
def get_address(self):
return self.address
async def _autoclose(self): async def _autoclose(self):
await self.process.wait() await self.process.wait()
self.writer.close() if self.server is not None:
self.server[0].close()
self.server = None
if self.ready.is_set():
self.writer.close()
async def create_subprocess(self, *args, **kwargs):
loop = asyncio.get_event_loop()
def factory():
reader = asyncio.StreamReader(loop=loop)
protocol = asyncio.StreamReaderProtocol(reader,
self._child_connected,
loop=loop)
return protocol
self.server = await loop.start_serving_pipe(
factory, self.address)
self.process = await asyncio.create_subprocess_exec(
*args, **kwargs)
asyncio.ensure_future(self._autoclose())
def _child_connected(self, reader, writer):
self.server[0].close()
self.server = None
self.reader = reader
self.writer = writer
if self.write_buffer:
self.writer.write(self.write_buffer)
self.write_buffer = b""
self.ready.set()
def write(self, data):
if self.ready.is_set():
self.writer.write(data)
else:
self.write_buffer += data
async def drain(self):
await self.ready.wait()
await self.writer.drain()
async def readline(self):
await self.ready.wait()
return await self.reader.readline()
async def read(self, n):
await self.ready.wait()
return await self.reader.read(n)
class AsyncioChildComm(_BaseIO): class AsyncioChildComm(_BaseIO):
@ -109,9 +180,26 @@ else: # windows
reader_protocol = asyncio.StreamReaderProtocol( reader_protocol = asyncio.StreamReaderProtocol(
self.reader, loop=loop) self.reader, loop=loop)
transport, _ = await loop.create_pipe_connection( transport, _ = await loop.create_pipe_connection(
self.address, lambda: reader_protocol) lambda: reader_protocol, self.address)
self.writer = asyncio.StreamWriter(transport, reader_protocol, self.writer = asyncio.StreamWriter(transport, reader_protocol,
self.reader, loop) self.reader, loop)
def close(self):
self.writer.close()
class ChildComm: class ChildComm:
pass def __init__(self, address):
self.f = open(address, "a+b", 0)
def read(self, n):
return self.f.read(n)
def readline(self):
return self.f.readline()
def write(self, data):
return self.f.write(data)
def close(self):
self.f.close()

View File

@ -74,7 +74,10 @@ class _Encoder:
def encode_str(self, x): def encode_str(self, x):
# Do not use repr() for JSON compatibility. # Do not use repr() for JSON compatibility.
tt = {ord("\""): "\\\"", ord("\\"): "\\\\", ord("\n"): "\\n"} tt = {
ord("\""): "\\\"", ord("\\"): "\\\\",
ord("\n"): "\\n", ord("\r"): "\\r"
}
return "\"" + x.translate(tt) + "\"" return "\"" + x.translate(tt) + "\""
def encode_bytes(self, x): def encode_bytes(self, x):

View File

@ -235,7 +235,7 @@ class Publisher(AsyncioServer):
await writer.drain() await writer.drain()
finally: finally:
self._recipients[notifier_name].remove(queue) self._recipients[notifier_name].remove(queue)
except (ConnectionResetError, BrokenPipeError): except (ConnectionResetError, ConnectionAbortedError, BrokenPipeError):
# subscribers disconnecting are a normal occurence # subscribers disconnecting are a normal occurence
pass pass
finally: finally:

View File

@ -0,0 +1,12 @@
# RUN: %python -m artiq.compiler.testbench.signature +diag +delay %s >%t
# RUN: OutputCheck %s --file-to-check=%t
def f(x):
delay_mu(x)
x = 1
def g():
# CHECK-L: ${LINE:+2}: error: call cannot be interleaved because an argument cannot be statically evaluated
# CHECK-L: ${LINE:+1}: note: this expression is not supported in the expression for argument 'x' that affects I/O delay
f(x if True else x)

View File

@ -3,19 +3,19 @@
def f(a): def f(a):
b = 1.0 b = 1.0
# CHECK-L: ${LINE:+3}: error: this call cannot be interleaved # CHECK-L: ${LINE:+3}: error: call cannot be interleaved
# CHECK-L: ${LINE:+2}: note: this variable is not an argument of the innermost function # CHECK-L: ${LINE:+2}: note: this variable is not an argument of the innermost function
# CHECK-L: ${LINE:-4}: note: only these arguments are in scope of analysis # CHECK-L: ${LINE:-4}: note: only these arguments are in scope of analysis
delay(b) delay(b)
def g(): def g():
# CHECK-L: ${LINE:+2}: error: this call cannot be interleaved # CHECK-L: ${LINE:+2}: error: call cannot be interleaved
# CHECK-L: ${LINE:+1}: note: this operator is not supported # CHECK-L: ${LINE:+1}: note: this operator is not supported as an argument for delay()
delay(2.0**2) delay(2.0**2)
def h(): def h():
# CHECK-L: ${LINE:+2}: error: this call cannot be interleaved # CHECK-L: ${LINE:+2}: error: call cannot be interleaved
# CHECK-L: ${LINE:+1}: note: this expression is not supported # CHECK-L: ${LINE:+1}: note: this expression is not supported as an argument for delay_mu()
delay_mu(1 if False else 2) delay_mu(1 if False else 2)
f(1) f(1)

View File

@ -3,12 +3,12 @@
def f(): def f():
x = 1 x = 1
# CHECK-L: ${LINE:+1}: error: this call cannot be interleaved because an argument cannot be statically evaluated # CHECK-L: ${LINE:+1}: error: call cannot be interleaved because an argument cannot be statically evaluated
delay_mu(x) delay_mu(x)
def g(): def g():
x = 1.0 x = 1.0
# CHECK-L: ${LINE:+1}: error: this call cannot be interleaved # CHECK-L: ${LINE:+1}: error: call cannot be interleaved
delay(x) delay(x)

View File

@ -2,7 +2,7 @@
# RUN: OutputCheck %s --file-to-check=%t # RUN: OutputCheck %s --file-to-check=%t
def f(): def f():
# CHECK-L: ${LINE:+1}: error: this call cannot be interleaved # CHECK-L: ${LINE:+1}: error: call cannot be interleaved
delay(1.0**2) delay(1.0**2)
def g(): def g():

View File

@ -9,5 +9,5 @@ def pulse(len):
def f(): def f():
a = 100 a = 100
# CHECK-L: ${LINE:+1}: error: this call cannot be interleaved # CHECK-L: ${LINE:+1}: error: call cannot be interleaved
pulse(a) pulse(a)

View File

@ -3,7 +3,7 @@
def f(): def f():
r = range(10) r = range(10)
# CHECK-L: ${LINE:+2}: error: for statement cannot be interleaved because trip count is indeterminate # CHECK-L: ${LINE:+2}: error: for statement cannot be interleaved because iteration count is indeterminate
# CHECK-L: ${LINE:+1}: note: this value is not a constant range literal # CHECK-L: ${LINE:+1}: note: this value is not a constant range literal
for _ in r: for _ in r:
delay_mu(1) delay_mu(1)

View File

@ -4,11 +4,11 @@
def f(): def f():
for _ in range(10): for _ in range(10):
delay_mu(10) delay_mu(10)
# CHECK-L: ${LINE:+1}: error: loop trip count is indeterminate because of control flow # CHECK-L: ${LINE:+1}: error: loop iteration count is indeterminate because of control flow
break break
def g(): def g():
for _ in range(10): for _ in range(10):
delay_mu(10) delay_mu(10)
# CHECK-L: ${LINE:+1}: error: loop trip count is indeterminate because of control flow # CHECK-L: ${LINE:+1}: error: loop iteration count is indeterminate because of control flow
continue continue

View File

@ -0,0 +1,10 @@
# RUN: %python -m artiq.compiler.testbench.signature +diag +delay %s >%t
# RUN: OutputCheck %s --file-to-check=%t
x = 1
def f():
# CHECK-L: ${LINE:+2}: error: for statement cannot be interleaved because iteration count is indeterminate
# CHECK-L: ${LINE:+1}: note: this expression is not supported in an iterable used in a for loop that is being interleaved
for _ in range(x if True else x):
delay_mu(10)

View File

@ -26,7 +26,7 @@ class PYON(unittest.TestCase):
_json_test_object = { _json_test_object = {
"a": "b", "a": "b",
"x": [1, 2, {}], "x": [1, 2, {}],
"foo\nbaz\\qux\"": ["bar", 1.2, {"x": "y"}], "foo\nbaz\\qux\"\r2": ["bar", 1.2, {"x": "y"}],
"bar": [True, False, None] "bar": [True, False, None]
} }

View File

@ -54,7 +54,7 @@ requirements:
- pyqtgraph - pyqtgraph
- pygit2 - pygit2
- aiohttp - aiohttp
- binutils-or1k-linux - binutils-or1k-linux # [linux]
- pythonparser - pythonparser
- levenshtein - levenshtein