Merge branch 'master' into testbench-controllers

* master:
  gui/experiments: float/bring into focus already open docks when opening experiments
  gui: reduce size of console dock
  protocols/logging,pc_rpc: do not print errors on Windows when clients disconnect
  gui: reduce size of schedule dock
  worker: Windows VMs are slow, increase send_timeout
  protocol/sync_struct: Windows also raises ConnectionAbortedError on disconnection
  gui: reduce size of log dock
  gui: reduce size of experiment dock
  protocols/logging/LogParser: handle Windows CRLF
  pyon: handle \r
  test/pipe_ipc: re-enable
  protocols/asyncio_server: minor cleanup
  protocols/pipe_ipc: Windows support
  Revert "Revert "test/pipe_ipc: temporarily skip test""
  Revert "try debugging weird unittest failure"
  try debugging weird unittest failure
  conda: restrict binutils-or1k-linux dependency to linux.
  transforms.iodelay_estimator: make diagnostics much more clear.
  Fix typo.

artiq/compiler/transforms/iodelay_estimator.py

@@ -24,11 +24,11 @@ class IODelayEstimator(algorithm.Visitor):
         self.current_goto   = None
         self.current_return = None
 
-    def evaluate(self, node, abort):
+    def evaluate(self, node, abort, context):
         if isinstance(node, asttyped.NumT):
             return iodelay.Const(node.n)
         elif isinstance(node, asttyped.CoerceT):
-            return self.evaluate(node.value, abort)
+            return self.evaluate(node.value, abort, context)
         elif isinstance(node, asttyped.NameT):
             if self.current_args is None:
                 note = diagnostic.Diagnostic("note",
@@ -48,8 +48,8 @@ class IODelayEstimator(algorithm.Visitor):
                 ]
                 abort(notes)
         elif isinstance(node, asttyped.BinOpT):
-            lhs = self.evaluate(node.left, abort)
-            rhs = self.evaluate(node.right, abort)
+            lhs = self.evaluate(node.left, abort, context)
+            rhs = self.evaluate(node.right, abort, context)
             if isinstance(node.op, ast.Add):
                 return lhs + rhs
             elif isinstance(node.op, ast.Sub):
@@ -62,12 +62,14 @@ class IODelayEstimator(algorithm.Visitor):
                 return lhs // rhs
             else:
                 note = diagnostic.Diagnostic("note",
-                    "this operator is not supported", {},
+                    "this operator is not supported {context}",
+                    {"context": context},
                     node.op.loc)
                 abort([note])
         else:
             note = diagnostic.Diagnostic("note",
-                "this expression is not supported", {},
+                "this expression is not supported {context}",
+                {"context": context},
                 node.loc)
             abort([note])
 
@@ -143,14 +145,14 @@ class IODelayEstimator(algorithm.Visitor):
     def visit_LambdaT(self, node):
         self.visit_function(node.args, node.body, node.type.find(), node.loc)
 
-    def get_iterable_length(self, node):
+    def get_iterable_length(self, node, context):
         def abort(notes):
             self.abort("for statement cannot be interleaved because "
-                       "trip count is indeterminate",
+                       "iteration count is indeterminate",
                        node.loc, notes)
 
         def evaluate(node):
-            return self.evaluate(node, abort)
+            return self.evaluate(node, abort, context)
 
         if isinstance(node, asttyped.CallT) and types.is_builtin(node.func.type, "range"):
             range_min, range_max, range_step = iodelay.Const(0), None, iodelay.Const(1)
@@ -177,10 +179,11 @@ class IODelayEstimator(algorithm.Visitor):
             self.current_delay = old_delay
         else:
             if self.current_goto is not None:
-                self.abort("loop trip count is indeterminate because of control flow",
+                self.abort("loop iteration count is indeterminate because of control flow",
                            self.current_goto.loc)
 
-            node.trip_count    = self.get_iterable_length(node.iter).fold()
+            context            = "in an iterable used in a for loop that is being interleaved"
+            node.trip_count    = self.get_iterable_length(node.iter, context).fold()
             node.trip_interval = self.current_delay.fold()
             self.current_delay = old_delay + node.trip_interval * node.trip_count
         self.current_goto = old_goto
@@ -231,6 +234,10 @@ class IODelayEstimator(algorithm.Visitor):
                 # Interleave failures inside `with` statements are hard failures,
                 # since there's no chance that the code will never actually execute
                 # inside a `with` statement after all.
+                note = diagnostic.Diagnostic("note",
+                    "while interleaving this 'with parallel:' statement", {},
+                    node.loc)
+                error.cause.notes += [note]
                 self.engine.process(error.cause)
 
             flow_stmt = None
@@ -258,15 +265,17 @@ class IODelayEstimator(algorithm.Visitor):
     def visit_CallT(self, node):
         typ = node.func.type.find()
         def abort(notes):
-            self.abort("this call cannot be interleaved because "
+            self.abort("call cannot be interleaved because "
                        "an argument cannot be statically evaluated",
                        node.loc, notes)
 
         if types.is_builtin(typ, "delay"):
-            value = self.evaluate(node.args[0], abort=abort)
+            value = self.evaluate(node.args[0], abort=abort,
+                                  context="as an argument for delay()")
             call_delay = iodelay.SToMU(value, ref_period=self.ref_period)
         elif types.is_builtin(typ, "delay_mu"):
-            value = self.evaluate(node.args[0], abort=abort)
+            value = self.evaluate(node.args[0], abort=abort,
+                                  context="as an argument for delay_mu()")
             call_delay = value
         elif not types.is_builtin(typ):
             if types.is_function(typ):
@@ -297,7 +306,12 @@ class IODelayEstimator(algorithm.Visitor):
                     args[arg_name] = arg_node
 
                 free_vars = delay.duration.free_vars()
-                node.arg_exprs = { arg: self.evaluate(args[arg], abort=abort) for arg in free_vars }
+                node.arg_exprs = {
+                    arg: self.evaluate(args[arg], abort=abort,
+                                       context="in the expression for argument '{}' "
+                                               "that affects I/O delay".format(arg))
+                    for arg in free_vars
+                }
                 call_delay = delay.duration.fold(node.arg_exprs)
             else:
                 assert False

artiq/coredevice/core.py

@@ -19,7 +19,7 @@ from artiq.coredevice import exceptions
 def _render_diagnostic(diagnostic, colored):
     def shorten_path(path):
         return path.replace(artiq_dir, "<artiq>")
-    lines = [shorten_path(path) for path in diagnostic.render(colored)]
+    lines = [shorten_path(path) for path in diagnostic.render(colored=colored)]
     return "\n".join(lines)
 
 class _DiagnosticEngine(diagnostic.Engine):

artiq/gui/console.py

@@ -17,7 +17,7 @@ The following functions are available:
 class ConsoleDock(dockarea.Dock):
     def __init__(self, dataset_sub, dataset_ctl):
         dockarea.Dock.__init__(self, "Console")
-        self.setMinimumSize(QtCore.QSize(850, 300))
+        self.setMinimumSize(QtCore.QSize(720, 300))
         self.dataset_sub = dataset_sub
         self.dataset_ctl = dataset_ctl
         ns = {

artiq/gui/experiments.py

@@ -240,7 +240,7 @@ class _ArgumentEditor(QtGui.QTreeWidget):
 class _ExperimentDock(dockarea.Dock):
     def __init__(self, manager, expurl):
         dockarea.Dock.__init__(self, "Exp: " + expurl, closable=True)
-        self.setMinimumSize(QtCore.QSize(1100, 700))
+        self.setMinimumSize(QtCore.QSize(740, 470))
         self.layout.setSpacing(5)
         self.layout.setContentsMargins(5, 5, 5, 5)
 
@@ -488,7 +488,9 @@ class ExperimentManager:
 
     def open_experiment(self, expurl):
         if expurl in self.open_experiments:
-            return self.open_experiments[expurl]
+            dock = self.open_experiments[expurl]
+            self.dock_area.floatDock(dock)
+            return dock
         dock = _ExperimentDock(self, expurl)
         self.open_experiments[expurl] = dock
         self.dock_area.floatDock(dock)

artiq/gui/log.py

@@ -147,7 +147,7 @@ class _LogFilterProxyModel(QSortFilterProxyModel):
 class _LogDock(dockarea.Dock):
     def __init__(self, manager, name, log_sub):
         dockarea.Dock.__init__(self, name, label="Log")
-        self.setMinimumSize(QtCore.QSize(850, 450))
+        self.setMinimumSize(QtCore.QSize(720, 250))
 
         grid = LayoutWidget()
         self.addWidget(grid)

artiq/gui/schedule.py

@@ -58,7 +58,7 @@ class Model(DictSyncModel):
 class ScheduleDock(dockarea.Dock):
     def __init__(self, status_bar, schedule_ctl, schedule_sub):
         dockarea.Dock.__init__(self, "Schedule")
-        self.setMinimumSize(QtCore.QSize(850, 300))
+        self.setMinimumSize(QtCore.QSize(740, 200))
 
         self.status_bar = status_bar
         self.schedule_ctl = schedule_ctl

artiq/master/worker.py

@@ -42,7 +42,7 @@ def log_worker_exception():
 
 
 class Worker:
-    def __init__(self, handlers=dict(), send_timeout=0.5):
+    def __init__(self, handlers=dict(), send_timeout=1.0):
         self.handlers = handlers
         self.send_timeout = send_timeout
 

artiq/protocols/asyncio_server.py

@@ -7,7 +7,6 @@ class AsyncioServer:
 
     Users of this class must derive from it and define the
     ``_handle_connection_cr`` method and coroutine.
-
     """
     def __init__(self):
         self._client_tasks = set()
@@ -23,15 +22,12 @@ class AsyncioServer:
         :param host: Bind address of the server (see ``asyncio.start_server``
             from the Python standard library).
         :param port: TCP port to bind to.
-
         """
         self.server = await asyncio.start_server(self._handle_connection,
                                                  host, port)
 
     async def stop(self):
-        """Stops the server.
-
-        """
+        """Stops the server."""
         wait_for = copy(self._client_tasks)
         for task in self._client_tasks:
             task.cancel()
@@ -48,6 +44,6 @@ class AsyncioServer:
         self._client_tasks.remove(task)
 
     def _handle_connection(self, reader, writer):
-        task = asyncio.Task(self._handle_connection_cr(reader, writer))
+        task = asyncio.ensure_future(self._handle_connection_cr(reader, writer))
         self._client_tasks.add(task)
         task.add_done_callback(self._client_done)

artiq/protocols/logging.py

@@ -78,7 +78,7 @@ class LogParser:
                 entry = (await stream.readline())
                 if not entry:
                     break
-                self.line_input(entry[:-1].decode())
+                self.line_input(entry.decode().rstrip("\r\n"))
             except:
                 logger.debug("exception in log forwarding", exc_info=True)
                 break
@@ -125,6 +125,9 @@ class Server(AsyncioServer):
                         return
                     source, remainder = linesplit
                     parser.line_input(remainder)
+        except (ConnectionResetError, ConnectionAbortedError, BrokenPipeError):
+            # May happens on Windows when client disconnects
+            pass
         finally:
             writer.close()
 

artiq/protocols/pc_rpc.py

@@ -502,6 +502,9 @@ class Server(_AsyncioServer):
                            "message": traceback.format_exc()}
                 line = pyon.encode(obj) + "\n"
                 writer.write(line.encode())
+        except (ConnectionResetError, ConnectionAbortedError, BrokenPipeError):
+            # May happens on Windows when client disconnects
+            pass
         finally:
             writer.close()
 

artiq/protocols/pipe_ipc.py

@@ -3,6 +3,9 @@ import asyncio
 from asyncio.streams import FlowControlMixin
 
 
+__all__ = ["AsyncioParentComm", "AsyncioChildComm", "ChildComm"]
+
+
 class _BaseIO:
     def write(self, data):
         self.writer.write(data)
@@ -92,11 +95,79 @@ if os.name != "nt":
 
 
 else:  # windows
-    class AsyncioParentComm(_BaseIO):
+    import itertools
+
+
+    _pipe_count = itertools.count()
+
+
+    class AsyncioParentComm:
+        """Requires ProactorEventLoop"""
+        def __init__(self):
+            # We cannot use anonymous pipes on Windows, because we do not know
+            # in advance if the child process wants a handle open in overlapped
+            # mode or not.
+            self.address = "\\\\.\\pipe\\artiq-{}-{}".format(os.getpid(),
+                                                             next(_pipe_count))
+            self.server = None
+            self.ready = asyncio.Event()
+            self.write_buffer = b""
+
+        def get_address(self):
+            return self.address
+
         async def _autoclose(self):
             await self.process.wait()
+            if self.server is not None:
+                self.server[0].close()
+                self.server = None
+            if self.ready.is_set():
                 self.writer.close()
 
+        async def create_subprocess(self, *args, **kwargs):
+            loop = asyncio.get_event_loop()
+
+            def factory():
+                reader = asyncio.StreamReader(loop=loop)
+                protocol = asyncio.StreamReaderProtocol(reader,
+                                                self._child_connected,
+                                                loop=loop)
+                return protocol
+            self.server = await loop.start_serving_pipe(
+                factory, self.address)
+
+            self.process = await asyncio.create_subprocess_exec(
+                *args, **kwargs)
+            asyncio.ensure_future(self._autoclose())
+
+        def _child_connected(self, reader, writer):
+            self.server[0].close()
+            self.server = None
+            self.reader = reader
+            self.writer = writer
+            if self.write_buffer:
+                self.writer.write(self.write_buffer)
+                self.write_buffer = b""
+            self.ready.set()
+
+        def write(self, data):
+            if self.ready.is_set():
+                self.writer.write(data)
+            else:
+                self.write_buffer += data
+
+        async def drain(self):
+            await self.ready.wait()
+            await self.writer.drain()
+
+        async def readline(self):
+            await self.ready.wait()
+            return await self.reader.readline()
+
+        async def read(self, n):
+            await self.ready.wait()
+            return await self.reader.read(n)
+
 
     class AsyncioChildComm(_BaseIO):
         """Requires ProactorEventLoop"""
@@ -109,9 +180,26 @@ else:  # windows
             reader_protocol = asyncio.StreamReaderProtocol(
                 self.reader, loop=loop)
             transport, _ = await loop.create_pipe_connection(
-                self.address, lambda: reader_protocol)
+                lambda: reader_protocol, self.address)
             self.writer = asyncio.StreamWriter(transport, reader_protocol,
                                                self.reader, loop)
 
+        def close(self):
+            self.writer.close()
+
+
     class ChildComm:
-        pass
+        def __init__(self, address):
+            self.f = open(address, "a+b", 0)
+
+        def read(self, n):
+            return self.f.read(n)
+
+        def readline(self):
+            return self.f.readline()
+
+        def write(self, data):
+            return self.f.write(data)
+
+        def close(self):
+            self.f.close()

artiq/protocols/pyon.py

@@ -74,7 +74,10 @@ class _Encoder:
 
     def encode_str(self, x):
         # Do not use repr() for JSON compatibility.
-        tt = {ord("\""): "\\\"", ord("\\"): "\\\\", ord("\n"): "\\n"}
+        tt = {
+              ord("\""): "\\\"", ord("\\"): "\\\\",
+              ord("\n"): "\\n", ord("\r"): "\\r"
+        }
         return "\"" + x.translate(tt) + "\""
 
     def encode_bytes(self, x):

artiq/protocols/sync_struct.py

@@ -235,7 +235,7 @@ class Publisher(AsyncioServer):
                     await writer.drain()
             finally:
                 self._recipients[notifier_name].remove(queue)
-        except (ConnectionResetError, BrokenPipeError):
+        except (ConnectionResetError, ConnectionAbortedError, BrokenPipeError):
             # subscribers disconnecting are a normal occurence
             pass
         finally:

artiq/test/lit/iodelay/error_argument.py

@@ -0,0 +1,12 @@
+# RUN: %python -m artiq.compiler.testbench.signature +diag +delay %s >%t
+# RUN: OutputCheck %s --file-to-check=%t
+
+def f(x):
+    delay_mu(x)
+
+x = 1
+
+def g():
+    # CHECK-L: ${LINE:+2}: error: call cannot be interleaved because an argument cannot be statically evaluated
+    # CHECK-L: ${LINE:+1}: note: this expression is not supported in the expression for argument 'x' that affects I/O delay
+    f(x if True else x)

artiq/test/lit/iodelay/error_arith.py

@@ -3,19 +3,19 @@
 
 def f(a):
     b = 1.0
-    # CHECK-L: ${LINE:+3}: error: this call cannot be interleaved
+    # CHECK-L: ${LINE:+3}: error: call cannot be interleaved
     # CHECK-L: ${LINE:+2}: note: this variable is not an argument of the innermost function
     # CHECK-L: ${LINE:-4}: note: only these arguments are in scope of analysis
     delay(b)
 
 def g():
-    # CHECK-L: ${LINE:+2}: error: this call cannot be interleaved
-    # CHECK-L: ${LINE:+1}: note: this operator is not supported
+    # CHECK-L: ${LINE:+2}: error: call cannot be interleaved
+    # CHECK-L: ${LINE:+1}: note: this operator is not supported as an argument for delay()
     delay(2.0**2)
 
 def h():
-    # CHECK-L: ${LINE:+2}: error: this call cannot be interleaved
-    # CHECK-L: ${LINE:+1}: note: this expression is not supported
+    # CHECK-L: ${LINE:+2}: error: call cannot be interleaved
+    # CHECK-L: ${LINE:+1}: note: this expression is not supported as an argument for delay_mu()
     delay_mu(1 if False else 2)
 
 f(1)

artiq/test/lit/iodelay/error_builtinfn.py

@@ -3,12 +3,12 @@
 
 def f():
     x = 1
-    # CHECK-L: ${LINE:+1}: error: this call cannot be interleaved because an argument cannot be statically evaluated
+    # CHECK-L: ${LINE:+1}: error: call cannot be interleaved because an argument cannot be statically evaluated
     delay_mu(x)
 
 def g():
     x = 1.0
-    # CHECK-L: ${LINE:+1}: error: this call cannot be interleaved
+    # CHECK-L: ${LINE:+1}: error: call cannot be interleaved
     delay(x)
 
 

artiq/test/lit/iodelay/error_call_nested.py

@@ -2,7 +2,7 @@
 # RUN: OutputCheck %s --file-to-check=%t
 
 def f():
-    # CHECK-L: ${LINE:+1}: error: this call cannot be interleaved
+    # CHECK-L: ${LINE:+1}: error: call cannot be interleaved
     delay(1.0**2)
 
 def g():

artiq/test/lit/iodelay/error_call_subst.py

@@ -9,5 +9,5 @@ def pulse(len):
 
 def f():
     a = 100
-    # CHECK-L: ${LINE:+1}: error: this call cannot be interleaved
+    # CHECK-L: ${LINE:+1}: error: call cannot be interleaved
     pulse(a)

artiq/test/lit/iodelay/error_for.py

@@ -3,7 +3,7 @@
 
 def f():
     r = range(10)
-    # CHECK-L: ${LINE:+2}: error: for statement cannot be interleaved because trip count is indeterminate
+    # CHECK-L: ${LINE:+2}: error: for statement cannot be interleaved because iteration count is indeterminate
     # CHECK-L: ${LINE:+1}: note: this value is not a constant range literal
     for _ in r:
         delay_mu(1)

artiq/test/lit/iodelay/error_goto.py

@@ -4,11 +4,11 @@
 def f():
     for _ in range(10):
         delay_mu(10)
-        # CHECK-L: ${LINE:+1}: error: loop trip count is indeterminate because of control flow
+        # CHECK-L: ${LINE:+1}: error: loop iteration count is indeterminate because of control flow
         break
 
 def g():
     for _ in range(10):
         delay_mu(10)
-        # CHECK-L: ${LINE:+1}: error: loop trip count is indeterminate because of control flow
+        # CHECK-L: ${LINE:+1}: error: loop iteration count is indeterminate because of control flow
         continue

artiq/test/lit/iodelay/error_iterable.py

@@ -0,0 +1,10 @@
+# RUN: %python -m artiq.compiler.testbench.signature +diag +delay %s >%t
+# RUN: OutputCheck %s --file-to-check=%t
+
+x = 1
+
+def f():
+    # CHECK-L: ${LINE:+2}: error: for statement cannot be interleaved because iteration count is indeterminate
+    # CHECK-L: ${LINE:+1}: note: this expression is not supported in an iterable used in a for loop that is being interleaved
+    for _ in range(x if True else x):
+        delay_mu(10)

artiq/test/serialization.py

@@ -26,7 +26,7 @@ class PYON(unittest.TestCase):
 _json_test_object = {
     "a": "b",
     "x": [1, 2, {}],
-    "foo\nbaz\\qux\"": ["bar", 1.2, {"x": "y"}],
+    "foo\nbaz\\qux\"\r2": ["bar", 1.2, {"x": "y"}],
     "bar": [True, False, None]
 }
 

conda/artiq/meta.yaml

@@ -54,7 +54,7 @@ requirements:
     - pyqtgraph
     - pygit2
     - aiohttp
-    - binutils-or1k-linux
+    - binutils-or1k-linux # [linux]
     - pythonparser
     - levenshtein