Merge remote-tracking branch 'm-labs/newtesting' into subprocess-termination

* m-labs/newtesting:
  style
  assume 'import artiq' works before running tests
  test/ctlmgr: use sys.executable, use default localhost binding

artiq/compiler/testbench/jit.py

@@ -5,7 +5,7 @@ from ..module import Module, Source
 from ..targets import NativeTarget
 
 def main():
-    libartiq_support = os.getenv('LIBARTIQ_SUPPORT')
+    libartiq_support = os.getenv("LIBARTIQ_SUPPORT")
     if libartiq_support is not None:
         llvm.load_library_permanently(libartiq_support)
 

artiq/test/harness.py

@@ -1,35 +0,0 @@
-"""
-The purpose of this harness is to emulate the behavior of
-the python executable, but add the ARTIQ root to sys.path
-beforehand.
-
-This is necessary because eggs override the PYTHONPATH environment
-variable, but not current directory; therefore `python -m artiq...`
-ran from the ARTIQ root would work, but there is no simple way to
-emulate the same behavior when invoked under lit.
-"""
-
-import sys, os, argparse, importlib
-
-def main():
-    parser = argparse.ArgumentParser(description=__doc__)
-    parser.add_argument("-m", metavar="mod", type=str,
-                        help="run library module as a script")
-    parser.add_argument("args", type=str, nargs="+",
-                        help="arguments passed to program in sys.argv[1:]")
-    args = parser.parse_args(sys.argv[1:])
-
-    artiq_path = os.path.abspath(os.path.join(os.path.dirname(__file__), ".."))
-    sys.path.insert(1, artiq_path)
-
-    if args.m:
-        sys.argv[1:] = args.args
-        importlib.import_module(args.m).main()
-    else:
-        sys.argv[1:] = args.args[1:]
-        with open(args.args[0]) as f:
-            code = compile(f.read(), args.args[0], "exec")
-            exec(code, globals())
-
-if __name__ == "__main__":
-    main()

artiq/test/lit/lit.cfg

@@ -1,35 +1,35 @@
 # -*- python -*-
 
-import os, subprocess
+import os
+import sys
+import subprocess
 import lit.util
 import lit.formats
 
-root = os.path.join(os.path.dirname(__file__), '..')
+root = os.path.join(os.path.dirname(__file__), "..")
 
-config.name = 'ARTIQ'
+config.name = "ARTIQ"
 config.test_format = lit.formats.ShTest()
-config.suffixes = ['.py']
+config.suffixes = [".py"]
 
-python_executable = 'python3.5'
+if os.getenv("COVERAGE"):
-
+    config.environment["COVERAGE_FILE"] = os.path.join(root, "..", ".coverage")
-harness = os.path.join(root, 'harness.py')
+    python = "coverage run --parallel-mode --source=artiq"
-if os.getenv('COVERAGE'):
-    harness = 'coverage run --parallel-mode --source=artiq {}'.format(harness)
-    config.environment['COVERAGE_FILE'] = os.path.join(root, '..', '.coverage')
-    config.substitutions.append( ('%python', harness) )
 else:
-    harness = '{} {}'.format(python_executable, harness)
+    python = sys.executable
-    config.substitutions.append( ('%python', harness) )
+config.substitutions.append( ("%python", python) )
 
-not_ = '{} {}'.format(python_executable, os.path.join(root, 'not.py'))
+config.environment["PYTHONPATH"] = os.getenv("PYTHONPATH")
-config.substitutions.append( ('%not', not_) )
 
-if os.name == 'posix':
+not_ = "{} {}".format(sys.executable, os.path.join(root, "not.py"))
-    support_build = os.path.join(root, 'libartiq_support')
+config.substitutions.append( ("%not", not_) )
-    if subprocess.call(['make', '-sC', support_build]) != 0:
+
+if os.name == "posix":
+    support_build = os.path.join(root, "libartiq_support")
+    if subprocess.call(["make", "-sC", support_build]) != 0:
         lit_config.fatal("Unable to build JIT support library")
 
-    support_lib = os.path.join(support_build, 'libartiq_support.so')
+    support_lib = os.path.join(support_build, "libartiq_support.so")
-    config.environment['LIBARTIQ_SUPPORT'] = support_lib
+    config.environment["LIBARTIQ_SUPPORT"] = support_lib
 
-    config.available_features.add('exceptions')
+    config.available_features.add("exceptions")

artiq/test/test_ctlmgr.py

@@ -1,4 +1,5 @@
 import os
+import sys
 import unittest
 import logging
 import asyncio
@@ -55,12 +56,13 @@ class ControllerCase(unittest.TestCase):
         raise asyncio.TimeoutError
 
     def test_start_ping_stop_controller(self):
+        command = sys.executable + " -m "
         entry = {
             "type": "controller",
             "host": "::1",
             "port": 3253,
-            "command": "lda_controller -p {port} --bind {bind} "
+            "command": sys.executable + " -m artiq.frontend.lda_controller "
-                    "--no-localhost-bind --simulation",
+                    "-p {port} --simulation"
         }
         async def test():
             await self.start("lda_sim", entry)

doc/manual/faq.rst

@@ -86,3 +86,8 @@ URL: it allows to select the serial device by its USB vendor ID, product
 ID and/or serial number. Those never change, unlike the device file name.
 
 See the :ref:`TDC001 documentation <tdc001-controller-usage-example>` for an example of ``hwgrep://`` usage.
+
+run unit tests?
+---------------
+
+The unit tests assume that the Python environment has been set up in such a way that ``import artiq`` will import the code being tested, and that this is still true for any subprocess created. This is not the way setuptools operates as it adds the path to ARTIQ to ``sys.path`` which is not passed to subprocesses; as a result, running the tests via ``setup.py`` is not supported. The user must first install the package or set ``PYTHONPATH``, and then run the tests with e.g. ``python3.5 -m unittest discover`` in the ``artiq/test`` folder and ``lit .`` in the ``artiq/test/lit`` folder.

setup.py

@@ -57,7 +57,6 @@ setup(
     ],
     packages=find_packages(),
     namespace_packages=[],
-    test_suite="artiq.test",
     include_package_data=True,
     ext_modules=[],
     entry_points={