compiler.testbench.perf_embedding: implement.

artiq/compiler/testbench/__init__.py

@@ -0,0 +1,21 @@
+import time, cProfile as profile, pstats
+
+def benchmark(f, name):
+    profiler = profile.Profile()
+    profiler.enable()
+
+    start = time.perf_counter()
+    end   = 0
+    runs  = 0
+    while end - start < 5 or runs < 10:
+        f()
+        runs += 1
+        end = time.perf_counter()
+
+    profiler.create_stats()
+
+    print("{} {} runs: {:.2f}s, {:.2f}ms/run".format(
+            runs, name, end - start, (end - start) / runs * 1000))
+
+    stats = pstats.Stats(profiler)
+    stats.strip_dirs().sort_stats('time').print_stats(10)

artiq/compiler/testbench/perf.py

@@ -1,11 +1,12 @@
-import sys, os, time, cProfile as profile, pstats
+import sys, os
 from pythonparser import diagnostic
 from .. import Module, Source
 from ..targets import OR1KTarget
+from . import benchmark
 
 def main():
-    if not len(sys.argv) > 1:
+    if not len(sys.argv) == 2:
-        print("Expected at least one module filename", file=sys.stderr)
+        print("Expected exactly one module filename", file=sys.stderr)
         exit(1)
 
     def process_diagnostic(diag):
@@ -17,38 +18,19 @@ def main():
     engine.process = process_diagnostic
 
     # Make sure everything's valid
-    modules = [Module(Source.from_filename(filename, engine=engine))
+    filename = sys.argv[1]
-               for filename in sys.argv[1:]]
-
-    def benchmark(f, name):
-        profiler = profile.Profile()
-        profiler.enable()
-
-        start = time.perf_counter()
-        end   = 0
-        runs  = 0
-        while end - start < 5 or runs < 10:
-            f()
-            runs += 1
-            end = time.perf_counter()
-
-        profiler.create_stats()
-
-        print("{} {} runs: {:.2f}s, {:.2f}ms/run".format(
-                runs, name, end - start, (end - start) / runs * 1000))
-
-        stats = pstats.Stats(profiler)
-        stats.strip_dirs().sort_stats('time').print_stats(10)
-
-    sources = []
-    for filename in sys.argv[1:]:
     with open(filename) as f:
-            sources.append(f.read())
+        code = f.read()
+    source = Source.from_string(code, filename, engine=engine)
+    module = Module(source)
 
-    benchmark(lambda: [Module.from_string(src) for src in sources],
+    benchmark(lambda: Source.from_string(code, filename),
-              "ARTIQ typechecking and transforms")
+              "ARTIQ parsing and inference")
 
-    benchmark(lambda: OR1KTarget().compile_and_link(modules),
+    benchmark(lambda: Module(source),
+              "ARTIQ transforms and validators")
+
+    benchmark(lambda: OR1KTarget().compile_and_link([module]),
               "LLVM optimization and linking")
 
 if __name__ == "__main__":

artiq/compiler/testbench/perf_embedding.py

@@ -0,0 +1,52 @@
+import sys, os
+from pythonparser import diagnostic
+from ...protocols.file_db import FlatFileDB
+from ...master.worker_db import DeviceManager
+from .. import Module
+from ..embedding import Stitcher
+from ..targets import OR1KTarget
+from . import benchmark
+
+def main():
+    if not len(sys.argv) == 2:
+        print("Expected exactly one module filename", file=sys.stderr)
+        exit(1)
+
+    def process_diagnostic(diag):
+        print("\n".join(diag.render()), file=sys.stderr)
+        if diag.level in ("fatal", "error"):
+            exit(1)
+
+    engine = diagnostic.Engine()
+    engine.process = process_diagnostic
+
+    with open(sys.argv[1]) as f:
+        testcase_code = compile(f.read(), f.name, "exec")
+        testcase_vars = {'__name__': 'testbench'}
+        exec(testcase_code, testcase_vars)
+
+    ddb_path = os.path.join(os.path.dirname(sys.argv[1]), "ddb.pyon")
+    dmgr = DeviceManager(FlatFileDB(ddb_path))
+
+    def embed():
+        experiment = testcase_vars["Benchmark"](dmgr)
+
+        stitcher = Stitcher()
+        stitcher.stitch_call(experiment.run, (experiment,), {})
+        stitcher.finalize()
+        return stitcher
+
+    stitcher = embed()
+    module = Module(stitcher)
+
+    benchmark(lambda: embed(),
+              "ARTIQ embedding")
+
+    benchmark(lambda: Module(stitcher),
+              "ARTIQ transforms and validators")
+
+    benchmark(lambda: OR1KTarget().compile_and_link([module]),
+              "LLVM optimization and linking")
+
+if __name__ == "__main__":
+    main()