use tokenize.open() to open Python source files

Fixes encoding issues especially with device databases modified in obscure editors.

artiq/compiler/testbench/embedding.py

@@ -1,4 +1,4 @@
-import sys, os
+import sys, os, tokenize
 
 from artiq.master.databases import DeviceDB
 from artiq.master.worker_db import DeviceManager
@@ -27,7 +27,7 @@ def main():
     ddb_path = os.path.join(os.path.dirname(sys.argv[1]), "device_db.py")
     dmgr = DeviceManager(DeviceDB(ddb_path))
 
-    with open(sys.argv[1]) as f:
+    with tokenize.open(sys.argv[1]) as f:
         testcase_code = compile(f.read(), f.name, "exec")
         testcase_vars = {'__name__': 'testbench', 'dmgr': dmgr}
         exec(testcase_code, testcase_vars)

artiq/compiler/testbench/perf_embedding.py

@@ -1,4 +1,4 @@
-import sys, os
+import sys, os, tokenize
 from pythonparser import diagnostic
 from ...language.environment import ProcessArgumentManager
 from ...master.databases import DeviceDB, DatasetDB
@@ -22,7 +22,7 @@ def main():
     engine = diagnostic.Engine()
     engine.process = process_diagnostic
 
-    with open(sys.argv[1]) as f:
+    with tokenize.open(sys.argv[1]) as f:
         testcase_code = compile(f.read(), f.name, "exec")
         testcase_vars = {'__name__': 'testbench'}
         exec(testcase_code, testcase_vars)

artiq/master/databases.py

@@ -1,4 +1,5 @@
 import asyncio
+import tokenize
 
 from artiq.protocols.sync_struct import Notifier, process_mod
 from artiq.protocols import pyon
@@ -7,7 +8,7 @@ from artiq.tools import TaskObject
 
 def device_db_from_file(filename):
     glbs = dict()
-    with open(filename, "r") as f:
+    with tokenize.open(filename) as f:
         exec(f.read(), glbs)
     return glbs["device_db"]