use tokenize.open() to open Python source files

Fixes encoding issues especially with device databases modified in obscure editors.
This commit is contained in:
Sebastien Bourdeauducq 2018-07-07 17:04:56 +08:00
parent 4420046502
commit 9153c4d8a3
3 changed files with 6 additions and 5 deletions

View File

@ -1,4 +1,4 @@
import sys, os
import sys, os, tokenize
from artiq.master.databases import DeviceDB
from artiq.master.worker_db import DeviceManager
@ -27,7 +27,7 @@ def main():
ddb_path = os.path.join(os.path.dirname(sys.argv[1]), "device_db.py")
dmgr = DeviceManager(DeviceDB(ddb_path))
with open(sys.argv[1]) as f:
with tokenize.open(sys.argv[1]) as f:
testcase_code = compile(f.read(), f.name, "exec")
testcase_vars = {'__name__': 'testbench', 'dmgr': dmgr}
exec(testcase_code, testcase_vars)

View File

@ -1,4 +1,4 @@
import sys, os
import sys, os, tokenize
from pythonparser import diagnostic
from ...language.environment import ProcessArgumentManager
from ...master.databases import DeviceDB, DatasetDB
@ -22,7 +22,7 @@ def main():
engine = diagnostic.Engine()
engine.process = process_diagnostic
with open(sys.argv[1]) as f:
with tokenize.open(sys.argv[1]) as f:
testcase_code = compile(f.read(), f.name, "exec")
testcase_vars = {'__name__': 'testbench'}
exec(testcase_code, testcase_vars)

View File

@ -1,4 +1,5 @@
import asyncio
import tokenize
from artiq.protocols.sync_struct import Notifier, process_mod
from artiq.protocols import pyon
@ -7,7 +8,7 @@ from artiq.tools import TaskObject
def device_db_from_file(filename):
glbs = dict()
with open(filename, "r") as f:
with tokenize.open(filename) as f:
exec(f.read(), glbs)
return glbs["device_db"]