From 9153c4d8a33470a6bf5cfb11dd7eaf06d0587b0d Mon Sep 17 00:00:00 2001 From: Sebastien Bourdeauducq Date: Sat, 7 Jul 2018 17:04:56 +0800 Subject: [PATCH] use tokenize.open() to open Python source files Fixes encoding issues especially with device databases modified in obscure editors. --- artiq/compiler/testbench/embedding.py | 4 ++-- artiq/compiler/testbench/perf_embedding.py | 4 ++-- artiq/master/databases.py | 3 ++- 3 files changed, 6 insertions(+), 5 deletions(-) diff --git a/artiq/compiler/testbench/embedding.py b/artiq/compiler/testbench/embedding.py index f79bb5a68..c637c4b3a 100644 --- a/artiq/compiler/testbench/embedding.py +++ b/artiq/compiler/testbench/embedding.py @@ -1,4 +1,4 @@ -import sys, os +import sys, os, tokenize from artiq.master.databases import DeviceDB from artiq.master.worker_db import DeviceManager @@ -27,7 +27,7 @@ def main(): ddb_path = os.path.join(os.path.dirname(sys.argv[1]), "device_db.py") dmgr = DeviceManager(DeviceDB(ddb_path)) - with open(sys.argv[1]) as f: + with tokenize.open(sys.argv[1]) as f: testcase_code = compile(f.read(), f.name, "exec") testcase_vars = {'__name__': 'testbench', 'dmgr': dmgr} exec(testcase_code, testcase_vars) diff --git a/artiq/compiler/testbench/perf_embedding.py b/artiq/compiler/testbench/perf_embedding.py index b94b88735..41f09cb04 100644 --- a/artiq/compiler/testbench/perf_embedding.py +++ b/artiq/compiler/testbench/perf_embedding.py @@ -1,4 +1,4 @@ -import sys, os +import sys, os, tokenize from pythonparser import diagnostic from ...language.environment import ProcessArgumentManager from ...master.databases import DeviceDB, DatasetDB @@ -22,7 +22,7 @@ def main(): engine = diagnostic.Engine() engine.process = process_diagnostic - with open(sys.argv[1]) as f: + with tokenize.open(sys.argv[1]) as f: testcase_code = compile(f.read(), f.name, "exec") testcase_vars = {'__name__': 'testbench'} exec(testcase_code, testcase_vars) diff --git a/artiq/master/databases.py b/artiq/master/databases.py index e71862d44..e279b3c56 100644 --- a/artiq/master/databases.py +++ b/artiq/master/databases.py @@ -1,4 +1,5 @@ import asyncio +import tokenize from artiq.protocols.sync_struct import Notifier, process_mod from artiq.protocols import pyon @@ -7,7 +8,7 @@ from artiq.tools import TaskObject def device_db_from_file(filename): glbs = dict() - with open(filename, "r") as f: + with tokenize.open(filename) as f: exec(f.read(), glbs) return glbs["device_db"]