From 9153c4d8a33470a6bf5cfb11dd7eaf06d0587b0d Mon Sep 17 00:00:00 2001
From: Sebastien Bourdeauducq <sb@m-labs.hk>
Date: Sat, 7 Jul 2018 17:04:56 +0800
Subject: [PATCH] use tokenize.open() to open Python source files

Fixes encoding issues especially with device databases modified in obscure editors.
---
 artiq/compiler/testbench/embedding.py      | 4 ++--
 artiq/compiler/testbench/perf_embedding.py | 4 ++--
 artiq/master/databases.py                  | 3 ++-
 3 files changed, 6 insertions(+), 5 deletions(-)

diff --git a/artiq/compiler/testbench/embedding.py b/artiq/compiler/testbench/embedding.py
index f79bb5a68..c637c4b3a 100644
--- a/artiq/compiler/testbench/embedding.py
+++ b/artiq/compiler/testbench/embedding.py
@@ -1,4 +1,4 @@
-import sys, os
+import sys, os, tokenize
 
 from artiq.master.databases import DeviceDB
 from artiq.master.worker_db import DeviceManager
@@ -27,7 +27,7 @@ def main():
     ddb_path = os.path.join(os.path.dirname(sys.argv[1]), "device_db.py")
     dmgr = DeviceManager(DeviceDB(ddb_path))
 
-    with open(sys.argv[1]) as f:
+    with tokenize.open(sys.argv[1]) as f:
         testcase_code = compile(f.read(), f.name, "exec")
         testcase_vars = {'__name__': 'testbench', 'dmgr': dmgr}
         exec(testcase_code, testcase_vars)
diff --git a/artiq/compiler/testbench/perf_embedding.py b/artiq/compiler/testbench/perf_embedding.py
index b94b88735..41f09cb04 100644
--- a/artiq/compiler/testbench/perf_embedding.py
+++ b/artiq/compiler/testbench/perf_embedding.py
@@ -1,4 +1,4 @@
-import sys, os
+import sys, os, tokenize
 from pythonparser import diagnostic
 from ...language.environment import ProcessArgumentManager
 from ...master.databases import DeviceDB, DatasetDB
@@ -22,7 +22,7 @@ def main():
     engine = diagnostic.Engine()
     engine.process = process_diagnostic
 
-    with open(sys.argv[1]) as f:
+    with tokenize.open(sys.argv[1]) as f:
         testcase_code = compile(f.read(), f.name, "exec")
         testcase_vars = {'__name__': 'testbench'}
         exec(testcase_code, testcase_vars)
diff --git a/artiq/master/databases.py b/artiq/master/databases.py
index e71862d44..e279b3c56 100644
--- a/artiq/master/databases.py
+++ b/artiq/master/databases.py
@@ -1,4 +1,5 @@
 import asyncio
+import tokenize
 
 from artiq.protocols.sync_struct import Notifier, process_mod
 from artiq.protocols import pyon
@@ -7,7 +8,7 @@ from artiq.tools import TaskObject
 
 def device_db_from_file(filename):
     glbs = dict()
-    with open(filename, "r") as f:
+    with tokenize.open(filename) as f:
         exec(f.read(), glbs)
     return glbs["device_db"]