From 7ce9bdf54d24052f95ac49ebc9ba10f5319dd1a2 Mon Sep 17 00:00:00 2001 From: whitequark Date: Thu, 2 Jul 2015 19:35:35 +0300 Subject: [PATCH] Move transforms to artiq.compiler.transforms, add artiq.Module. --- artiq/compiler/__init__.py | 1 + artiq/compiler/module.py | 33 ++ artiq/compiler/testbench/__init__.py | 0 artiq/compiler/testbench/inferencer.py | 75 +++ artiq/compiler/transforms/__init__.py | 2 + .../compiler/transforms/asttyped_rewriter.py | 377 +++++++++++++++ .../{typing.py => transforms/inferencer.py} | 452 +----------------- .../{typing => inferencer}/builtin_calls.py | 2 +- .../compiler/{typing => inferencer}/coerce.py | 2 +- .../error_builtin_calls.py | 2 +- .../{typing => inferencer}/error_call.py | 2 +- .../{typing => inferencer}/error_coerce.py | 2 +- .../error_control_flow.py | 2 +- .../{typing => inferencer}/error_exception.py | 2 +- .../{typing => inferencer}/error_iterable.py | 2 +- .../error_local_unbound.py | 2 +- .../{typing => inferencer}/error_locals.py | 2 +- .../{typing => inferencer}/error_return.py | 2 +- .../{typing => inferencer}/error_unify.py | 2 +- .../{typing => inferencer}/exception.py | 2 +- .../compiler/{typing => inferencer}/gcd.py | 2 +- .../{typing => inferencer}/prelude.py | 2 +- .../{typing => inferencer}/scoping.py | 2 +- .../compiler/{typing => inferencer}/unify.py | 2 +- 24 files changed, 508 insertions(+), 466 deletions(-) create mode 100644 artiq/compiler/module.py create mode 100644 artiq/compiler/testbench/__init__.py create mode 100644 artiq/compiler/testbench/inferencer.py create mode 100644 artiq/compiler/transforms/__init__.py create mode 100644 artiq/compiler/transforms/asttyped_rewriter.py rename artiq/compiler/{typing.py => transforms/inferencer.py} (68%) rename lit-test/compiler/{typing => inferencer}/builtin_calls.py (92%) rename lit-test/compiler/{typing => inferencer}/coerce.py (95%) rename lit-test/compiler/{typing => inferencer}/error_builtin_calls.py (86%) rename lit-test/compiler/{typing => inferencer}/error_call.py (86%) rename lit-test/compiler/{typing => inferencer}/error_coerce.py (96%) rename lit-test/compiler/{typing => inferencer}/error_control_flow.py (87%) rename lit-test/compiler/{typing => inferencer}/error_exception.py (80%) rename lit-test/compiler/{typing => inferencer}/error_iterable.py (64%) rename lit-test/compiler/{typing => inferencer}/error_local_unbound.py (61%) rename lit-test/compiler/{typing => inferencer}/error_locals.py (93%) rename lit-test/compiler/{typing => inferencer}/error_return.py (88%) rename lit-test/compiler/{typing => inferencer}/error_unify.py (92%) rename lit-test/compiler/{typing => inferencer}/exception.py (76%) rename lit-test/compiler/{typing => inferencer}/gcd.py (80%) rename lit-test/compiler/{typing => inferencer}/prelude.py (71%) rename lit-test/compiler/{typing => inferencer}/scoping.py (63%) rename lit-test/compiler/{typing => inferencer}/unify.py (94%) diff --git a/artiq/compiler/__init__.py b/artiq/compiler/__init__.py index e69de29bb..8f210ac92 100644 --- a/artiq/compiler/__init__.py +++ b/artiq/compiler/__init__.py @@ -0,0 +1 @@ +from .module import Module diff --git a/artiq/compiler/module.py b/artiq/compiler/module.py new file mode 100644 index 000000000..82b1ef455 --- /dev/null +++ b/artiq/compiler/module.py @@ -0,0 +1,33 @@ +""" +The :class:`Module` class encapsulates a single Python +""" + +import os +from pythonparser import source, diagnostic, parse_buffer +from . import prelude, types, transforms + +class Module: + def __init__(self, source_buffer, engine=diagnostic.Engine(all_errors_are_fatal=True)): + parsetree, comments = parse_buffer(source_buffer, engine=engine) + self.name = os.path.basename(source_buffer.name) + + asttyped_rewriter = transforms.ASTTypedRewriter(engine=engine) + typedtree = asttyped_rewriter.visit(parsetree) + self.globals = asttyped_rewriter.globals + + inferencer = transforms.Inferencer(engine=engine) + inferencer.visit(typedtree) + + @classmethod + def from_string(klass, source_string, name="input.py", first_line=1): + return klass(source.Buffer(source_string + "\n", name, first_line)) + + @classmethod + def from_filename(klass, filename): + with open(filename) as f: + return klass(source.Buffer(f.read(), filename, 1)) + + def __repr__(self): + printer = types.TypePrinter() + globals = ["%s: %s" % (var, printer.name(self.globals[var])) for var in self.globals] + return "" % (repr(self.name), ",\n ".join(globals)) diff --git a/artiq/compiler/testbench/__init__.py b/artiq/compiler/testbench/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/artiq/compiler/testbench/inferencer.py b/artiq/compiler/testbench/inferencer.py new file mode 100644 index 000000000..b330bf055 --- /dev/null +++ b/artiq/compiler/testbench/inferencer.py @@ -0,0 +1,75 @@ +import sys, fileinput, os +from pythonparser import source, diagnostic, algorithm, parse_buffer +from .. import prelude, types +from ..transforms import ASTTypedRewriter, Inferencer + +class Printer(algorithm.Visitor): + """ + :class:`Printer` prints ``:`` and the node type after every typed node, + and ``->`` and the node type before the colon in a function definition. + + In almost all cases (except function definition) this does not result + in valid Python syntax. + + :ivar rewriter: (:class:`pythonparser.source.Rewriter`) rewriter instance + """ + + def __init__(self, buf): + self.rewriter = source.Rewriter(buf) + self.type_printer = types.TypePrinter() + + def rewrite(self): + return self.rewriter.rewrite() + + def visit_FunctionDefT(self, node): + super().generic_visit(node) + + self.rewriter.insert_before(node.colon_loc, + "->{}".format(self.type_printer.name(node.return_type))) + + def visit_ExceptHandlerT(self, node): + super().generic_visit(node) + + if node.name_loc: + self.rewriter.insert_after(node.name_loc, + ":{}".format(self.type_printer.name(node.name_type))) + + def generic_visit(self, node): + super().generic_visit(node) + + if hasattr(node, "type"): + self.rewriter.insert_after(node.loc, + ":{}".format(self.type_printer.name(node.type))) + +def main(): + if len(sys.argv) > 1 and sys.argv[1] == '+diag': + del sys.argv[1] + def process_diagnostic(diag): + print("\n".join(diag.render(only_line=True))) + if diag.level == 'fatal': + exit() + else: + def process_diagnostic(diag): + print("\n".join(diag.render())) + if diag.level in ('fatal', 'error'): + exit(1) + + engine = diagnostic.Engine() + engine.process = process_diagnostic + + buf = source.Buffer("".join(fileinput.input()).expandtabs(), + os.path.basename(fileinput.filename())) + parsed, comments = parse_buffer(buf, engine=engine) + typed = ASTTypedRewriter(globals=prelude.globals(), engine=engine).visit(parsed) + Inferencer(engine=engine).visit(typed) + + printer = Printer(buf) + printer.visit(typed) + for comment in comments: + if comment.text.find("CHECK") >= 0: + printer.rewriter.remove(comment.loc) + print(printer.rewrite().source) + + +if __name__ == "__main__": + main() diff --git a/artiq/compiler/transforms/__init__.py b/artiq/compiler/transforms/__init__.py new file mode 100644 index 000000000..749508a5a --- /dev/null +++ b/artiq/compiler/transforms/__init__.py @@ -0,0 +1,2 @@ +from .asttyped_rewriter import ASTTypedRewriter +from .inferencer import Inferencer diff --git a/artiq/compiler/transforms/asttyped_rewriter.py b/artiq/compiler/transforms/asttyped_rewriter.py new file mode 100644 index 000000000..065c05ee0 --- /dev/null +++ b/artiq/compiler/transforms/asttyped_rewriter.py @@ -0,0 +1,377 @@ +from pythonparser import algorithm, diagnostic +from .. import asttyped, types, builtins, prelude + +# This visitor will be called for every node with a scope, +# i.e.: class, function, comprehension, lambda +class LocalExtractor(algorithm.Visitor): + def __init__(self, env_stack, engine): + super().__init__() + self.env_stack = env_stack + self.engine = engine + + self.in_root = False + self.in_assign = False + self.typing_env = {} + + # which names are global have to be recorded in the current scope + self.global_ = set() + + # which names are nonlocal only affects whether the current scope + # gets a new binding or not, so we throw this away + self.nonlocal_ = set() + + # parameters can't be declared as global or nonlocal + self.params = set() + + def visit_in_assign(self, node): + try: + self.in_assign = True + return self.visit(node) + finally: + self.in_assign = False + + def visit_Assign(self, node): + self.visit(node.value) + for target in node.targets: + self.visit_in_assign(target) + + def visit_For(self, node): + self.visit(node.iter) + self.visit_in_assign(node.target) + self.visit(node.body) + self.visit(node.orelse) + + def visit_withitem(self, node): + self.visit(node.context_expr) + if node.optional_vars is not None: + self.visit_in_assign(node.optional_vars) + + def visit_comprehension(self, node): + self.visit(node.iter) + self.visit_in_assign(node.target) + for if_ in node.ifs: + self.visit(node.ifs) + + def visit_root(self, node): + if self.in_root: + return + self.in_root = True + self.generic_visit(node) + + visit_Module = visit_root # don't look at inner scopes + visit_ClassDef = visit_root + visit_Lambda = visit_root + visit_DictComp = visit_root + visit_ListComp = visit_root + visit_SetComp = visit_root + visit_GeneratorExp = visit_root + + def visit_FunctionDef(self, node): + if self.in_root: + self._assignable(node.name) + self.visit_root(node) + + def _assignable(self, name): + if name not in self.typing_env and name not in self.nonlocal_: + self.typing_env[name] = types.TVar() + + def visit_arg(self, node): + if node.arg in self.params: + diag = diagnostic.Diagnostic("error", + "duplicate parameter '{name}'", {"name": node.arg}, + node.loc) + self.engine.process(diag) + return + self._assignable(node.arg) + self.params.add(node.arg) + + def visit_Name(self, node): + if self.in_assign: + # code like: + # x = 1 + # def f(): + # x = 1 + # creates a new binding for x in f's scope + self._assignable(node.id) + + def _check_not_in(self, name, names, curkind, newkind, loc): + if name in names: + diag = diagnostic.Diagnostic("error", + "name '{name}' cannot be {curkind} and {newkind} simultaneously", + {"name": name, "curkind": curkind, "newkind": newkind}, loc) + self.engine.process(diag) + return True + return False + + def visit_Global(self, node): + for name, loc in zip(node.names, node.name_locs): + if self._check_not_in(name, self.nonlocal_, "nonlocal", "global", loc) or \ + self._check_not_in(name, self.params, "a parameter", "global", loc): + continue + + self.global_.add(name) + self._assignable(name) + self.env_stack[1][name] = self.typing_env[name] + + def visit_Nonlocal(self, node): + for name, loc in zip(node.names, node.name_locs): + if self._check_not_in(name, self.global_, "global", "nonlocal", loc) or \ + self._check_not_in(name, self.params, "a parameter", "nonlocal", loc): + continue + + # nonlocal does not search prelude and global scopes + found = False + for outer_env in reversed(self.env_stack[2:]): + if name in outer_env: + found = True + break + if not found: + diag = diagnostic.Diagnostic("error", + "cannot declare name '{name}' as nonlocal: it is not bound in any outer scope", + {"name": name}, + loc, [node.keyword_loc]) + self.engine.process(diag) + continue + + self.nonlocal_.add(name) + + def visit_ExceptHandler(self, node): + self.visit(node.type) + self._assignable(node.name) + for stmt in node.body: + self.visit(stmt) + + +class ASTTypedRewriter(algorithm.Transformer): + """ + :class:`ASTTypedRewriter` converts an untyped AST to a typed AST + where all type fields of non-literals are filled with fresh type variables, + and type fields of literals are filled with corresponding types. + + :class:`ASTTypedRewriter` also discovers the scope of variable bindings + via :class:`LocalExtractor`. + """ + + def __init__(self, engine): + self.engine = engine + self.globals = None + self.env_stack = [prelude.globals()] + + def _find_name(self, name, loc): + for typing_env in reversed(self.env_stack): + if name in typing_env: + return typing_env[name] + diag = diagnostic.Diagnostic("fatal", + "name '{name}' is not bound to anything", {"name":name}, loc) + self.engine.process(diag) + + # Visitors that replace node with a typed node + # + def visit_Module(self, node): + extractor = LocalExtractor(env_stack=self.env_stack, engine=self.engine) + extractor.visit(node) + + node = asttyped.ModuleT( + typing_env=extractor.typing_env, globals_in_scope=extractor.global_, + body=node.body, loc=node.loc) + self.globals = node.typing_env + + try: + self.env_stack.append(node.typing_env) + return self.generic_visit(node) + finally: + self.env_stack.pop() + + def visit_FunctionDef(self, node): + extractor = LocalExtractor(env_stack=self.env_stack, engine=self.engine) + extractor.visit(node) + + node = asttyped.FunctionDefT( + typing_env=extractor.typing_env, globals_in_scope=extractor.global_, + signature_type=self._find_name(node.name, node.name_loc), return_type=types.TVar(), + name=node.name, args=node.args, returns=node.returns, + body=node.body, decorator_list=node.decorator_list, + keyword_loc=node.keyword_loc, name_loc=node.name_loc, + arrow_loc=node.arrow_loc, colon_loc=node.colon_loc, at_locs=node.at_locs, + loc=node.loc) + + try: + self.env_stack.append(node.typing_env) + return self.generic_visit(node) + finally: + self.env_stack.pop() + + def visit_arg(self, node): + return asttyped.argT(type=self._find_name(node.arg, node.loc), + arg=node.arg, annotation=self.visit(node.annotation), + arg_loc=node.arg_loc, colon_loc=node.colon_loc, loc=node.loc) + + def visit_Num(self, node): + if isinstance(node.n, int): + typ = builtins.TInt() + elif isinstance(node.n, float): + typ = builtins.TFloat() + else: + diag = diagnostic.Diagnostic("fatal", + "numeric type {type} is not supported", {"type": node.n.__class__.__name__}, + node.loc) + self.engine.process(diag) + return asttyped.NumT(type=typ, + n=node.n, loc=node.loc) + + def visit_Name(self, node): + return asttyped.NameT(type=self._find_name(node.id, node.loc), + id=node.id, ctx=node.ctx, loc=node.loc) + + def visit_NameConstant(self, node): + if node.value is True or node.value is False: + typ = builtins.TBool() + elif node.value is None: + typ = builtins.TNone() + return asttyped.NameConstantT(type=typ, value=node.value, loc=node.loc) + + def visit_Tuple(self, node): + node = self.generic_visit(node) + return asttyped.TupleT(type=types.TTuple([x.type for x in node.elts]), + elts=node.elts, ctx=node.ctx, loc=node.loc) + + def visit_List(self, node): + node = self.generic_visit(node) + node = asttyped.ListT(type=builtins.TList(), + elts=node.elts, ctx=node.ctx, loc=node.loc) + return self.visit(node) + + def visit_Attribute(self, node): + node = self.generic_visit(node) + node = asttyped.AttributeT(type=types.TVar(), + value=node.value, attr=node.attr, ctx=node.ctx, + dot_loc=node.dot_loc, attr_loc=node.attr_loc, loc=node.loc) + return self.visit(node) + + def visit_Subscript(self, node): + node = self.generic_visit(node) + node = asttyped.SubscriptT(type=types.TVar(), + value=node.value, slice=node.slice, ctx=node.ctx, + loc=node.loc) + return self.visit(node) + + def visit_BoolOp(self, node): + node = self.generic_visit(node) + node = asttyped.BoolOpT(type=types.TVar(), + op=node.op, values=node.values, + op_locs=node.op_locs, loc=node.loc) + return self.visit(node) + + def visit_UnaryOp(self, node): + node = self.generic_visit(node) + node = asttyped.UnaryOpT(type=types.TVar(), + op=node.op, operand=node.operand, + loc=node.loc) + return self.visit(node) + + def visit_BinOp(self, node): + node = self.generic_visit(node) + node = asttyped.BinOpT(type=types.TVar(), + left=node.left, op=node.op, right=node.right, + loc=node.loc) + return self.visit(node) + + def visit_Compare(self, node): + node = self.generic_visit(node) + node = asttyped.CompareT(type=types.TVar(), + left=node.left, ops=node.ops, comparators=node.comparators, + loc=node.loc) + return self.visit(node) + + def visit_IfExp(self, node): + node = self.generic_visit(node) + node = asttyped.IfExpT(type=types.TVar(), + test=node.test, body=node.body, orelse=node.orelse, + if_loc=node.if_loc, else_loc=node.else_loc, loc=node.loc) + return self.visit(node) + + def visit_ListComp(self, node): + extractor = LocalExtractor(env_stack=self.env_stack, engine=self.engine) + extractor.visit(node) + + node = asttyped.ListCompT( + typing_env=extractor.typing_env, globals_in_scope=extractor.global_, + type=types.TVar(), + elt=node.elt, generators=node.generators, + begin_loc=node.begin_loc, end_loc=node.end_loc, loc=node.loc) + + try: + self.env_stack.append(node.typing_env) + return self.generic_visit(node) + finally: + self.env_stack.pop() + + def visit_Call(self, node): + node = self.generic_visit(node) + node = asttyped.CallT(type=types.TVar(), + func=node.func, args=node.args, keywords=node.keywords, + starargs=node.starargs, kwargs=node.kwargs, + star_loc=node.star_loc, dstar_loc=node.dstar_loc, + begin_loc=node.begin_loc, end_loc=node.end_loc, loc=node.loc) + return node + + def visit_Lambda(self, node): + extractor = LocalExtractor(env_stack=self.env_stack, engine=self.engine) + extractor.visit(node) + + node = asttyped.LambdaT( + typing_env=extractor.typing_env, globals_in_scope=extractor.global_, + type=types.TVar(), + args=node.args, body=node.body, + lambda_loc=node.lambda_loc, colon_loc=node.colon_loc, loc=node.loc) + + try: + self.env_stack.append(node.typing_env) + return self.generic_visit(node) + finally: + self.env_stack.pop() + + def visit_ExceptHandler(self, node): + node = self.generic_visit(node) + node = asttyped.ExceptHandlerT( + name_type=self._find_name(node.name, node.name_loc), + filter=node.type, name=node.name, body=node.body, + except_loc=node.except_loc, as_loc=node.as_loc, name_loc=node.name_loc, + colon_loc=node.colon_loc, loc=node.loc) + return node + + def visit_Raise(self, node): + node = self.generic_visit(node) + if node.cause: + diag = diagnostic.Diagnostic("error", + "'raise from' syntax is not supported", {}, + node.from_loc) + self.engine.process(diag) + return node + + # Unsupported visitors + # + def visit_unsupported(self, node): + diag = diagnostic.Diagnostic("fatal", + "this syntax is not supported", {}, + node.loc) + self.engine.process(diag) + + # expr + visit_Dict = visit_unsupported + visit_DictComp = visit_unsupported + visit_Ellipsis = visit_unsupported + visit_GeneratorExp = visit_unsupported + visit_Set = visit_unsupported + visit_SetComp = visit_unsupported + visit_Str = visit_unsupported + visit_Starred = visit_unsupported + visit_Yield = visit_unsupported + visit_YieldFrom = visit_unsupported + + # stmt + visit_Assert = visit_unsupported + visit_ClassDef = visit_unsupported + visit_Delete = visit_unsupported + visit_Import = visit_unsupported + visit_ImportFrom = visit_unsupported diff --git a/artiq/compiler/typing.py b/artiq/compiler/transforms/inferencer.py similarity index 68% rename from artiq/compiler/typing.py rename to artiq/compiler/transforms/inferencer.py index c7ae71327..f78acea22 100644 --- a/artiq/compiler/typing.py +++ b/artiq/compiler/transforms/inferencer.py @@ -1,378 +1,6 @@ -from pythonparser import source, ast, algorithm, diagnostic, parse_buffer from collections import OrderedDict -from . import asttyped, types, builtins - -# This visitor will be called for every node with a scope, -# i.e.: class, function, comprehension, lambda -class LocalExtractor(algorithm.Visitor): - def __init__(self, env_stack, engine): - super().__init__() - self.env_stack = env_stack - self.engine = engine - - self.in_root = False - self.in_assign = False - self.typing_env = {} - - # which names are global have to be recorded in the current scope - self.global_ = set() - - # which names are nonlocal only affects whether the current scope - # gets a new binding or not, so we throw this away - self.nonlocal_ = set() - - # parameters can't be declared as global or nonlocal - self.params = set() - - if len(self.env_stack) == 1: - self.env_stack.append(self.typing_env) - - def visit_in_assign(self, node): - try: - self.in_assign = True - return self.visit(node) - finally: - self.in_assign = False - - def visit_Assign(self, node): - self.visit(node.value) - for target in node.targets: - self.visit_in_assign(target) - - def visit_For(self, node): - self.visit(node.iter) - self.visit_in_assign(node.target) - self.visit(node.body) - self.visit(node.orelse) - - def visit_withitem(self, node): - self.visit(node.context_expr) - if node.optional_vars is not None: - self.visit_in_assign(node.optional_vars) - - def visit_comprehension(self, node): - self.visit(node.iter) - self.visit_in_assign(node.target) - for if_ in node.ifs: - self.visit(node.ifs) - - def visit_root(self, node): - if self.in_root: - return - self.in_root = True - self.generic_visit(node) - - visit_Module = visit_root # don't look at inner scopes - visit_ClassDef = visit_root - visit_Lambda = visit_root - visit_DictComp = visit_root - visit_ListComp = visit_root - visit_SetComp = visit_root - visit_GeneratorExp = visit_root - - def visit_FunctionDef(self, node): - if self.in_root: - self._assignable(node.name) - self.visit_root(node) - - def _assignable(self, name): - if name not in self.typing_env and name not in self.nonlocal_: - self.typing_env[name] = types.TVar() - - def visit_arg(self, node): - if node.arg in self.params: - diag = diagnostic.Diagnostic("error", - "duplicate parameter '{name}'", {"name": node.arg}, - node.loc) - self.engine.process(diag) - return - self._assignable(node.arg) - self.params.add(node.arg) - - def visit_Name(self, node): - if self.in_assign: - # code like: - # x = 1 - # def f(): - # x = 1 - # creates a new binding for x in f's scope - self._assignable(node.id) - - def _check_not_in(self, name, names, curkind, newkind, loc): - if name in names: - diag = diagnostic.Diagnostic("error", - "name '{name}' cannot be {curkind} and {newkind} simultaneously", - {"name": name, "curkind": curkind, "newkind": newkind}, loc) - self.engine.process(diag) - return True - return False - - def visit_Global(self, node): - for name, loc in zip(node.names, node.name_locs): - if self._check_not_in(name, self.nonlocal_, "nonlocal", "global", loc) or \ - self._check_not_in(name, self.params, "a parameter", "global", loc): - continue - - self.global_.add(name) - self._assignable(name) - self.env_stack[1][name] = self.typing_env[name] - - def visit_Nonlocal(self, node): - for name, loc in zip(node.names, node.name_locs): - if self._check_not_in(name, self.global_, "global", "nonlocal", loc) or \ - self._check_not_in(name, self.params, "a parameter", "nonlocal", loc): - continue - - # nonlocal does not search prelude and global scopes - found = False - for outer_env in reversed(self.env_stack[2:]): - if name in outer_env: - found = True - break - if not found: - diag = diagnostic.Diagnostic("error", - "cannot declare name '{name}' as nonlocal: it is not bound in any outer scope", - {"name": name}, - loc, [node.keyword_loc]) - self.engine.process(diag) - continue - - self.nonlocal_.add(name) - - def visit_ExceptHandler(self, node): - self.visit(node.type) - self._assignable(node.name) - for stmt in node.body: - self.visit(stmt) - - -class ASTTypedRewriter(algorithm.Transformer): - """ - :class:`ASTTypedRewriter` converts an untyped AST to a typed AST - where all type fields of non-literals are filled with fresh type variables, - and type fields of literals are filled with corresponding types. - - :class:`ASTTypedRewriter` also discovers the scope of variable bindings - via :class:`LocalExtractor`. - """ - - def __init__(self, engine, globals={}): - self.engine = engine - self.env_stack = [globals] - - def _find_name(self, name, loc): - for typing_env in reversed(self.env_stack): - if name in typing_env: - return typing_env[name] - diag = diagnostic.Diagnostic("fatal", - "name '{name}' is not bound to anything", {"name":name}, loc) - self.engine.process(diag) - - # Visitors that replace node with a typed node - # - def visit_Module(self, node): - extractor = LocalExtractor(env_stack=self.env_stack, engine=self.engine) - extractor.visit(node) - - node = asttyped.ModuleT( - typing_env=extractor.typing_env, globals_in_scope=extractor.global_, - body=node.body, loc=node.loc) - return self.generic_visit(node) - - def visit_FunctionDef(self, node): - extractor = LocalExtractor(env_stack=self.env_stack, engine=self.engine) - extractor.visit(node) - - node = asttyped.FunctionDefT( - typing_env=extractor.typing_env, globals_in_scope=extractor.global_, - signature_type=self._find_name(node.name, node.name_loc), return_type=types.TVar(), - name=node.name, args=node.args, returns=node.returns, - body=node.body, decorator_list=node.decorator_list, - keyword_loc=node.keyword_loc, name_loc=node.name_loc, - arrow_loc=node.arrow_loc, colon_loc=node.colon_loc, at_locs=node.at_locs, - loc=node.loc) - - try: - self.env_stack.append(node.typing_env) - return self.generic_visit(node) - finally: - self.env_stack.pop() - - def visit_arg(self, node): - return asttyped.argT(type=self._find_name(node.arg, node.loc), - arg=node.arg, annotation=self.visit(node.annotation), - arg_loc=node.arg_loc, colon_loc=node.colon_loc, loc=node.loc) - - def visit_Num(self, node): - if isinstance(node.n, int): - typ = builtins.TInt() - elif isinstance(node.n, float): - typ = builtins.TFloat() - else: - diag = diagnostic.Diagnostic("fatal", - "numeric type {type} is not supported", {"type": node.n.__class__.__name__}, - node.loc) - self.engine.process(diag) - return asttyped.NumT(type=typ, - n=node.n, loc=node.loc) - - def visit_Name(self, node): - return asttyped.NameT(type=self._find_name(node.id, node.loc), - id=node.id, ctx=node.ctx, loc=node.loc) - - def visit_NameConstant(self, node): - if node.value is True or node.value is False: - typ = builtins.TBool() - elif node.value is None: - typ = builtins.TNone() - return asttyped.NameConstantT(type=typ, value=node.value, loc=node.loc) - - def visit_Tuple(self, node): - node = self.generic_visit(node) - return asttyped.TupleT(type=types.TTuple([x.type for x in node.elts]), - elts=node.elts, ctx=node.ctx, loc=node.loc) - - def visit_List(self, node): - node = self.generic_visit(node) - node = asttyped.ListT(type=builtins.TList(), - elts=node.elts, ctx=node.ctx, loc=node.loc) - return self.visit(node) - - def visit_Attribute(self, node): - node = self.generic_visit(node) - node = asttyped.AttributeT(type=types.TVar(), - value=node.value, attr=node.attr, ctx=node.ctx, - dot_loc=node.dot_loc, attr_loc=node.attr_loc, loc=node.loc) - return self.visit(node) - - def visit_Subscript(self, node): - node = self.generic_visit(node) - node = asttyped.SubscriptT(type=types.TVar(), - value=node.value, slice=node.slice, ctx=node.ctx, - loc=node.loc) - return self.visit(node) - - def visit_BoolOp(self, node): - node = self.generic_visit(node) - node = asttyped.BoolOpT(type=types.TVar(), - op=node.op, values=node.values, - op_locs=node.op_locs, loc=node.loc) - return self.visit(node) - - def visit_UnaryOp(self, node): - node = self.generic_visit(node) - node = asttyped.UnaryOpT(type=types.TVar(), - op=node.op, operand=node.operand, - loc=node.loc) - return self.visit(node) - - def visit_BinOp(self, node): - node = self.generic_visit(node) - node = asttyped.BinOpT(type=types.TVar(), - left=node.left, op=node.op, right=node.right, - loc=node.loc) - return self.visit(node) - - def visit_Compare(self, node): - node = self.generic_visit(node) - node = asttyped.CompareT(type=types.TVar(), - left=node.left, ops=node.ops, comparators=node.comparators, - loc=node.loc) - return self.visit(node) - - def visit_IfExp(self, node): - node = self.generic_visit(node) - node = asttyped.IfExpT(type=types.TVar(), - test=node.test, body=node.body, orelse=node.orelse, - if_loc=node.if_loc, else_loc=node.else_loc, loc=node.loc) - return self.visit(node) - - def visit_ListComp(self, node): - extractor = LocalExtractor(env_stack=self.env_stack, engine=self.engine) - extractor.visit(node) - - node = asttyped.ListCompT( - typing_env=extractor.typing_env, globals_in_scope=extractor.global_, - type=types.TVar(), - elt=node.elt, generators=node.generators, - begin_loc=node.begin_loc, end_loc=node.end_loc, loc=node.loc) - - try: - self.env_stack.append(node.typing_env) - return self.generic_visit(node) - finally: - self.env_stack.pop() - - def visit_Call(self, node): - node = self.generic_visit(node) - node = asttyped.CallT(type=types.TVar(), - func=node.func, args=node.args, keywords=node.keywords, - starargs=node.starargs, kwargs=node.kwargs, - star_loc=node.star_loc, dstar_loc=node.dstar_loc, - begin_loc=node.begin_loc, end_loc=node.end_loc, loc=node.loc) - return node - - def visit_Lambda(self, node): - extractor = LocalExtractor(env_stack=self.env_stack, engine=self.engine) - extractor.visit(node) - - node = asttyped.LambdaT( - typing_env=extractor.typing_env, globals_in_scope=extractor.global_, - type=types.TVar(), - args=node.args, body=node.body, - lambda_loc=node.lambda_loc, colon_loc=node.colon_loc, loc=node.loc) - - try: - self.env_stack.append(node.typing_env) - return self.generic_visit(node) - finally: - self.env_stack.pop() - - def visit_ExceptHandler(self, node): - node = self.generic_visit(node) - node = asttyped.ExceptHandlerT( - name_type=self._find_name(node.name, node.name_loc), - filter=node.type, name=node.name, body=node.body, - except_loc=node.except_loc, as_loc=node.as_loc, name_loc=node.name_loc, - colon_loc=node.colon_loc, loc=node.loc) - return node - - def visit_Raise(self, node): - node = self.generic_visit(node) - if node.cause: - diag = diagnostic.Diagnostic("error", - "'raise from' syntax is not supported", {}, - node.from_loc) - self.engine.process(diag) - return node - - # Unsupported visitors - # - def visit_unsupported(self, node): - diag = diagnostic.Diagnostic("fatal", - "this syntax is not supported", {}, - node.loc) - self.engine.process(diag) - - # expr - visit_Dict = visit_unsupported - visit_DictComp = visit_unsupported - visit_Ellipsis = visit_unsupported - visit_GeneratorExp = visit_unsupported - visit_Set = visit_unsupported - visit_SetComp = visit_unsupported - visit_Str = visit_unsupported - visit_Starred = visit_unsupported - visit_Yield = visit_unsupported - visit_YieldFrom = visit_unsupported - - # stmt - visit_Assert = visit_unsupported - visit_ClassDef = visit_unsupported - visit_Delete = visit_unsupported - visit_Import = visit_unsupported - visit_ImportFrom = visit_unsupported - +from pythonparser import algorithm, diagnostic, ast +from .. import asttyped, types, builtins class Inferencer(algorithm.Visitor): """ @@ -444,7 +72,7 @@ class Inferencer(algorithm.Visitor): def visit_ListT(self, node): self.generic_visit(node) for elt in node.elts: - self._unify(node.type, builtins.TList(elt.type), + self._unify(node.type["elt"], elt.type, node.loc, elt.loc, self._makenotes_elts(node.elts, "a list element")) def visit_AttributeT(self, node): @@ -1173,77 +801,3 @@ class Inferencer(algorithm.Visitor): else: self._unify(self.function.return_type, node.value.type, self.function.name_loc, node.value.loc, makenotes) - -class Printer(algorithm.Visitor): - """ - :class:`Printer` prints ``:`` and the node type after every typed node, - and ``->`` and the node type before the colon in a function definition. - - In almost all cases (except function definition) this does not result - in valid Python syntax. - - :ivar rewriter: (:class:`pythonparser.source.Rewriter`) rewriter instance - """ - - def __init__(self, buf): - self.rewriter = source.Rewriter(buf) - self.type_printer = types.TypePrinter() - - def rewrite(self): - return self.rewriter.rewrite() - - def visit_FunctionDefT(self, node): - super().generic_visit(node) - - self.rewriter.insert_before(node.colon_loc, - "->{}".format(self.type_printer.name(node.return_type))) - - def visit_ExceptHandlerT(self, node): - super().generic_visit(node) - - if node.name_loc: - self.rewriter.insert_after(node.name_loc, - ":{}".format(self.type_printer.name(node.name_type))) - - def generic_visit(self, node): - super().generic_visit(node) - - if hasattr(node, "type"): - self.rewriter.insert_after(node.loc, - ":{}".format(self.type_printer.name(node.type))) - -def main(): - import sys, fileinput, os - from . import prelude - - if len(sys.argv) > 1 and sys.argv[1] == '+diag': - del sys.argv[1] - def process_diagnostic(diag): - print("\n".join(diag.render(only_line=True))) - if diag.level == 'fatal': - exit() - else: - def process_diagnostic(diag): - print("\n".join(diag.render())) - if diag.level in ('fatal', 'error'): - exit(1) - - engine = diagnostic.Engine() - engine.process = process_diagnostic - - buf = source.Buffer("".join(fileinput.input()).expandtabs(), - os.path.basename(fileinput.filename())) - parsed, comments = parse_buffer(buf, engine=engine) - typed = ASTTypedRewriter(globals=prelude.globals(), engine=engine).visit(parsed) - Inferencer(engine=engine).visit(typed) - - printer = Printer(buf) - printer.visit(typed) - for comment in comments: - if comment.text.find("CHECK") >= 0: - printer.rewriter.remove(comment.loc) - print(printer.rewrite().source) - - -if __name__ == "__main__": - main() diff --git a/lit-test/compiler/typing/builtin_calls.py b/lit-test/compiler/inferencer/builtin_calls.py similarity index 92% rename from lit-test/compiler/typing/builtin_calls.py rename to lit-test/compiler/inferencer/builtin_calls.py index 82c750bf1..50597151f 100644 --- a/lit-test/compiler/typing/builtin_calls.py +++ b/lit-test/compiler/inferencer/builtin_calls.py @@ -1,4 +1,4 @@ -# RUN: %python -m artiq.compiler.typing %s >%t +# RUN: %python -m artiq.compiler.testbench.inferencer %s >%t # RUN: OutputCheck %s --file-to-check=%t # CHECK-L: bool:():bool diff --git a/lit-test/compiler/typing/coerce.py b/lit-test/compiler/inferencer/coerce.py similarity index 95% rename from lit-test/compiler/typing/coerce.py rename to lit-test/compiler/inferencer/coerce.py index 5b161ab30..3b0190c49 100644 --- a/lit-test/compiler/typing/coerce.py +++ b/lit-test/compiler/inferencer/coerce.py @@ -1,4 +1,4 @@ -# RUN: %python -m artiq.compiler.typing %s >%t +# RUN: %python -m artiq.compiler.testbench.inferencer %s >%t # RUN: OutputCheck %s --file-to-check=%t 1 | 2 diff --git a/lit-test/compiler/typing/error_builtin_calls.py b/lit-test/compiler/inferencer/error_builtin_calls.py similarity index 86% rename from lit-test/compiler/typing/error_builtin_calls.py rename to lit-test/compiler/inferencer/error_builtin_calls.py index 858df6d4a..32ce8b294 100644 --- a/lit-test/compiler/typing/error_builtin_calls.py +++ b/lit-test/compiler/inferencer/error_builtin_calls.py @@ -1,4 +1,4 @@ -# RUN: %python -m artiq.compiler.typing +diag %s >%t +# RUN: %python -m artiq.compiler.testbench.inferencer +diag %s >%t # RUN: OutputCheck %s --file-to-check=%t a = 1 diff --git a/lit-test/compiler/typing/error_call.py b/lit-test/compiler/inferencer/error_call.py similarity index 86% rename from lit-test/compiler/typing/error_call.py rename to lit-test/compiler/inferencer/error_call.py index 2ff169faf..21ba5bdf6 100644 --- a/lit-test/compiler/typing/error_call.py +++ b/lit-test/compiler/inferencer/error_call.py @@ -1,4 +1,4 @@ -# RUN: %python -m artiq.compiler.typing +diag %s >%t +# RUN: %python -m artiq.compiler.testbench.inferencer +diag %s >%t # RUN: OutputCheck %s --file-to-check=%t # CHECK-L: ${LINE:+1}: error: cannot call this expression of type int diff --git a/lit-test/compiler/typing/error_coerce.py b/lit-test/compiler/inferencer/error_coerce.py similarity index 96% rename from lit-test/compiler/typing/error_coerce.py rename to lit-test/compiler/inferencer/error_coerce.py index feb7447b8..bf4a5dd36 100644 --- a/lit-test/compiler/typing/error_coerce.py +++ b/lit-test/compiler/inferencer/error_coerce.py @@ -1,4 +1,4 @@ -# RUN: %python -m artiq.compiler.typing +diag %s >%t +# RUN: %python -m artiq.compiler.testbench.inferencer +diag %s >%t # RUN: OutputCheck %s --file-to-check=%t # CHECK-L: ${LINE:+1}: error: expected '<<' operand to be of integer type, not float diff --git a/lit-test/compiler/typing/error_control_flow.py b/lit-test/compiler/inferencer/error_control_flow.py similarity index 87% rename from lit-test/compiler/typing/error_control_flow.py rename to lit-test/compiler/inferencer/error_control_flow.py index c5a913bdd..65e300511 100644 --- a/lit-test/compiler/typing/error_control_flow.py +++ b/lit-test/compiler/inferencer/error_control_flow.py @@ -1,4 +1,4 @@ -# RUN: %python -m artiq.compiler.typing +diag %s >%t +# RUN: %python -m artiq.compiler.testbench.inferencer +diag %s >%t # RUN: OutputCheck %s --file-to-check=%t # CHECK-L: ${LINE:+1}: error: return statement outside of a function diff --git a/lit-test/compiler/typing/error_exception.py b/lit-test/compiler/inferencer/error_exception.py similarity index 80% rename from lit-test/compiler/typing/error_exception.py rename to lit-test/compiler/inferencer/error_exception.py index 4172f7bde..4e4b340b5 100644 --- a/lit-test/compiler/typing/error_exception.py +++ b/lit-test/compiler/inferencer/error_exception.py @@ -1,4 +1,4 @@ -# RUN: %python -m artiq.compiler.typing +diag %s >%t +# RUN: %python -m artiq.compiler.testbench.inferencer +diag %s >%t # RUN: OutputCheck %s --file-to-check=%t try: diff --git a/lit-test/compiler/typing/error_iterable.py b/lit-test/compiler/inferencer/error_iterable.py similarity index 64% rename from lit-test/compiler/typing/error_iterable.py rename to lit-test/compiler/inferencer/error_iterable.py index d3334a616..68ed1b002 100644 --- a/lit-test/compiler/typing/error_iterable.py +++ b/lit-test/compiler/inferencer/error_iterable.py @@ -1,4 +1,4 @@ -# RUN: %python -m artiq.compiler.typing +diag %s >%t +# RUN: %python -m artiq.compiler.testbench.inferencer +diag %s >%t # RUN: OutputCheck %s --file-to-check=%t # CHECK-L: ${LINE:+1}: error: type int(width='a) is not iterable diff --git a/lit-test/compiler/typing/error_local_unbound.py b/lit-test/compiler/inferencer/error_local_unbound.py similarity index 61% rename from lit-test/compiler/typing/error_local_unbound.py rename to lit-test/compiler/inferencer/error_local_unbound.py index fdf1b3047..6b4350157 100644 --- a/lit-test/compiler/typing/error_local_unbound.py +++ b/lit-test/compiler/inferencer/error_local_unbound.py @@ -1,4 +1,4 @@ -# RUN: %python -m artiq.compiler.typing +diag %s >%t +# RUN: %python -m artiq.compiler.testbench.inferencer +diag %s >%t # RUN: OutputCheck %s --file-to-check=%t # CHECK-L: ${LINE:+1}: fatal: name 'x' is not bound to anything diff --git a/lit-test/compiler/typing/error_locals.py b/lit-test/compiler/inferencer/error_locals.py similarity index 93% rename from lit-test/compiler/typing/error_locals.py rename to lit-test/compiler/inferencer/error_locals.py index 048295d2c..3a5185ca8 100644 --- a/lit-test/compiler/typing/error_locals.py +++ b/lit-test/compiler/inferencer/error_locals.py @@ -1,4 +1,4 @@ -# RUN: %python -m artiq.compiler.typing +diag %s >%t +# RUN: %python -m artiq.compiler.testbench.inferencer +diag %s >%t # RUN: OutputCheck %s --file-to-check=%t x = 1 diff --git a/lit-test/compiler/typing/error_return.py b/lit-test/compiler/inferencer/error_return.py similarity index 88% rename from lit-test/compiler/typing/error_return.py rename to lit-test/compiler/inferencer/error_return.py index fff45ed0b..ba4ddc2b3 100644 --- a/lit-test/compiler/typing/error_return.py +++ b/lit-test/compiler/inferencer/error_return.py @@ -1,4 +1,4 @@ -# RUN: %python -m artiq.compiler.typing +diag %s >%t +# RUN: %python -m artiq.compiler.testbench.inferencer +diag %s >%t # RUN: OutputCheck %s --file-to-check=%t # CHECK-L: ${LINE:+2}: error: cannot unify int(width='a) with NoneType diff --git a/lit-test/compiler/typing/error_unify.py b/lit-test/compiler/inferencer/error_unify.py similarity index 92% rename from lit-test/compiler/typing/error_unify.py rename to lit-test/compiler/inferencer/error_unify.py index 076be96f4..e81537d29 100644 --- a/lit-test/compiler/typing/error_unify.py +++ b/lit-test/compiler/inferencer/error_unify.py @@ -1,4 +1,4 @@ -# RUN: %python -m artiq.compiler.typing +diag %s >%t +# RUN: %python -m artiq.compiler.testbench.inferencer +diag %s >%t # RUN: OutputCheck %s --file-to-check=%t a = 1 diff --git a/lit-test/compiler/typing/exception.py b/lit-test/compiler/inferencer/exception.py similarity index 76% rename from lit-test/compiler/typing/exception.py rename to lit-test/compiler/inferencer/exception.py index 32046157f..bfdbbfb7c 100644 --- a/lit-test/compiler/typing/exception.py +++ b/lit-test/compiler/inferencer/exception.py @@ -1,4 +1,4 @@ -# RUN: %python -m artiq.compiler.typing %s >%t +# RUN: %python -m artiq.compiler.testbench.inferencer %s >%t # RUN: OutputCheck %s --file-to-check=%t # CHECK-L: Exception: diff --git a/lit-test/compiler/typing/gcd.py b/lit-test/compiler/inferencer/gcd.py similarity index 80% rename from lit-test/compiler/typing/gcd.py rename to lit-test/compiler/inferencer/gcd.py index c79052d89..e2d4b4779 100644 --- a/lit-test/compiler/typing/gcd.py +++ b/lit-test/compiler/inferencer/gcd.py @@ -1,4 +1,4 @@ -# RUN: %python -m artiq.compiler.typing %s >%t +# RUN: %python -m artiq.compiler.testbench.inferencer %s >%t def _gcd(a, b): if a < 0: diff --git a/lit-test/compiler/typing/prelude.py b/lit-test/compiler/inferencer/prelude.py similarity index 71% rename from lit-test/compiler/typing/prelude.py rename to lit-test/compiler/inferencer/prelude.py index 1b70be299..643895e4a 100644 --- a/lit-test/compiler/typing/prelude.py +++ b/lit-test/compiler/inferencer/prelude.py @@ -1,4 +1,4 @@ -# RUN: %python -m artiq.compiler.typing %s >%t +# RUN: %python -m artiq.compiler.testbench.inferencer %s >%t # RUN: OutputCheck %s --file-to-check=%t # CHECK-L: x: diff --git a/lit-test/compiler/typing/scoping.py b/lit-test/compiler/inferencer/scoping.py similarity index 63% rename from lit-test/compiler/typing/scoping.py rename to lit-test/compiler/inferencer/scoping.py index 1d1da612c..595daecf6 100644 --- a/lit-test/compiler/typing/scoping.py +++ b/lit-test/compiler/inferencer/scoping.py @@ -1,4 +1,4 @@ -# RUN: %python -m artiq.compiler.typing %s >%t +# RUN: %python -m artiq.compiler.testbench.inferencer %s >%t # RUN: OutputCheck %s --file-to-check=%t def f(): diff --git a/lit-test/compiler/typing/unify.py b/lit-test/compiler/inferencer/unify.py similarity index 94% rename from lit-test/compiler/typing/unify.py rename to lit-test/compiler/inferencer/unify.py index 9621c5b59..04c8aecbd 100644 --- a/lit-test/compiler/typing/unify.py +++ b/lit-test/compiler/inferencer/unify.py @@ -1,4 +1,4 @@ -# RUN: %python -m artiq.compiler.typing %s >%t +# RUN: %python -m artiq.compiler.testbench.inferencer %s >%t # RUN: OutputCheck %s --file-to-check=%t a = 1