Implement shared object linking.

artiq/compiler/ir.py

@@ -387,12 +387,17 @@ class Argument(NamedValue):
 class Function:
     """
     A function containing SSA IR.
+
+    :ivar is_internal:
+        (bool) if True, the function should not be accessible from outside
+        the module it is contained in
     """
 
     def __init__(self, typ, name, arguments):
         self.type, self.name = typ, name
         self.names, self.arguments, self.basic_blocks = set(), [], []
         self.set_arguments(arguments)
+        self.is_internal = False
 
     def _remove_name(self, name):
         self.names.remove(name)

artiq/compiler/module.py

@@ -21,7 +21,6 @@ class Module:
         artiq_ir_generator = transforms.ARTIQIRGenerator(engine=engine, module_name=self.name)
         dead_code_eliminator = transforms.DeadCodeEliminator(engine=engine)
         local_access_validator = validators.LocalAccessValidator(engine=engine)
-        llvm_ir_generator = transforms.LLVMIRGenerator(engine=engine, module_name=self.name)
 
         self.parsetree, self.comments = parse_buffer(source_buffer, engine=engine)
         self.typedtree = asttyped_rewriter.visit(self.parsetree)
@@ -34,7 +33,15 @@ class Module:
         self.artiq_ir = artiq_ir_generator.visit(self.typedtree)
         dead_code_eliminator.process(self.artiq_ir)
         # local_access_validator.process(self.artiq_ir)
-        self.llvm_ir = llvm_ir_generator.process(self.artiq_ir)
+
+    def build_llvm_ir(self, target):
+        """Compile the module to LLVM IR for the specified target."""
+        llvm_ir_generator = transforms.LLVMIRGenerator(module_name=self.name, target=target)
+        return llvm_ir_generator.process(self.artiq_ir)
+
+    def entry_point(self):
+        """Return the name of the function that is the entry point of this module."""
+        return self.name + ".__modinit__"
 
     @classmethod
     def from_string(cls, source_string, name="input.py", first_line=1, engine=None):

artiq/compiler/targets/__init__.py

@@ -0,0 +1,87 @@
+import tempfile, subprocess
+from llvmlite_artiq import ir as ll, binding as llvm
+
+llvm.initialize()
+llvm.initialize_all_targets()
+llvm.initialize_all_asmprinters()
+
+class Target:
+    """
+    A description of the target environment where the binaries
+    generaed by the ARTIQ compiler will be deployed.
+
+    :var triple: (string)
+        LLVM target triple, e.g. ``"or1k"``
+    :var features: (list of string)
+        LLVM target CPU features, e.g. ``["mul", "div", "ffl1"]``
+    :var print_function: (string)
+        Name of a formatted print functions (with the signature of ``printf``)
+        provided by the target, e.g. ``"printf"``.
+    """
+    triple = "unknown"
+    features = []
+    print_function = "printf"
+
+    def __init__(self):
+        self.llcontext = ll.Context()
+
+    def link(self, objects, init_fn):
+        """Link the relocatable objects into a shared library for this target."""
+        files = []
+
+        def make_tempfile(data=b""):
+            f = tempfile.NamedTemporaryFile()
+            files.append(f)
+            f.write(data)
+            f.flush()
+            return f
+
+        output_file = make_tempfile()
+        cmdline = [self.triple + "-ld", "-shared", "--eh-frame-hdr", "-init", init_fn] + \
+                  [make_tempfile(obj).name for obj in objects] + \
+                  ["-o", output_file.name]
+        linker = subprocess.Popen(cmdline, stderr=subprocess.PIPE)
+        stdout, stderr = linker.communicate()
+        if linker.returncode != 0:
+            raise Exception("Linker invocation failed: " + stderr.decode('utf-8'))
+
+        output = output_file.read()
+
+        for f in files:
+            f.close()
+
+        return output
+
+    def compile(self, module):
+        """Compile the module to a relocatable object for this target."""
+        llmod = module.build_llvm_ir(self)
+        llparsedmod = llvm.parse_assembly(str(llmod))
+        llparsedmod.verify()
+
+        llpassmgrbuilder = llvm.create_pass_manager_builder()
+        llpassmgrbuilder.opt_level  = 2 # -O2
+        llpassmgrbuilder.size_level = 1 # -Os
+
+        llpassmgr = llvm.create_module_pass_manager()
+        llpassmgrbuilder.populate(llpassmgr)
+        llpassmgr.run(llparsedmod)
+
+        lltarget = llvm.Target.from_triple(self.triple)
+        llmachine = lltarget.create_target_machine(
+                        features=",".join(self.features),
+                        reloc="pic", codemodel="default")
+        return llmachine.emit_object(llparsedmod)
+
+    def compile_and_link(self, modules):
+        return self.link([self.compile(module) for module in modules],
+                         init_fn=modules[0].entry_point())
+
+class NativeTarget(Target):
+    def __init__(self):
+        super().__init__()
+        self.triple = llvm.get_default_triple()
+
+class OR1KTarget(Target):
+    triple = "or1k-linux"
+    attributes = ["mul", "div", "ffl1", "cmov", "addc"]
+    print_function = "log"

artiq/compiler/testbench/jit.py

@@ -2,10 +2,7 @@ import os, sys, fileinput, ctypes
 from pythonparser import diagnostic
 from llvmlite_artiq import binding as llvm
 from .. import Module
-
+from ..targets import NativeTarget
-llvm.initialize()
-llvm.initialize_native_target()
-llvm.initialize_native_asmprinter()
 
 def main():
     libartiq_personality = os.getenv('LIBARTIQ_PERSONALITY')
@@ -22,14 +19,15 @@ def main():
 
     source = "".join(fileinput.input())
     source = source.replace("#ARTIQ#", "")
-    llmod = Module.from_string(source.expandtabs(), engine=engine).llvm_ir
+    mod = Module.from_string(source.expandtabs(), engine=engine)
 
-    lltarget = llvm.Target.from_default_triple()
+    target = NativeTarget()
-    llmachine = lltarget.create_target_machine()
+    llmod = mod.build_llvm_ir(target)
     llparsedmod = llvm.parse_assembly(str(llmod))
     llparsedmod.verify()
+
+    llmachine = llvm.Target.from_triple(target.triple).create_target_machine()
     lljit = llvm.create_mcjit_compiler(llparsedmod, llmachine)
-    lljit.finalize_object()
     llmain = lljit.get_pointer_to_global(llparsedmod.get_function(llmod.name + ".__modinit__"))
     ctypes.CFUNCTYPE(None)(llmain)()
 

artiq/compiler/testbench/llvmgen.py

@@ -2,6 +2,7 @@ import sys, fileinput
 from pythonparser import diagnostic
 from llvmlite_artiq import ir as ll
 from .. import Module
+from ..targets import NativeTarget
 
 def main():
     def process_diagnostic(diag):
@@ -12,7 +13,10 @@ def main():
     engine = diagnostic.Engine()
     engine.process = process_diagnostic
 
-    llmod = Module.from_string("".join(fileinput.input()).expandtabs(), engine=engine).llvm_ir
+    mod = Module.from_string("".join(fileinput.input()).expandtabs(), engine=engine)
+
+    target = NativeTarget()
+    llmod = mod.build_llvm_ir(target=target)
 
     # Add main so that the result can be executed with lli
     llmain = ll.Function(llmod, ll.FunctionType(ll.VoidType(), []), "main")

artiq/compiler/testbench/shlib.py

@@ -0,0 +1,30 @@
+import sys, os
+from pythonparser import diagnostic
+from .. import Module
+from ..targets import OR1KTarget
+
+def main():
+    if not len(sys.argv) > 1:
+        print("Expected at least one module filename", file=sys.stderr)
+        exit(1)
+
+    def process_diagnostic(diag):
+        print("\n".join(diag.render()), file=sys.stderr)
+        if diag.level in ("fatal", "error"):
+            exit(1)
+
+    engine = diagnostic.Engine()
+    engine.process = process_diagnostic
+
+    modules = []
+    for filename in sys.argv[1:]:
+        modules.append(Module.from_filename(filename, engine=engine))
+
+    llobj = OR1KTarget().compile_and_link(modules)
+
+    basename, ext = os.path.splitext(sys.argv[-1])
+    with open(basename + ".so", "wb") as f:
+        f.write(llobj)
+
+if __name__ == "__main__":
+    main()

artiq/compiler/transforms/artiq_ir_generator.py

@@ -155,7 +155,7 @@ class ARTIQIRGenerator(algorithm.Visitor):
 
     # Statement visitors
 
-    def visit_function(self, node, is_lambda):
+    def visit_function(self, node, is_lambda, is_internal):
         if is_lambda:
             name = "lambda.{}.{}".format(node.loc.line(), node.loc.column())
             typ = node.type.find()
@@ -185,6 +185,7 @@ class ARTIQIRGenerator(algorithm.Visitor):
                 optargs.append(ir.Argument(ir.TOption(typ.optargs[arg_name]), "arg." + arg_name))
 
             func = ir.Function(typ, ".".join(self.name), [env_arg] + args + optargs)
+            func.is_internal = is_internal
             self.functions.append(func)
             old_func, self.current_function = self.current_function, func
 
@@ -237,7 +238,7 @@ class ARTIQIRGenerator(algorithm.Visitor):
         return self.append(ir.Closure(func, self.current_env))
 
     def visit_FunctionDefT(self, node):
-        func = self.visit_function(node, is_lambda=False)
+        func = self.visit_function(node, is_lambda=False, is_internal=len(name) > 2)
         self._set_local(node.name, func)
 
     def visit_Return(self, node):
@@ -614,7 +615,7 @@ class ARTIQIRGenerator(algorithm.Visitor):
     # the IR.
 
     def visit_LambdaT(self, node):
-        return self.visit_function(node, is_lambda=True)
+        return self.visit_function(node, is_lambda=True, is_internal=True)
 
     def visit_IfExpT(self, node):
         cond = self.visit(node.test)

artiq/compiler/transforms/llvm_ir_generator.py

@@ -8,10 +8,11 @@ from llvmlite_artiq import ir as ll
 from .. import types, builtins, ir
 
 class LLVMIRGenerator:
-    def __init__(self, engine, module_name, context=ll.Context()):
+    def __init__(self, module_name, target):
-        self.engine = engine
+        self.target = target
-        self.llcontext = context
+        self.llcontext = target.llcontext
         self.llmodule = ll.Module(context=self.llcontext, name=module_name)
+        self.llmodule.triple = target.triple
         self.llfunction = None
         self.llmap = {}
         self.fixups = []
@@ -135,7 +136,7 @@ class LLVMIRGenerator:
             llty = ll.FunctionType(ll.DoubleType(), [ll.DoubleType(), ll.IntType(32)])
         elif name == "llvm.copysign.f64":
             llty = ll.FunctionType(ll.DoubleType(), [ll.DoubleType(), ll.DoubleType()])
-        elif name == "printf":
+        elif name == self.target.print_function:
             llty = ll.FunctionType(ll.VoidType(), [ll.IntType(8).as_pointer()], var_arg=True)
         elif name == "__artiq_personality":
             llty = ll.FunctionType(ll.IntType(32), [], var_arg=True)
@@ -161,9 +162,6 @@ class LLVMIRGenerator:
             if llfun is None:
                 llfun = ll.Function(self.llmodule, self.llty_of_type(value.type, bare=True),
                                     value.name)
-                llfun.linkage = 'internal'
-                return llfun
-            else:
             return llfun
         else:
             assert False
@@ -184,6 +182,8 @@ class LLVMIRGenerator:
                 llfunty = ll.FunctionType(args=llargtys,
                                           return_type=self.llty_of_type(func.type.ret, for_return=True))
                 self.llfunction = ll.Function(self.llmodule, llfunty, func.name)
+                self.llfunction.attributes.add('uwtable')
+            if func.is_internal:
                 self.llfunction.linkage = 'internal'
 
             self.llmap = {}
@@ -527,7 +527,7 @@ class LLVMIRGenerator:
         elif insn.op == "printf":
             # We only get integers, floats, pointers and strings here.
             llargs = map(self.map, insn.operands)
-            return self.llbuilder.call(self.llbuiltin("printf"), llargs,
+            return self.llbuilder.call(self.llbuiltin(self.target.print_function), llargs,
                                        name=insn.name)
         elif insn.op == "exncast":
             # This is an identity cast at LLVM IR level.