use DSO instead of even messier LLVM JIT

.cargo/config.toml

@@ -0,0 +1,5 @@
+[target.x86_64-unknown-openbsd]
+rustflags = [ "-C", "link-args=-Wl,--export-dynamic-symbol=__nac3_personality", "-C", "link-args=-Wl,-z,nobtcfi" ]
+
+[target.x86_64-unknown-linux-gnu]
+rustflags = [ "-C", "link-args=-Wl,--export-dynamic-symbol=__nac3_personality" ]

Cargo.lock

@@ -245,9 +245,9 @@ dependencies = [
 
 [[package]]
 name = "async-executor"
-version = "1.13.0"
+version = "1.13.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d7ebdfa2ebdab6b1760375fa7d6f382b9f486eac35fc994625a00e89280bdbb7"
+checksum = "30ca9a001c1e8ba5149f91a74362376cc6bc5b919d92d988668657bd570bdcec"
 dependencies = [
  "async-task",
  "concurrent-queue",
@@ -671,9 +671,9 @@ dependencies = [
 
 [[package]]
 name = "cc"
-version = "1.1.16"
+version = "1.1.18"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e9d013ecb737093c0e86b151a7b837993cf9ec6c502946cfb44bedc392421e0b"
+checksum = "b62ac837cdb5cb22e10a256099b4fc502b1dfe560cb282963a974d7abd80e476"
 dependencies = [
  "jobserver",
  "libc",
@@ -686,8 +686,10 @@ version = "0.1.0"
 dependencies = [
  "eframe",
  "egui_extras",
+ "libloading 0.8.5",
  "nac3core",
  "parking_lot",
+ "tempfile",
 ]
 
 [[package]]
@@ -828,9 +830,9 @@ dependencies = [
 
 [[package]]
 name = "cpufeatures"
-version = "0.2.13"
+version = "0.2.14"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "51e852e6dc9a5bed1fae92dd2375037bf2b768725bf3be87811edee3249d09ad"
+checksum = "608697df725056feaccfa42cffdaeeec3fccc4ffc38358ecd19b243e716a78e0"
 dependencies = [
  "libc",
 ]
@@ -2102,7 +2104,7 @@ dependencies = [
 [[package]]
 name = "nac3ast"
 version = "0.1.0"
-source = "git+https://git.m-labs.hk/M-Labs/nac3?rev=6098b1b853d759de6a2980220faa217c25a2fb2b#6098b1b853d759de6a2980220faa217c25a2fb2b"
+source = "git+https://git.m-labs.hk/M-Labs/nac3?rev=abbaa506adabcbe8de9fa88c66138c81907333a8#abbaa506adabcbe8de9fa88c66138c81907333a8"
 dependencies = [
  "fxhash",
  "lazy_static",
@@ -2113,7 +2115,7 @@ dependencies = [
 [[package]]
 name = "nac3core"
 version = "0.1.0"
-source = "git+https://git.m-labs.hk/M-Labs/nac3?rev=6098b1b853d759de6a2980220faa217c25a2fb2b#6098b1b853d759de6a2980220faa217c25a2fb2b"
+source = "git+https://git.m-labs.hk/M-Labs/nac3?rev=abbaa506adabcbe8de9fa88c66138c81907333a8#abbaa506adabcbe8de9fa88c66138c81907333a8"
 dependencies = [
  "crossbeam",
  "indexmap",
@@ -2130,7 +2132,7 @@ dependencies = [
 [[package]]
 name = "nac3parser"
 version = "0.1.2"
-source = "git+https://git.m-labs.hk/M-Labs/nac3?rev=6098b1b853d759de6a2980220faa217c25a2fb2b#6098b1b853d759de6a2980220faa217c25a2fb2b"
+source = "git+https://git.m-labs.hk/M-Labs/nac3?rev=abbaa506adabcbe8de9fa88c66138c81907333a8#abbaa506adabcbe8de9fa88c66138c81907333a8"
 dependencies = [
  "ahash",
  "lalrpop",
@@ -2441,9 +2443,9 @@ dependencies = [
 
 [[package]]
 name = "parking"
-version = "2.2.0"
+version = "2.2.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "bb813b8af86854136c6922af0598d719255ecb2179515e6e7730d468f05c9cae"
+checksum = "f38d5652c16fde515bb1ecef450ab0f6a219d619a7274976324d5e377f7dceba"
 
 [[package]]
 name = "parking_lot"
@@ -2953,18 +2955,18 @@ checksum = "61697e0a1c7e512e84a621326239844a24d8207b4669b41bc18b32ea5cbf988b"
 
 [[package]]
 name = "serde"
-version = "1.0.209"
+version = "1.0.210"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "99fce0ffe7310761ca6bf9faf5115afbc19688edd00171d81b1bb1b116c63e09"
+checksum = "c8e3592472072e6e22e0a54d5904d9febf8508f65fb8552499a1abc7d1078c3a"
 dependencies = [
  "serde_derive",
 ]
 
 [[package]]
 name = "serde_derive"
-version = "1.0.209"
+version = "1.0.210"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a5831b979fd7b5439637af1752d535ff49f4860c0f341d1baeb6faf0f4242170"
+checksum = "243902eda00fad750862fc144cea25caca5e20d615af0a81bee94ca738f1df1f"
 dependencies = [
  "proc-macro2",
  "quote",
@@ -4411,9 +4413,9 @@ checksum = "b9cc00251562a284751c9973bace760d86c0276c471b4be569fe6b068ee97a56"
 
 [[package]]
 name = "xml-rs"
-version = "0.8.21"
+version = "0.8.22"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "539a77ee7c0de333dcc6da69b177380a0b81e0dacfa4f7344c465a36871ee601"
+checksum = "af4e2e2f7cba5a093896c1e150fbfe177d1883e7448200efb81d40b9d339ef26"
 
 [[package]]
 name = "yaml-rust"

Cargo.toml

@@ -7,7 +7,9 @@ edition = "2021"
 eframe = "0.28"
 egui_extras = { version = "0.28", features = ["syntect"]}
 parking_lot = "0.12"
+tempfile = "3.12"
+libloading = "0.8"
 
 [dependencies.nac3core]
 git = "https://git.m-labs.hk/M-Labs/nac3"
-rev = "6098b1b853d759de6a2980220faa217c25a2fb2b"
+rev = "abbaa506adabcbe8de9fa88c66138c81907333a8"

flake.lock

@@ -1,42 +1,24 @@
 {
   "nodes": {
-    "nac3": {
-      "inputs": {
-        "nixpkgs": "nixpkgs"
-      },
-      "locked": {
-        "lastModified": 1725593528,
-        "narHash": "sha256-pUgwm4mgbXNh5QtkJ9+Fr55dKf3uRSpKeFwlvZz1OkQ=",
-        "ref": "refs/heads/master",
-        "rev": "6098b1b853d759de6a2980220faa217c25a2fb2b",
-        "revCount": 1329,
-        "type": "git",
-        "url": "https://git.m-labs.hk/m-labs/nac3.git"
-      },
-      "original": {
-        "type": "git",
-        "url": "https://git.m-labs.hk/m-labs/nac3.git"
-      }
-    },
     "nixpkgs": {
       "locked": {
-        "lastModified": 1725432240,
+        "lastModified": 1725693463,
-        "narHash": "sha256-+yj+xgsfZaErbfYM3T+QvEE2hU7UuE+Jf0fJCJ8uPS0=",
+        "narHash": "sha256-ZPzhebbWBOr0zRWW10FfqfbJlan3G96/h3uqhiFqmwg=",
-        "owner": "NixOS",
+        "owner": "nixos",
         "repo": "nixpkgs",
-        "rev": "ad416d066ca1222956472ab7d0555a6946746a80",
+        "rev": "68e7dce0a6532e876980764167ad158174402c6f",
         "type": "github"
       },
       "original": {
-        "owner": "NixOS",
+        "owner": "nixos",
-        "ref": "nixos-unstable",
+        "ref": "nixos-24.05",
         "repo": "nixpkgs",
         "type": "github"
       }
     },
     "root": {
       "inputs": {
-        "nac3": "nac3"
+        "nixpkgs": "nixpkgs"
       }
     }
   },

flake.nix

@@ -1,11 +1,17 @@
 {
   description = "Cells";
 
-  inputs.nac3 = { type = "git"; url = "https://git.m-labs.hk/m-labs/nac3.git"; };
+  inputs.nixpkgs.url = "github:nixos/nixpkgs?ref=nixos-24.05";
 
-  outputs = { self, nac3 }:
+  outputs = { self, nixpkgs }:
     let
-      pkgs = import nac3.inputs.nixpkgs { system = "x86_64-linux"; };
+      pkgs = import nixpkgs { system = "x86_64-linux"; };
+      llvm-tools-irrt = pkgs.runCommandNoCC "llvm-tools-irrt" {}
+        ''
+        mkdir -p $out/bin
+        ln -s ${pkgs.llvmPackages_14.clang-unwrapped}/bin/clang $out/bin/clang-irrt
+        ln -s ${pkgs.llvmPackages_14.llvm.out}/bin/llvm-as $out/bin/llvm-as-irrt
+        '';
       libraryPath = pkgs.lib.makeLibraryPath [ pkgs.wayland pkgs.libxkbcommon pkgs.libGL ];
     in {
       packages.x86_64-linux.default = pkgs.rustPlatform.buildRustPackage {
@@ -14,15 +20,19 @@
         cargoLock = {
           lockFile = ./Cargo.lock;
           outputHashes = {
-            "nac3ast-0.1.0" = "sha256-pUgwm4mgbXNh5QtkJ9+Fr55dKf3uRSpKeFwlvZz1OkQ=";
+            "nac3ast-0.1.0" = "sha256-HJDMxhjUE2w1LEBOBBrZP0JpZFXkuS2csUC9I1+Dy00=";
           };
         };
         nativeBuildInputs = [
-          nac3.packages.x86_64-linux.llvm-nac3
+          pkgs.llvmPackages_14.llvm
-          nac3.packages.x86_64-linux.llvm-tools-irrt
+          llvm-tools-irrt
           pkgs.makeWrapper
         ];
-        buildInputs = [ nac3.packages.x86_64-linux.llvm-nac3 ];
+        propagatedbuildInputs = [
+          pkgs.llvmPackages_14.llvm
+          pkgs.llvmPackages_14.lld
+        ];
+        doCheck = false;
         postFixup = "wrapProgram $out/bin/cells --set LD_LIBRARY_PATH ${libraryPath}";
       };
       devShells.x86_64-linux.default = pkgs.mkShell {
@@ -31,8 +41,9 @@
           pkgs.cargo
           pkgs.rustc
           pkgs.rustfmt
-          nac3.packages.x86_64-linux.llvm-nac3
+          pkgs.llvmPackages_14.llvm
-          nac3.packages.x86_64-linux.llvm-tools-irrt
+          pkgs.llvmPackages_14.lld
+          llvm-tools-irrt
         ];
         shellHook = "export LD_LIBRARY_PATH=${libraryPath}";
       };

src/main.rs

@@ -1,5 +1,7 @@
 use std::collections::HashMap;
 use std::num::NonZeroUsize;
+use std::path::Path;
+use std::process::Command;
 use std::sync::Arc;
 
 use parking_lot::Mutex;
@@ -16,25 +18,20 @@ use nac3core::typecheck::{type_inferencer, typedef};
 mod basic_symbol_resolver;
 use basic_symbol_resolver::{Resolver, ResolverInternal};
 
-type RunFn = unsafe extern "C" fn() -> i32;
+fn compile(code: &String, output_filename: &Path) -> Result<(), String> {
-
+    let mut target_machine_options = codegen::CodeGenTargetMachineOptions::from_host();
-fn compile<'a>(
+    target_machine_options.reloc_mode = inkwell::targets::RelocMode::PIC;
-    context: &'a inkwell::context::Context,
-    code: &String,
-) -> Result<inkwell::execution_engine::JitFunction<'a, RunFn>, String> {
     let llvm_options = codegen::CodeGenLLVMOptions {
         opt_level: inkwell::OptimizationLevel::Default,
-        target: codegen::CodeGenTargetMachineOptions::from_host(),
+        target: target_machine_options,
     };
-    let size_t = context
+    let context = inkwell::context::Context::create();
-        .ptr_sized_int_type(
+    let target_machine = llvm_options
-            &llvm_options
         .target
         .create_target_machine(llvm_options.opt_level)
-                .map(|tm| tm.get_target_data())
+        .expect("couldn't create target machine");
-                .unwrap(),
+    let size_t = context
-            None,
+        .ptr_sized_int_type(&target_machine.get_target_data(), None)
-        )
         .get_bit_width();
     let primitive: type_inferencer::PrimitiveStore =
         composer::TopLevelComposer::make_primitives(size_t).0;
@@ -180,37 +177,117 @@ fn compile<'a>(
     }
     main.link_in_module(irrt).unwrap();
 
-    let execution_engine = main
+    // Private all functions except "run"
-        .create_jit_execution_engine(llvm_options.opt_level)
+    let mut function_iter = main.get_first_function();
+    while let Some(func) = function_iter {
+        if func.count_basic_blocks() > 0 && func.get_name().to_str().unwrap() != "run" {
+            func.set_linkage(inkwell::module::Linkage::Private);
+        }
+        function_iter = func.get_next_function();
+    }
+
+    let pass_options = inkwell::passes::PassBuilderOptions::create();
+    pass_options.set_merge_functions(true);
+    main.run_passes("default<O2>", &target_machine, pass_options)
         .unwrap();
-    return Ok(unsafe { execution_engine.get_function("run").unwrap() });
+
+    target_machine
+        .write_to_file(&main, inkwell::targets::FileType::Object, output_filename)
+        .unwrap();
+    Ok(())
 }
 
-struct Cell<'a> {
+// The year is 2024, and compiler toolchains are still a trash fire.
+fn link_with_lld(elf_filename: &Path, obj_filename: &Path) -> Result<(), String> {
+    let linker_args = vec![
+        "-shared".to_string(),
+        "--eh-frame-hdr".to_string(),
+        "-x".to_string(),
+        "-o".to_string(),
+        elf_filename.to_str().unwrap().to_string(),
+        obj_filename.to_str().unwrap().to_string(),
+    ];
+
+    if let Ok(linker_status) = Command::new("ld.lld").args(linker_args).status() {
+        if !linker_status.success() {
+            return Err("failed to start linker".to_string());
+        }
+    } else {
+        return Err("linker returned non-zero status code".to_string());
+    }
+
+    Ok(())
+}
+
+type RunFn = unsafe extern "C" fn() -> i32;
+
+struct CellBin {
+    cell_id: usize,
+    // note destructor order
+    run_fn: Option<RunFn>,
+    library: Option<libloading::Library>,
+    directory: tempfile::TempDir,
+}
+
+impl CellBin {
+    fn new(cell_id: usize) -> Self {
+        Self {
+            cell_id,
+            run_fn: None,
+            library: None,
+            directory: tempfile::tempdir().unwrap(),
+        }
+    }
+
+    fn compile_and_load(&mut self, code: &String) -> Result<(), String> {
+        assert!(self.run_fn.is_none());
+        assert!(self.library.is_none());
+        let object = self.directory.path().join("module.o");
+        let library = self.directory.path().join("module.so");
+        compile(code, &object)?;
+        link_with_lld(&library, &object)?;
+        unsafe {
+            self.library = Some(libloading::Library::new(library).or_else(|e| Err(e.to_string()))?);
+            let raw_fun_ptr = self
+                .library
+                .as_ref()
+                .unwrap()
+                .get::<RunFn>(b"run")
+                .unwrap()
+                .try_as_raw_ptr()
+                .unwrap();
+            self.run_fn = Some(std::mem::transmute(raw_fun_ptr));
+        }
+        Ok(())
+    }
+}
+
+struct Cell {
     code: String,
-    context: inkwell::context::Context,
+    result: Result<CellBin, String>,
-    result: Result<inkwell::execution_engine::JitFunction<'a, RunFn>, String>,
 }
 
-impl<'a> Cell<'a> {
+impl Cell {
     fn new() -> Self {
         Self {
             code: "".to_string(),
-            context: inkwell::context::Context::create(),
             result: Err("".to_string()),
         }
     }
 
     fn update(&mut self) {
-        self.result = Err("borrow checker problems".to_string()) //compile(&self.context, &self.code);
+        let mut new_bin = CellBin::new(0usize);
+        self.result = new_bin
+            .compile_and_load(&self.code)
+            .and_then(|_| Ok(new_bin));
     }
 }
 
-struct Cells<'a> {
+struct Cells {
-    cells: Vec<Cell<'a>>,
+    cells: Vec<Cell>,
 }
 
-impl Cells<'_> {
+impl Cells {
     fn new() -> Self {
         Self {
             cells: vec![Cell::new()],
@@ -261,8 +338,8 @@ impl Cells<'_> {
                     );
                 });
                 match &cell.result {
-                    Ok(run_fn) => {
+                    Ok(bin) => {
-                        ui.label(format!("{}", unsafe { run_fn.call() }));
+                        ui.label(format!("{}", unsafe { bin.run_fn.unwrap()() }));
                     }
                     Err(msg) => {
                         if !msg.is_empty() {
@@ -276,7 +353,8 @@ impl Cells<'_> {
 }
 
 fn main() -> eframe::Result {
-    inkwell::targets::Target::initialize_all(&inkwell::targets::InitializationConfig::default());
+    inkwell::targets::Target::initialize_native(&inkwell::targets::InitializationConfig::default())
+        .unwrap();
 
     let options = eframe::NativeOptions {
         viewport: egui::ViewportBuilder::default().with_inner_size([1024.0, 768.0]),
@@ -297,3 +375,8 @@ fn main() -> eframe::Result {
         });
     })
 }
+
+#[no_mangle]
+pub extern "C" fn __nac3_personality(_state: u32, _exception_object: u32, _context: u32) -> u32 {
+    unimplemented!()
+}