From 9f316a3294344c8de822d45fa824980b6119c085 Mon Sep 17 00:00:00 2001 From: Sebastien Bourdeauducq Date: Fri, 26 Nov 2021 17:00:20 +0800 Subject: [PATCH 01/54] flake: revert nixpkgs to unbreak rust cross-compilation --- flake.lock | 8 ++++---- flake.nix | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/flake.lock b/flake.lock index b7922587..1930cc3d 100644 --- a/flake.lock +++ b/flake.lock @@ -2,17 +2,17 @@ "nodes": { "nixpkgs": { "locked": { - "lastModified": 1637636156, - "narHash": "sha256-E2ym4Vcpqu9JYoQDXJZR48gVD+LPPbaCoYveIk7Xu3Y=", + "lastModified": 1637328665, + "narHash": "sha256-z6ufVwquLM0IiNZxd5oT1M33Lv0aB3WICpk8ZKwpxjw=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "b026e1cf87a108dd06fe521f224fdc72fd0b013d", + "rev": "0f4b4b85d959200f52c16bbb74036994e7db5f74", "type": "github" }, "original": { "owner": "NixOS", - "ref": "release-21.11", "repo": "nixpkgs", + "rev": "0f4b4b85d959200f52c16bbb74036994e7db5f74", "type": "github" } }, diff --git a/flake.nix b/flake.nix index 0e89187f..e6367367 100644 --- a/flake.nix +++ b/flake.nix @@ -1,7 +1,7 @@ { description = "The third-generation ARTIQ compiler"; - inputs.nixpkgs.url = github:NixOS/nixpkgs/release-21.11; + inputs.nixpkgs.url = github:NixOS/nixpkgs/0f4b4b85d959200f52c16bbb74036994e7db5f74; outputs = { self, nixpkgs }: let From 5e1b0a10a0fdac966212bdea0b208cc7d8c72e66 Mon Sep 17 00:00:00 2001 From: Sebastien Bourdeauducq Date: Fri, 26 Nov 2021 17:01:44 +0800 Subject: [PATCH 02/54] flake: patch nixpkgs to fix mingw llvm_12 build --- flake.nix | 2 +- llvm-mingw-crosscompile.diff | 35 +++++++++++++++++++++++++++++++++++ 2 files changed, 36 insertions(+), 1 deletion(-) create mode 100644 llvm-mingw-crosscompile.diff diff --git a/flake.nix b/flake.nix index e6367367..c75bec59 100644 --- a/flake.nix +++ b/flake.nix @@ -10,7 +10,7 @@ nixpkgs-patched = pkgs-orig.applyPatches { name = "nixpkgs"; src = nixpkgs; - patches = [ ./llvm-future-riscv-abi.diff ./llvm-restrict-targets.diff ]; + patches = [ ./llvm-future-riscv-abi.diff ./llvm-restrict-targets.diff ./llvm-mingw-crosscompile.diff ]; }; pkgs = import nixpkgs-patched { system = "x86_64-linux"; }; in rec { diff --git a/llvm-mingw-crosscompile.diff b/llvm-mingw-crosscompile.diff new file mode 100644 index 00000000..09fc6599 --- /dev/null +++ b/llvm-mingw-crosscompile.diff @@ -0,0 +1,35 @@ +diff --git a/pkgs/development/compilers/llvm/12/llvm/default.nix b/pkgs/development/compilers/llvm/12/llvm/default.nix +index 30a1a7a16df..4f9435d1819 100644 +--- a/pkgs/development/compilers/llvm/12/llvm/default.nix ++++ b/pkgs/development/compilers/llvm/12/llvm/default.nix +@@ -15,10 +15,11 @@ + , buildLlvmTools + , debugVersion ? false + , enableManpages ? false +-, enableSharedLibraries ? !stdenv.hostPlatform.isStatic ++, enableSharedLibraries ? (!stdenv.hostPlatform.isStatic && !stdenv.targetPlatform.isMinGW) + , enablePFM ? !(stdenv.isDarwin + || stdenv.isAarch64 # broken for Ampere eMAG 8180 (c2.large.arm on Packet) #56245 + || stdenv.isAarch32 # broken for the armv7l builder ++ || stdenv.targetPlatform.isMinGW + ) + , enablePolly ? false + }: +@@ -120,7 +121,7 @@ in stdenv.mkDerivation (rec { + "-DLLVM_INSTALL_CMAKE_DIR=${placeholder "dev"}/lib/cmake/llvm/" + "-DCMAKE_BUILD_TYPE=${if debugVersion then "Debug" else "Release"}" + "-DLLVM_INSTALL_UTILS=ON" # Needed by rustc +- "-DLLVM_BUILD_TESTS=ON" ++ "-DLLVM_BUILD_TESTS=${if stdenv.targetPlatform.isMinGW then "OFF" else "ON"}" + "-DLLVM_ENABLE_FFI=ON" + "-DLLVM_ENABLE_RTTI=ON" + "-DLLVM_HOST_TRIPLE=${stdenv.hostPlatform.config}" +@@ -134,7 +135,7 @@ in stdenv.mkDerivation (rec { + "-DSPHINX_OUTPUT_MAN=ON" + "-DSPHINX_OUTPUT_HTML=OFF" + "-DSPHINX_WARNINGS_AS_ERRORS=OFF" +- ] ++ optionals (!isDarwin) [ ++ ] ++ optionals (!isDarwin && !stdenv.targetPlatform.isMinGW) [ + "-DLLVM_BINUTILS_INCDIR=${libbfd.dev}/include" + ] ++ optionals isDarwin [ + "-DLLVM_ENABLE_LIBCXX=ON" From 701ca36e99bbd0354b78f648ed3c6a4f21b7c192 Mon Sep 17 00:00:00 2001 From: Sebastien Bourdeauducq Date: Fri, 26 Nov 2021 17:26:18 +0800 Subject: [PATCH 03/54] flake: windows build WIP --- flake.nix | 41 ++++++++++++++++++++++++++++++++++++++++- 1 file changed, 40 insertions(+), 1 deletion(-) diff --git a/flake.nix b/flake.nix index c75bec59..3ccc5c3e 100644 --- a/flake.nix +++ b/flake.nix @@ -13,6 +13,17 @@ patches = [ ./llvm-future-riscv-abi.diff ./llvm-restrict-targets.diff ./llvm-mingw-crosscompile.diff ]; }; pkgs = import nixpkgs-patched { system = "x86_64-linux"; }; + pkgs-mingw = import nixpkgs-patched { system = "x86_64-linux"; crossSystem = { config = "x86_64-w64-mingw32"; libc = "msvcrt"; }; }; + cargoSha256 = "sha256-otKLhr58HYMjVXAof6AdObNpggPnvK6qOl7I+4LWIP8="; + msys2-python-tar = pkgs.fetchurl { + url = "https://mirror.msys2.org/mingw/mingw64/mingw-w64-x86_64-python-3.9.7-4-any.pkg.tar.zst"; + sha256 = "0iwlgbk4b457yn9djwqswid55xhyyi35qymz1lfh42xwdpxdm47c"; + }; + msys2-python = pkgs.runCommand "msys2-python" { buildInputs = [ pkgs.gnutar pkgs.zstd ]; } + '' + mkdir $out + tar xvf ${msys2-python-tar} -C $out + ''; in rec { inherit nixpkgs-patched; @@ -21,7 +32,7 @@ pkgs.rustPlatform.buildRustPackage { name = "nac3artiq"; src = self; - cargoSha256 = "sha256-otKLhr58HYMjVXAof6AdObNpggPnvK6qOl7I+4LWIP8="; + inherit cargoSha256; nativeBuildInputs = [ pkgs.python3 pkgs.llvm_12 ]; buildInputs = [ pkgs.python3 pkgs.libffi pkgs.libxml2 pkgs.llvm_12 ]; cargoBuildFlags = [ "--package" "nac3artiq" ]; @@ -36,6 +47,34 @@ ); }; + packages.x86_64-w64-mingw32 = { + nac3artiq = pkgs-mingw.python3Packages.toPythonModule ( + pkgs-mingw.rustPlatform.buildRustPackage { + name = "nac3artiq"; + src = self; + inherit cargoSha256; + nativeBuildInputs = [ pkgs-mingw.llvm_12 ]; + buildInputs = [ pkgs-mingw.libffi pkgs-mingw.libxml2 pkgs-mingw.llvm_12 ]; + configurePhase = + '' + export PYO3_CROSS_PYTHON_VERSION=3.9 + export PYO3_CROSS_LIB_DIR=${msys2-python}/mingw64/lib + echo Using Python $PYO3_CROSS_PYTHON_VERSION in $PYO3_CROSS_LIB_DIR + ''; + cargoBuildFlags = [ "--package" "nac3artiq" ]; + doCheck = false; + installPhase = + '' + TARGET_DIR=$out/${pkgs.python3Packages.python.sitePackages} + mkdir -p $TARGET_DIR + #cp target/x86_64-unknown-linux-gnu/release/libnac3artiq.so $TARGET_DIR/nac3artiq.so + ls target + ''; + meta.platforms = ["x86_64-windows"]; + } + ); + }; + devShell.x86_64-linux = pkgs.mkShell { name = "nac3-dev-shell"; buildInputs = with pkgs; [ From 7ede4f15b6728ba9340eea580c5daa5791cdb438 Mon Sep 17 00:00:00 2001 From: ychenfo Date: Wed, 1 Dec 2021 02:52:00 +0800 Subject: [PATCH 04/54] nac3core: move builtin definitions to another file --- nac3core/src/toplevel/builtins.rs | 407 ++++++++++++++++++++++++++++++ nac3core/src/toplevel/composer.rs | 401 +---------------------------- nac3core/src/toplevel/mod.rs | 1 + 3 files changed, 409 insertions(+), 400 deletions(-) create mode 100644 nac3core/src/toplevel/builtins.rs diff --git a/nac3core/src/toplevel/builtins.rs b/nac3core/src/toplevel/builtins.rs new file mode 100644 index 00000000..97d9f8ff --- /dev/null +++ b/nac3core/src/toplevel/builtins.rs @@ -0,0 +1,407 @@ +use std::cell::RefCell; + +use inkwell::{IntPredicate, FloatPredicate}; + +use crate::symbol_resolver::SymbolValue; + +use super::*; + +pub fn get_built_ins(primitives: &mut (PrimitiveStore, Unifier)) -> Vec<(Arc>, Option)> { + let int32 = primitives.0.int32; + let int64 = primitives.0.int64; + let float = primitives.0.float; + let boolean = primitives.0.bool; + let range = primitives.0.range; + let string = primitives.0.str; + let num_ty = primitives.1.get_fresh_var_with_range(&[int32, int64, float, boolean]); + let var_map: HashMap<_, _> = vec![(num_ty.1, num_ty.0)].into_iter().collect(); + + let top_level_def_list = vec![ + Arc::new(RwLock::new(TopLevelComposer::make_top_level_class_def( + 0, + None, + "int32".into(), + None, + ))), + Arc::new(RwLock::new(TopLevelComposer::make_top_level_class_def( + 1, + None, + "int64".into(), + None, + ))), + Arc::new(RwLock::new(TopLevelComposer::make_top_level_class_def( + 2, + None, + "float".into(), + None, + ))), + Arc::new(RwLock::new(TopLevelComposer::make_top_level_class_def(3, None, "bool".into(), None))), + Arc::new(RwLock::new(TopLevelComposer::make_top_level_class_def(4, None, "none".into(), None))), + Arc::new(RwLock::new(TopLevelComposer::make_top_level_class_def( + 5, + None, + "range".into(), + None, + ))), + Arc::new(RwLock::new(TopLevelComposer::make_top_level_class_def(6, None, "str".into(), None))), + Arc::new(RwLock::new(TopLevelDef::Function { + name: "int32".into(), + simple_name: "int32".into(), + signature: primitives.1.add_ty(TypeEnum::TFunc(RefCell::new(FunSignature { + args: vec![FuncArg { name: "_".into(), ty: num_ty.0, default_value: None }], + ret: int32, + vars: var_map.clone(), + }))), + var_id: Default::default(), + instance_to_symbol: Default::default(), + instance_to_stmt: Default::default(), + resolver: None, + codegen_callback: Some(Arc::new(GenCall::new(Box::new( + |ctx, _, fun, args| { + let int32 = ctx.primitives.int32; + let int64 = ctx.primitives.int64; + let float = ctx.primitives.float; + let boolean = ctx.primitives.bool; + let arg_ty = fun.0.args[0].ty; + let arg = args[0].1; + if ctx.unifier.unioned(arg_ty, boolean) { + Some( + ctx.builder + .build_int_s_extend( + arg.into_int_value(), + ctx.ctx.i32_type(), + "sext", + ) + .into(), + ) + } else if ctx.unifier.unioned(arg_ty, int32) { + Some(arg) + } else if ctx.unifier.unioned(arg_ty, int64) { + Some( + ctx.builder + .build_int_truncate( + arg.into_int_value(), + ctx.ctx.i32_type(), + "trunc", + ) + .into(), + ) + } else if ctx.unifier.unioned(arg_ty, float) { + let val = ctx + .builder + .build_float_to_signed_int( + arg.into_float_value(), + ctx.ctx.i32_type(), + "fptosi", + ) + .into(); + Some(val) + } else { + unreachable!() + } + }, + )))), + })), + Arc::new(RwLock::new(TopLevelDef::Function { + name: "int64".into(), + simple_name: "int64".into(), + signature: primitives.1.add_ty(TypeEnum::TFunc(RefCell::new(FunSignature { + args: vec![FuncArg { name: "_".into(), ty: num_ty.0, default_value: None }], + ret: int64, + vars: var_map.clone(), + }))), + var_id: Default::default(), + instance_to_symbol: Default::default(), + instance_to_stmt: Default::default(), + resolver: None, + codegen_callback: Some(Arc::new(GenCall::new(Box::new( + |ctx, _, fun, args| { + let int32 = ctx.primitives.int32; + let int64 = ctx.primitives.int64; + let float = ctx.primitives.float; + let boolean = ctx.primitives.bool; + let arg_ty = fun.0.args[0].ty; + let arg = args[0].1; + if ctx.unifier.unioned(arg_ty, boolean) + || ctx.unifier.unioned(arg_ty, int32) + { + Some( + ctx.builder + .build_int_s_extend( + arg.into_int_value(), + ctx.ctx.i64_type(), + "sext", + ) + .into(), + ) + } else if ctx.unifier.unioned(arg_ty, int64) { + Some(arg) + } else if ctx.unifier.unioned(arg_ty, float) { + let val = ctx + .builder + .build_float_to_signed_int( + arg.into_float_value(), + ctx.ctx.i64_type(), + "fptosi", + ) + .into(); + Some(val) + } else { + unreachable!() + } + }, + )))), + })), + Arc::new(RwLock::new(TopLevelDef::Function { + name: "float".into(), + simple_name: "float".into(), + signature: primitives.1.add_ty(TypeEnum::TFunc(RefCell::new(FunSignature { + args: vec![FuncArg { name: "_".into(), ty: num_ty.0, default_value: None }], + ret: float, + vars: var_map.clone(), + }))), + var_id: Default::default(), + instance_to_symbol: Default::default(), + instance_to_stmt: Default::default(), + resolver: None, + codegen_callback: Some(Arc::new(GenCall::new(Box::new( + |ctx, _, fun, args| { + let int32 = ctx.primitives.int32; + let int64 = ctx.primitives.int64; + let boolean = ctx.primitives.bool; + let float = ctx.primitives.float; + let arg_ty = fun.0.args[0].ty; + let arg = args[0].1; + if ctx.unifier.unioned(arg_ty, boolean) + || ctx.unifier.unioned(arg_ty, int32) + || ctx.unifier.unioned(arg_ty, int64) + { + let arg = args[0].1.into_int_value(); + let val = ctx + .builder + .build_signed_int_to_float(arg, ctx.ctx.f64_type(), "sitofp") + .into(); + Some(val) + } else if ctx.unifier.unioned(arg_ty, float) { + Some(arg) + } else { + unreachable!() + } + }, + )))), + })), + Arc::new(RwLock::new(TopLevelDef::Function { + name: "round".into(), + simple_name: "round".into(), + signature: primitives.1.add_ty(TypeEnum::TFunc(RefCell::new(FunSignature { + args: vec![FuncArg { name: "_".into(), ty: float, default_value: None }], + ret: int32, + vars: Default::default(), + }))), + var_id: Default::default(), + instance_to_symbol: Default::default(), + instance_to_stmt: Default::default(), + resolver: None, + codegen_callback: Some(Arc::new(GenCall::new(Box::new(|ctx, _, _, args| { + let arg = args[0].1; + let round_intrinsic = + ctx.module.get_function("llvm.round.f64").unwrap_or_else(|| { + let float = ctx.ctx.f64_type(); + let fn_type = float.fn_type(&[float.into()], false); + ctx.module.add_function("llvm.round.f64", fn_type, None) + }); + let val = ctx + .builder + .build_call(round_intrinsic, &[arg], "round") + .try_as_basic_value() + .left() + .unwrap(); + Some( + ctx.builder + .build_float_to_signed_int( + val.into_float_value(), + ctx.ctx.i32_type(), + "fptosi", + ) + .into(), + ) + })))), + })), + Arc::new(RwLock::new(TopLevelDef::Function { + name: "round64".into(), + simple_name: "round64".into(), + signature: primitives.1.add_ty(TypeEnum::TFunc(RefCell::new(FunSignature { + args: vec![FuncArg { name: "_".into(), ty: float, default_value: None }], + ret: int64, + vars: Default::default(), + }))), + var_id: Default::default(), + instance_to_symbol: Default::default(), + instance_to_stmt: Default::default(), + resolver: None, + codegen_callback: Some(Arc::new(GenCall::new(Box::new(|ctx, _, _, args| { + let arg = args[0].1; + let round_intrinsic = + ctx.module.get_function("llvm.round.f64").unwrap_or_else(|| { + let float = ctx.ctx.f64_type(); + let fn_type = float.fn_type(&[float.into()], false); + ctx.module.add_function("llvm.round.f64", fn_type, None) + }); + let val = ctx + .builder + .build_call(round_intrinsic, &[arg], "round") + .try_as_basic_value() + .left() + .unwrap(); + Some( + ctx.builder + .build_float_to_signed_int( + val.into_float_value(), + ctx.ctx.i64_type(), + "fptosi", + ) + .into(), + ) + })))), + })), + Arc::new(RwLock::new(TopLevelDef::Function { + name: "range".into(), + simple_name: "range".into(), + signature: primitives.1.add_ty(TypeEnum::TFunc(RefCell::new(FunSignature { + args: vec![ + FuncArg { name: "start".into(), ty: int32, default_value: None }, + FuncArg { + name: "stop".into(), + ty: int32, + // placeholder + default_value: Some(SymbolValue::I32(0)), + }, + FuncArg { + name: "step".into(), + ty: int32, + default_value: Some(SymbolValue::I32(1)), + }, + ], + ret: range, + vars: Default::default(), + }))), + var_id: Default::default(), + instance_to_symbol: Default::default(), + instance_to_stmt: Default::default(), + resolver: None, + codegen_callback: Some(Arc::new(GenCall::new(Box::new(|ctx, _, _, args| { + let mut start = None; + let mut stop = None; + let mut step = None; + let int32 = ctx.ctx.i32_type(); + let zero = int32.const_zero(); + for (i, arg) in args.iter().enumerate() { + if arg.0 == Some("start".into()) { + start = Some(arg.1); + } else if arg.0 == Some("stop".into()) { + stop = Some(arg.1); + } else if arg.0 == Some("step".into()) { + step = Some(arg.1); + } else if i == 0 { + start = Some(arg.1); + } else if i == 1 { + stop = Some(arg.1); + } else if i == 2 { + step = Some(arg.1); + } + } + // TODO: error when step == 0 + let step = step.unwrap_or_else(|| int32.const_int(1, false).into()); + let stop = stop.unwrap_or_else(|| { + let v = start.unwrap(); + start = None; + v + }); + let start = start.unwrap_or_else(|| int32.const_zero().into()); + let ty = int32.array_type(3); + let ptr = ctx.builder.build_alloca(ty, "range"); + unsafe { + let a = ctx.builder.build_in_bounds_gep(ptr, &[zero, zero], "start"); + let b = ctx.builder.build_in_bounds_gep( + ptr, + &[zero, int32.const_int(1, false)], + "end", + ); + let c = ctx.builder.build_in_bounds_gep( + ptr, + &[zero, int32.const_int(2, false)], + "step", + ); + ctx.builder.build_store(a, start); + ctx.builder.build_store(b, stop); + ctx.builder.build_store(c, step); + } + Some(ptr.into()) + })))), + })), + Arc::new(RwLock::new(TopLevelDef::Function { + name: "str".into(), + simple_name: "str".into(), + signature: primitives.1.add_ty(TypeEnum::TFunc(RefCell::new(FunSignature { + args: vec![FuncArg { name: "_".into(), ty: string, default_value: None }], + ret: string, + vars: Default::default(), + }))), + var_id: Default::default(), + instance_to_symbol: Default::default(), + instance_to_stmt: Default::default(), + resolver: None, + codegen_callback: Some(Arc::new(GenCall::new(Box::new(|_, _, _, args| { + Some(args[0].1) + })))), + })), + Arc::new(RwLock::new(TopLevelDef::Function { + name: "bool".into(), + simple_name: "bool".into(), + signature: primitives.1.add_ty(TypeEnum::TFunc(RefCell::new(FunSignature { + args: vec![FuncArg { name: "_".into(), ty: num_ty.0, default_value: None }], + ret: primitives.0.bool, + vars: var_map, + }))), + var_id: Default::default(), + instance_to_symbol: Default::default(), + instance_to_stmt: Default::default(), + resolver: None, + codegen_callback: Some(Arc::new(GenCall::new(Box::new( + |ctx, _, fun, args| { + let int32 = ctx.primitives.int32; + let int64 = ctx.primitives.int64; + let float = ctx.primitives.float; + let boolean = ctx.primitives.bool; + let arg_ty = fun.0.args[0].ty; + let arg = args[0].1; + if ctx.unifier.unioned(arg_ty, boolean) { + Some(arg) + } else if ctx.unifier.unioned(arg_ty, int32) || ctx.unifier.unioned(arg_ty, int64) { + Some(ctx.builder.build_int_compare( + IntPredicate::NE, + ctx.ctx.i64_type().const_zero(), + arg.into_int_value(), + "bool", + ).into()) + } else if ctx.unifier.unioned(arg_ty, float) { + let val = ctx.builder. + build_float_compare( + // UEQ as bool(nan) is True + FloatPredicate::UEQ, + arg.into_float_value(), + ctx.ctx.f64_type().const_zero(), + "bool" + ).into(); + Some(val) + } else { + unreachable!() + } + }, + )))), + })), + ]; + let ast_list: Vec>> = + (0..top_level_def_list.len()).map(|_| None).collect(); + izip!(top_level_def_list, ast_list).collect_vec() +} \ No newline at end of file diff --git a/nac3core/src/toplevel/composer.rs b/nac3core/src/toplevel/composer.rs index a059362a..4eabb983 100644 --- a/nac3core/src/toplevel/composer.rs +++ b/nac3core/src/toplevel/composer.rs @@ -42,406 +42,7 @@ impl TopLevelComposer { builtins: Vec<(StrRef, FunSignature, Arc)>, ) -> (Self, HashMap, HashMap) { let mut primitives = Self::make_primitives(); - - let int32 = primitives.0.int32; - let int64 = primitives.0.int64; - let float = primitives.0.float; - let boolean = primitives.0.bool; - let range = primitives.0.range; - let string = primitives.0.str; - let num_ty = primitives.1.get_fresh_var_with_range(&[int32, int64, float, boolean]); - let var_map: HashMap<_, _> = vec![(num_ty.1, num_ty.0)].into_iter().collect(); - - let mut definition_ast_list = { - let top_level_def_list = vec![ - Arc::new(RwLock::new(Self::make_top_level_class_def( - 0, - None, - "int32".into(), - None, - ))), - Arc::new(RwLock::new(Self::make_top_level_class_def( - 1, - None, - "int64".into(), - None, - ))), - Arc::new(RwLock::new(Self::make_top_level_class_def( - 2, - None, - "float".into(), - None, - ))), - Arc::new(RwLock::new(Self::make_top_level_class_def(3, None, "bool".into(), None))), - Arc::new(RwLock::new(Self::make_top_level_class_def(4, None, "none".into(), None))), - Arc::new(RwLock::new(Self::make_top_level_class_def( - 5, - None, - "range".into(), - None, - ))), - Arc::new(RwLock::new(Self::make_top_level_class_def(6, None, "str".into(), None))), - Arc::new(RwLock::new(TopLevelDef::Function { - name: "int32".into(), - simple_name: "int32".into(), - signature: primitives.1.add_ty(TypeEnum::TFunc(RefCell::new(FunSignature { - args: vec![FuncArg { name: "_".into(), ty: num_ty.0, default_value: None }], - ret: int32, - vars: var_map.clone(), - }))), - var_id: Default::default(), - instance_to_symbol: Default::default(), - instance_to_stmt: Default::default(), - resolver: None, - codegen_callback: Some(Arc::new(GenCall::new(Box::new( - |ctx, _, fun, args| { - let int32 = ctx.primitives.int32; - let int64 = ctx.primitives.int64; - let float = ctx.primitives.float; - let boolean = ctx.primitives.bool; - let arg_ty = fun.0.args[0].ty; - let arg = args[0].1; - if ctx.unifier.unioned(arg_ty, boolean) { - Some( - ctx.builder - .build_int_s_extend( - arg.into_int_value(), - ctx.ctx.i32_type(), - "sext", - ) - .into(), - ) - } else if ctx.unifier.unioned(arg_ty, int32) { - Some(arg) - } else if ctx.unifier.unioned(arg_ty, int64) { - Some( - ctx.builder - .build_int_truncate( - arg.into_int_value(), - ctx.ctx.i32_type(), - "trunc", - ) - .into(), - ) - } else if ctx.unifier.unioned(arg_ty, float) { - let val = ctx - .builder - .build_float_to_signed_int( - arg.into_float_value(), - ctx.ctx.i32_type(), - "fptosi", - ) - .into(); - Some(val) - } else { - unreachable!() - } - }, - )))), - })), - Arc::new(RwLock::new(TopLevelDef::Function { - name: "int64".into(), - simple_name: "int64".into(), - signature: primitives.1.add_ty(TypeEnum::TFunc(RefCell::new(FunSignature { - args: vec![FuncArg { name: "_".into(), ty: num_ty.0, default_value: None }], - ret: int64, - vars: var_map.clone(), - }))), - var_id: Default::default(), - instance_to_symbol: Default::default(), - instance_to_stmt: Default::default(), - resolver: None, - codegen_callback: Some(Arc::new(GenCall::new(Box::new( - |ctx, _, fun, args| { - let int32 = ctx.primitives.int32; - let int64 = ctx.primitives.int64; - let float = ctx.primitives.float; - let boolean = ctx.primitives.bool; - let arg_ty = fun.0.args[0].ty; - let arg = args[0].1; - if ctx.unifier.unioned(arg_ty, boolean) - || ctx.unifier.unioned(arg_ty, int32) - { - Some( - ctx.builder - .build_int_s_extend( - arg.into_int_value(), - ctx.ctx.i64_type(), - "sext", - ) - .into(), - ) - } else if ctx.unifier.unioned(arg_ty, int64) { - Some(arg) - } else if ctx.unifier.unioned(arg_ty, float) { - let val = ctx - .builder - .build_float_to_signed_int( - arg.into_float_value(), - ctx.ctx.i64_type(), - "fptosi", - ) - .into(); - Some(val) - } else { - unreachable!() - } - }, - )))), - })), - Arc::new(RwLock::new(TopLevelDef::Function { - name: "float".into(), - simple_name: "float".into(), - signature: primitives.1.add_ty(TypeEnum::TFunc(RefCell::new(FunSignature { - args: vec![FuncArg { name: "_".into(), ty: num_ty.0, default_value: None }], - ret: float, - vars: var_map.clone(), - }))), - var_id: Default::default(), - instance_to_symbol: Default::default(), - instance_to_stmt: Default::default(), - resolver: None, - codegen_callback: Some(Arc::new(GenCall::new(Box::new( - |ctx, _, fun, args| { - let int32 = ctx.primitives.int32; - let int64 = ctx.primitives.int64; - let boolean = ctx.primitives.bool; - let float = ctx.primitives.float; - let arg_ty = fun.0.args[0].ty; - let arg = args[0].1; - if ctx.unifier.unioned(arg_ty, boolean) - || ctx.unifier.unioned(arg_ty, int32) - || ctx.unifier.unioned(arg_ty, int64) - { - let arg = args[0].1.into_int_value(); - let val = ctx - .builder - .build_signed_int_to_float(arg, ctx.ctx.f64_type(), "sitofp") - .into(); - Some(val) - } else if ctx.unifier.unioned(arg_ty, float) { - Some(arg) - } else { - unreachable!() - } - }, - )))), - })), - Arc::new(RwLock::new(TopLevelDef::Function { - name: "round".into(), - simple_name: "round".into(), - signature: primitives.1.add_ty(TypeEnum::TFunc(RefCell::new(FunSignature { - args: vec![FuncArg { name: "_".into(), ty: float, default_value: None }], - ret: int32, - vars: Default::default(), - }))), - var_id: Default::default(), - instance_to_symbol: Default::default(), - instance_to_stmt: Default::default(), - resolver: None, - codegen_callback: Some(Arc::new(GenCall::new(Box::new(|ctx, _, _, args| { - let arg = args[0].1; - let round_intrinsic = - ctx.module.get_function("llvm.round.f64").unwrap_or_else(|| { - let float = ctx.ctx.f64_type(); - let fn_type = float.fn_type(&[float.into()], false); - ctx.module.add_function("llvm.round.f64", fn_type, None) - }); - let val = ctx - .builder - .build_call(round_intrinsic, &[arg], "round") - .try_as_basic_value() - .left() - .unwrap(); - Some( - ctx.builder - .build_float_to_signed_int( - val.into_float_value(), - ctx.ctx.i32_type(), - "fptosi", - ) - .into(), - ) - })))), - })), - Arc::new(RwLock::new(TopLevelDef::Function { - name: "round64".into(), - simple_name: "round64".into(), - signature: primitives.1.add_ty(TypeEnum::TFunc(RefCell::new(FunSignature { - args: vec![FuncArg { name: "_".into(), ty: float, default_value: None }], - ret: int64, - vars: Default::default(), - }))), - var_id: Default::default(), - instance_to_symbol: Default::default(), - instance_to_stmt: Default::default(), - resolver: None, - codegen_callback: Some(Arc::new(GenCall::new(Box::new(|ctx, _, _, args| { - let arg = args[0].1; - let round_intrinsic = - ctx.module.get_function("llvm.round.f64").unwrap_or_else(|| { - let float = ctx.ctx.f64_type(); - let fn_type = float.fn_type(&[float.into()], false); - ctx.module.add_function("llvm.round.f64", fn_type, None) - }); - let val = ctx - .builder - .build_call(round_intrinsic, &[arg], "round") - .try_as_basic_value() - .left() - .unwrap(); - Some( - ctx.builder - .build_float_to_signed_int( - val.into_float_value(), - ctx.ctx.i64_type(), - "fptosi", - ) - .into(), - ) - })))), - })), - Arc::new(RwLock::new(TopLevelDef::Function { - name: "range".into(), - simple_name: "range".into(), - signature: primitives.1.add_ty(TypeEnum::TFunc(RefCell::new(FunSignature { - args: vec![ - FuncArg { name: "start".into(), ty: int32, default_value: None }, - FuncArg { - name: "stop".into(), - ty: int32, - // placeholder - default_value: Some(SymbolValue::I32(0)), - }, - FuncArg { - name: "step".into(), - ty: int32, - default_value: Some(SymbolValue::I32(1)), - }, - ], - ret: range, - vars: Default::default(), - }))), - var_id: Default::default(), - instance_to_symbol: Default::default(), - instance_to_stmt: Default::default(), - resolver: None, - codegen_callback: Some(Arc::new(GenCall::new(Box::new(|ctx, _, _, args| { - let mut start = None; - let mut stop = None; - let mut step = None; - let int32 = ctx.ctx.i32_type(); - let zero = int32.const_zero(); - for (i, arg) in args.iter().enumerate() { - if arg.0 == Some("start".into()) { - start = Some(arg.1); - } else if arg.0 == Some("stop".into()) { - stop = Some(arg.1); - } else if arg.0 == Some("step".into()) { - step = Some(arg.1); - } else if i == 0 { - start = Some(arg.1); - } else if i == 1 { - stop = Some(arg.1); - } else if i == 2 { - step = Some(arg.1); - } - } - // TODO: error when step == 0 - let step = step.unwrap_or_else(|| int32.const_int(1, false).into()); - let stop = stop.unwrap_or_else(|| { - let v = start.unwrap(); - start = None; - v - }); - let start = start.unwrap_or_else(|| int32.const_zero().into()); - let ty = int32.array_type(3); - let ptr = ctx.builder.build_alloca(ty, "range"); - unsafe { - let a = ctx.builder.build_in_bounds_gep(ptr, &[zero, zero], "start"); - let b = ctx.builder.build_in_bounds_gep( - ptr, - &[zero, int32.const_int(1, false)], - "end", - ); - let c = ctx.builder.build_in_bounds_gep( - ptr, - &[zero, int32.const_int(2, false)], - "step", - ); - ctx.builder.build_store(a, start); - ctx.builder.build_store(b, stop); - ctx.builder.build_store(c, step); - } - Some(ptr.into()) - })))), - })), - Arc::new(RwLock::new(TopLevelDef::Function { - name: "str".into(), - simple_name: "str".into(), - signature: primitives.1.add_ty(TypeEnum::TFunc(RefCell::new(FunSignature { - args: vec![FuncArg { name: "_".into(), ty: string, default_value: None }], - ret: string, - vars: Default::default(), - }))), - var_id: Default::default(), - instance_to_symbol: Default::default(), - instance_to_stmt: Default::default(), - resolver: None, - codegen_callback: Some(Arc::new(GenCall::new(Box::new(|_, _, _, args| { - Some(args[0].1) - })))), - })), - Arc::new(RwLock::new(TopLevelDef::Function { - name: "bool".into(), - simple_name: "bool".into(), - signature: primitives.1.add_ty(TypeEnum::TFunc(RefCell::new(FunSignature { - args: vec![FuncArg { name: "_".into(), ty: num_ty.0, default_value: None }], - ret: primitives.0.bool, - vars: var_map, - }))), - var_id: Default::default(), - instance_to_symbol: Default::default(), - instance_to_stmt: Default::default(), - resolver: None, - codegen_callback: Some(Arc::new(GenCall::new(Box::new( - |ctx, _, fun, args| { - let int32 = ctx.primitives.int32; - let int64 = ctx.primitives.int64; - let float = ctx.primitives.float; - let boolean = ctx.primitives.bool; - let arg_ty = fun.0.args[0].ty; - let arg = args[0].1; - if ctx.unifier.unioned(arg_ty, boolean) { - Some(arg) - } else if ctx.unifier.unioned(arg_ty, int32) || ctx.unifier.unioned(arg_ty, int64) { - Some(ctx.builder.build_int_compare( - IntPredicate::NE, - ctx.ctx.i64_type().const_zero(), - arg.into_int_value(), - "bool", - ).into()) - } else if ctx.unifier.unioned(arg_ty, float) { - let val = ctx.builder. - build_float_compare( - // UEQ as bool(nan) is True - FloatPredicate::UEQ, - arg.into_float_value(), - ctx.ctx.f64_type().const_zero(), - "bool" - ).into(); - Some(val) - } else { - unreachable!() - } - }, - )))), - })), - ]; - let ast_list: Vec>> = - (0..top_level_def_list.len()).map(|_| None).collect(); - izip!(top_level_def_list, ast_list).collect_vec() - }; + let mut definition_ast_list = builtins::get_built_ins(&mut primitives); let primitives_ty = primitives.0; let mut unifier = primitives.1; let mut keyword_list: HashSet = HashSet::from_iter(vec![ diff --git a/nac3core/src/toplevel/mod.rs b/nac3core/src/toplevel/mod.rs index f6fc15b0..55e3fe92 100644 --- a/nac3core/src/toplevel/mod.rs +++ b/nac3core/src/toplevel/mod.rs @@ -25,6 +25,7 @@ pub struct DefinitionId(pub usize); pub mod composer; pub mod helper; mod type_annotation; +pub mod builtins; use composer::*; use type_annotation::*; #[cfg(test)] From fa2fe8ed5d98f7ac3bd6794030a3af6b5da4e7fa Mon Sep 17 00:00:00 2001 From: ychenfo Date: Wed, 1 Dec 2021 03:23:58 +0800 Subject: [PATCH 05/54] nac3core: add ceil and floor --- nac3core/src/toplevel/builtins.rs | 176 +++++++++++++++++++++++++++++- nac3core/src/toplevel/composer.rs | 6 +- 2 files changed, 173 insertions(+), 9 deletions(-) diff --git a/nac3core/src/toplevel/builtins.rs b/nac3core/src/toplevel/builtins.rs index 97d9f8ff..3d2f4b71 100644 --- a/nac3core/src/toplevel/builtins.rs +++ b/nac3core/src/toplevel/builtins.rs @@ -1,12 +1,14 @@ use std::cell::RefCell; - use inkwell::{IntPredicate, FloatPredicate}; - use crate::symbol_resolver::SymbolValue; - use super::*; -pub fn get_built_ins(primitives: &mut (PrimitiveStore, Unifier)) -> Vec<(Arc>, Option)> { +type BuiltinInfo = ( + Vec<(Arc>, Option)>, + &'static [&'static str] +); + +pub fn get_built_ins(primitives: &mut (PrimitiveStore, Unifier)) -> BuiltinInfo { let int32 = primitives.0.int32; let int64 = primitives.0.int64; let float = primitives.0.float; @@ -400,8 +402,172 @@ pub fn get_built_ins(primitives: &mut (PrimitiveStore, Unifier)) -> Vec<(Arc>> = (0..top_level_def_list.len()).map(|_| None).collect(); - izip!(top_level_def_list, ast_list).collect_vec() + ( + izip!(top_level_def_list, ast_list).collect_vec(), + &[ + "int32", + "int64", + "float", + "round", + "round64", + "range", + "str", + "bool", + "floor", + "floor64", + "ceil", + "ceil64" + ] + ) } \ No newline at end of file diff --git a/nac3core/src/toplevel/composer.rs b/nac3core/src/toplevel/composer.rs index 4eabb983..051af13f 100644 --- a/nac3core/src/toplevel/composer.rs +++ b/nac3core/src/toplevel/composer.rs @@ -42,7 +42,7 @@ impl TopLevelComposer { builtins: Vec<(StrRef, FunSignature, Arc)>, ) -> (Self, HashMap, HashMap) { let mut primitives = Self::make_primitives(); - let mut definition_ast_list = builtins::get_built_ins(&mut primitives); + let (mut definition_ast_list, builtin_name_list) = builtins::get_built_ins(&mut primitives); let primitives_ty = primitives.0; let mut unifier = primitives.1; let mut keyword_list: HashSet = HashSet::from_iter(vec![ @@ -68,9 +68,7 @@ impl TopLevelComposer { let mut built_in_id: HashMap = Default::default(); let mut built_in_ty: HashMap = Default::default(); - for (id, name) in - ["int32", "int64", "float", "round", "round64", "range", "str", "bool"].iter().rev().enumerate() - { + for (id, name) in builtin_name_list.iter().rev().enumerate() { let name = (**name).into(); let id = definition_ast_list.len() - id - 1; let def = definition_ast_list[id].0.read(); From 31fba04cee5052c572ca4b5472a66947d077a2ce Mon Sep 17 00:00:00 2001 From: Sebastien Bourdeauducq Date: Wed, 1 Dec 2021 18:30:26 +0800 Subject: [PATCH 06/54] flake: fix Windows build, now finding LLVM and Python --- flake.nix | 24 +++++++++++++++++++----- 1 file changed, 19 insertions(+), 5 deletions(-) diff --git a/flake.nix b/flake.nix index 3ccc5c3e..041e7133 100644 --- a/flake.nix +++ b/flake.nix @@ -24,6 +24,21 @@ mkdir $out tar xvf ${msys2-python-tar} -C $out ''; + pyo3-mingw-config = pkgs.writeTextFile { + name = "pyo3-mingw-config"; + text = + '' + implementation=CPython + version=3.9 + shared=true + abi3=false + lib_name=python3.9 + lib_dir=${msys2-python}/mingw64/lib + pointer_width=64 + build_flags=WITH_THREAD + suppress_build_script_link_lines=false + ''; + }; in rec { inherit nixpkgs-patched; @@ -53,13 +68,12 @@ name = "nac3artiq"; src = self; inherit cargoSha256; - nativeBuildInputs = [ pkgs-mingw.llvm_12 ]; - buildInputs = [ pkgs-mingw.libffi pkgs-mingw.libxml2 pkgs-mingw.llvm_12 ]; configurePhase = '' - export PYO3_CROSS_PYTHON_VERSION=3.9 - export PYO3_CROSS_LIB_DIR=${msys2-python}/mingw64/lib - echo Using Python $PYO3_CROSS_PYTHON_VERSION in $PYO3_CROSS_LIB_DIR + export PYO3_CONFIG_FILE=${pyo3-mingw-config} + mkdir llvm-cfg + ln -s ${pkgs-mingw.llvm_12.dev}/bin/llvm-config-native llvm-cfg/llvm-config + export PATH=$PATH:`pwd`/llvm-cfg ''; cargoBuildFlags = [ "--package" "nac3artiq" ]; doCheck = false; From dfd3548ed26d39e480c5c664a02dfd00f0922fdf Mon Sep 17 00:00:00 2001 From: ychenfo Date: Wed, 1 Dec 2021 22:44:53 +0800 Subject: [PATCH 07/54] TypeVar and virtual support in Symbol Resolver (#99) Add `TypeVar` and `virtual` support for Symbol Resolver in nac3artiq and nac3standalone Reviewed-on: https://git.m-labs.hk/M-Labs/nac3/pulls/99 Co-authored-by: ychenfo Co-committed-by: ychenfo --- nac3artiq/demo/min_artiq.py | 7 +- nac3artiq/src/lib.rs | 45 +++ nac3artiq/src/symbol_resolver.rs | 354 ++++++++++++++++++----- nac3core/src/symbol_resolver.rs | 290 ++++++++++--------- nac3core/src/toplevel/composer.rs | 2 +- nac3core/src/toplevel/mod.rs | 2 +- nac3core/src/toplevel/type_annotation.rs | 224 +++++++------- nac3standalone/src/main.rs | 97 ++++++- 8 files changed, 687 insertions(+), 334 deletions(-) diff --git a/nac3artiq/demo/min_artiq.py b/nac3artiq/demo/min_artiq.py index 2fbd0765..fcb59ce9 100644 --- a/nac3artiq/demo/min_artiq.py +++ b/nac3artiq/demo/min_artiq.py @@ -9,13 +9,18 @@ import nac3artiq __all__ = ["KernelInvariant", "extern", "kernel", "portable", "nac3", "ms", "us", "ns", "print_int32", "print_int64", - "Core", "TTLOut", "parallel", "sequential"] + "Core", "TTLOut", "parallel", "sequential", "virtual"] T = TypeVar('T') + class KernelInvariant(Generic[T]): pass +# place the `virtual` class infront of the construct of NAC3 object to ensure the +# virtual class is known during the initializing of NAC3 object +class virtual(Generic[T]): + pass import device_db core_arguments = device_db.device_db["core"]["arguments"] diff --git a/nac3artiq/src/lib.rs b/nac3artiq/src/lib.rs index 61e16b94..ac087994 100644 --- a/nac3artiq/src/lib.rs +++ b/nac3artiq/src/lib.rs @@ -55,6 +55,10 @@ pub struct PrimitivePythonId { bool: u64, list: u64, tuple: u64, + typevar: u64, + none: u64, + generic_alias: (u64, u64), + virtual_id: u64, } // TopLevelComposer is unsendable as it holds the unification table, which is @@ -96,10 +100,13 @@ impl Nac3 { let val = id_fn.call1((member.get_item(1)?,))?.extract()?; name_to_pyid.insert(key.into(), val); } + let typings = PyModule::import(py, "typing")?; let helper = PythonHelper { id_fn: builtins.getattr("id").unwrap().to_object(py), len_fn: builtins.getattr("len").unwrap().to_object(py), type_fn: builtins.getattr("type").unwrap().to_object(py), + origin_ty_fn: typings.getattr("get_origin").unwrap().to_object(py), + args_ty_fn: typings.getattr("get_args").unwrap().to_object(py), }; Ok(( module.getattr("__name__")?.extract()?, @@ -284,7 +291,42 @@ impl Nac3 { let builtins_mod = PyModule::import(py, "builtins").unwrap(); let id_fn = builtins_mod.getattr("id").unwrap(); let numpy_mod = PyModule::import(py, "numpy").unwrap(); + let typing_mod = PyModule::import(py, "typing").unwrap(); + let types_mod = PyModule::import(py, "types").unwrap(); let primitive_ids = PrimitivePythonId { + virtual_id: id_fn + .call1((builtins_mod + .getattr("globals") + .unwrap() + .call0() + .unwrap() + .get_item("virtual") + .unwrap(), + )).unwrap() + .extract() + .unwrap(), + generic_alias: ( + id_fn + .call1((typing_mod.getattr("_GenericAlias").unwrap(),)) + .unwrap() + .extract() + .unwrap(), + id_fn + .call1((types_mod.getattr("GenericAlias").unwrap(),)) + .unwrap() + .extract() + .unwrap(), + ), + none: id_fn + .call1((builtins_mod.getattr("None").unwrap(),)) + .unwrap() + .extract() + .unwrap(), + typevar: id_fn + .call1((typing_mod.getattr("TypeVar").unwrap(),)) + .unwrap() + .extract() + .unwrap(), int: id_fn .call1((builtins_mod.getattr("int").unwrap(),)) .unwrap() @@ -403,10 +445,13 @@ impl Nac3 { }; let mut synthesized = parse_program(&synthesized).unwrap(); let builtins = PyModule::import(py, "builtins")?; + let typings = PyModule::import(py, "typing")?; let helper = PythonHelper { id_fn: builtins.getattr("id").unwrap().to_object(py), len_fn: builtins.getattr("len").unwrap().to_object(py), type_fn: builtins.getattr("type").unwrap().to_object(py), + origin_ty_fn: typings.getattr("get_origin").unwrap().to_object(py), + args_ty_fn: typings.getattr("get_args").unwrap().to_object(py), }; let resolver = Arc::new(Resolver(Arc::new(InnerResolver { id_to_type: self.builtins_ty.clone().into(), diff --git a/nac3artiq/src/symbol_resolver.rs b/nac3artiq/src/symbol_resolver.rs index 0a85824b..f36c146f 100644 --- a/nac3artiq/src/symbol_resolver.rs +++ b/nac3artiq/src/symbol_resolver.rs @@ -43,6 +43,8 @@ pub struct PythonHelper { pub type_fn: PyObject, pub len_fn: PyObject, pub id_fn: PyObject, + pub origin_ty_fn: PyObject, + pub args_ty_fn: PyObject, } struct PythonValue { @@ -133,47 +135,46 @@ impl InnerResolver { })) } - fn get_obj_type( + // handle python objects that represent types themselves + // primitives and class types should be themselves, use `ty_id` to check, + // TypeVars and GenericAlias(`A[int, bool]`) should use `ty_ty_id` to check + // the `bool` value returned indicates whether they are instantiated or not + fn get_pyty_obj_type( &self, py: Python, - obj: &PyAny, + pyty: &PyAny, unifier: &mut Unifier, defs: &[Arc>], primitives: &PrimitiveStore, - ) -> PyResult> { + ) -> PyResult> { let ty_id: u64 = self .helper .id_fn - .call1(py, (self.helper.type_fn.call1(py, (obj,))?,))? + .call1(py, (pyty,))? + .extract(py)?; + let ty_ty_id: u64 = self + .helper + .id_fn + .call1(py, (self.helper.type_fn.call1(py, (pyty,))?,))? .extract(py)?; if ty_id == self.primitive_ids.int || ty_id == self.primitive_ids.int32 { - Ok(Some(primitives.int32)) + Ok(Ok((primitives.int32, true))) } else if ty_id == self.primitive_ids.int64 { - Ok(Some(primitives.int64)) + Ok(Ok((primitives.int64, true))) } else if ty_id == self.primitive_ids.bool { - Ok(Some(primitives.bool)) + Ok(Ok((primitives.bool, true))) } else if ty_id == self.primitive_ids.float { - Ok(Some(primitives.float)) + Ok(Ok((primitives.float, true))) } else if ty_id == self.primitive_ids.list { - let len: usize = self.helper.len_fn.call1(py, (obj,))?.extract(py)?; - if len == 0 { - let var = unifier.get_fresh_var().0; - let list = unifier.add_ty(TypeEnum::TList { ty: var }); - Ok(Some(list)) - } else { - let ty = self.get_list_elem_type(py, obj, len, unifier, defs, primitives)?; - Ok(ty.map(|ty| unifier.add_ty(TypeEnum::TList { ty }))) - } + // do not handle type var param and concrete check here + let var = unifier.get_fresh_var().0; + let list = unifier.add_ty(TypeEnum::TList { ty: var }); + Ok(Ok((list, false))) } else if ty_id == self.primitive_ids.tuple { - let elements: &PyTuple = obj.cast_as()?; - let types: Result>, _> = elements - .iter() - .map(|elem| self.get_obj_type(py, elem, unifier, defs, primitives)) - .collect(); - let types = types?; - Ok(types.map(|types| unifier.add_ty(TypeEnum::TTuple { ty: types }))) - } else if let Some(def_id) = self.pyid_to_def.read().get(&ty_id) { + // do not handle type var param and concrete check here + Ok(Ok((unifier.add_ty(TypeEnum::TTuple { ty: vec![] }), false))) + } else if let Some(def_id) = self.pyid_to_def.read().get(&ty_id).cloned() { let def = defs[def_id.0].read(); if let TopLevelDef::Class { object_id, @@ -183,54 +184,275 @@ impl InnerResolver { .. } = &*def { - let var_map: HashMap<_, _> = type_vars - .iter() - .map(|var| { - ( - if let TypeEnum::TVar { id, .. } = &*unifier.get_ty(*var) { - *id - } else { - unreachable!() - }, - unifier.get_fresh_var().0, - ) - }) - .collect(); - let mut fields_ty = HashMap::new(); - for method in methods.iter() { - fields_ty.insert(method.0, (method.1, false)); - } - for field in fields.iter() { - let name: String = field.0.into(); - let field_data = obj.getattr(&name)?; - let ty = self - .get_obj_type(py, field_data, unifier, defs, primitives)? - .unwrap_or(primitives.none); - let field_ty = unifier.subst(field.1, &var_map).unwrap_or(field.1); - if unifier.unify(ty, field_ty).is_err() { - // field type mismatch - return Ok(None); + // do not handle type var param and concrete check here, and no subst + Ok(Ok({ + let ty = TypeEnum::TObj { + obj_id: *object_id, + params: RefCell::new({ + type_vars + .iter() + .map(|x| { + if let TypeEnum::TVar { id, .. } = &*unifier.get_ty(*x) { + (*id, *x) + } else { + println!("{}", unifier.default_stringify(*x)); + unreachable!() + } + }).collect() + }), + fields: RefCell::new({ + let mut res = methods + .iter() + .map(|(iden, ty, _)| (*iden, (*ty, false))) + .collect::>(); + res.extend(fields.clone().into_iter().map(|x| (x.0, (x.1, x.2)))); + res + }) + }; + // here also false, later instantiation use python object to check compatible + (unifier.add_ty(ty), false) + })) + } else { + // only object is supported, functions are not supported + unreachable!("function type is not supported, should not be queried") + } + } else if ty_ty_id == self.primitive_ids.typevar { + let constraint_types = { + let constraints = pyty.getattr("__constraints__").unwrap(); + let mut result: Vec = vec![]; + for i in 0.. { + if let Ok(constr) = constraints.get_item(i) { + result.push({ + match self.get_pyty_obj_type(py, constr, unifier, defs, primitives)? { + Ok((ty, _)) => { + if unifier.is_concrete(ty, &[]) { + ty + } else { + return Ok(Err(format!( + "the {}th constraint of TypeVar `{}` is not concrete", + i + 1, + pyty.getattr("__name__")?.extract::()? + ))) + } + }, + Err(err) => return Ok(Err(err)) + } + }) + } else { + break; + } + } + result + }; + let res = unifier.get_fresh_var_with_range(&constraint_types).0; + Ok(Ok((res, true))) + } else if ty_ty_id == self.primitive_ids.generic_alias.0 || ty_ty_id == self.primitive_ids.generic_alias.1 { + let origin = self.helper.origin_ty_fn.call1(py, (pyty,))?; + let args = self.helper.args_ty_fn.call1(py, (pyty,))?; + let args: &PyTuple = args.cast_as(py)?; + let origin_ty = match self.get_pyty_obj_type(py, origin.as_ref(py), unifier, defs, primitives)? { + Ok((ty, false)) => ty, + Ok((_, true)) => return Ok(Err("instantiated type does not take type parameters".into())), + Err(err) => return Ok(Err(err)) + }; + + match &*unifier.get_ty(origin_ty) { + TypeEnum::TList { .. } => { + if args.len() == 1 { + let ty = match self.get_pyty_obj_type(py, args.get_item(0), unifier, defs, primitives)? { + Ok(ty) => ty, + Err(err) => return Ok(Err(err)) + }; + if !unifier.is_concrete(ty.0, &[]) && !ty.1 { + panic!("type list should take concrete parameters in type var ranges") + } + Ok(Ok((unifier.add_ty(TypeEnum::TList { ty: ty.0 }), true))) + } else { + return Ok(Err(format!("type list needs exactly 1 type parameters, found {}", args.len()))) + } + }, + TypeEnum::TTuple { .. } => { + let args = match args + .iter() + .map(|x| self.get_pyty_obj_type(py, x, unifier, defs, primitives)) + .collect::, _>>()? + .into_iter() + .collect::, _>>() { + Ok(args) if !args.is_empty() => args + .into_iter() + .map(|(x, check)| if !unifier.is_concrete(x, &[]) && !check { + panic!("type tuple should take concrete parameters in type var ranges") + } else { + x + } + ) + .collect::>(), + Err(err) => return Ok(Err(err)), + _ => return Ok(Err("tuple type needs at least 1 type parameters".to_string())) + }; + Ok(Ok((unifier.add_ty(TypeEnum::TTuple { ty: args }), true))) + }, + TypeEnum::TObj { params, obj_id, .. } => { + let subst = { + let params = &*params.borrow(); + if params.len() != args.len() { + return Ok(Err(format!( + "for class #{}, expect {} type parameters, got {}.", + obj_id.0, + params.len(), + args.len(), + ))) + } + let args = match args + .iter() + .map(|x| self.get_pyty_obj_type(py, x, unifier, defs, primitives)) + .collect::, _>>()? + .into_iter() + .collect::, _>>() { + Ok(args) => args + .into_iter() + .map(|(x, check)| if !unifier.is_concrete(x, &[]) && !check { + panic!("type class should take concrete parameters in type var ranges") + } else { + x + } + ) + .collect::>(), + Err(err) => return Ok(Err(err)), + }; + params + .iter() + .zip(args.iter()) + .map(|((id, _), ty)| (*id, *ty)) + .collect::>() + }; + Ok(Ok((unifier.subst(origin_ty, &subst).unwrap_or(origin_ty), true))) + }, + TypeEnum::TVirtual { .. } => { + if args.len() == 1 { + let ty = match self.get_pyty_obj_type(py, args.get_item(0), unifier, defs, primitives)? { + Ok(ty) => ty, + Err(err) => return Ok(Err(err)) + }; + if !unifier.is_concrete(ty.0, &[]) && !ty.1 { + panic!("virtual class should take concrete parameters in type var ranges") + } + Ok(Ok((unifier.add_ty(TypeEnum::TVirtual { ty: ty.0 }), true))) + } else { + return Ok(Err(format!("virtual class needs exactly 1 type parameters, found {}", args.len()))) + } + } + _ => unimplemented!() + } + } else if ty_id == self.primitive_ids.virtual_id { + Ok(Ok(({ + let ty = TypeEnum::TVirtual { ty: unifier.get_fresh_var().0 }; + unifier.add_ty(ty) + }, false))) + } else { + Ok(Err("unknown type".into())) + } + } + + fn get_obj_type( + &self, + py: Python, + obj: &PyAny, + unifier: &mut Unifier, + defs: &[Arc>], + primitives: &PrimitiveStore, + ) -> PyResult> { + let ty = self.helper.type_fn.call1(py, (obj,)).unwrap(); + let (extracted_ty, inst_check) = match self.get_pyty_obj_type( + py, + { + if [self.primitive_ids.typevar, + self.primitive_ids.generic_alias.0, + self.primitive_ids.generic_alias.1 + ].contains(&self.helper.id_fn.call1(py, (ty.clone(),))?.extract::(py)?) { + obj + } else { + ty.as_ref(py) + } + }, + unifier, + defs, + primitives + )? { + Ok(s) => s, + Err(_) => return Ok(None) + }; + return match (&*unifier.get_ty(extracted_ty), inst_check) { + // do the instantiation for these three types + (TypeEnum::TList { ty }, false) => { + let len: usize = self.helper.len_fn.call1(py, (obj,))?.extract(py)?; + if len == 0 { + assert!(matches!( + &*unifier.get_ty(extracted_ty), + TypeEnum::TVar { meta: nac3core::typecheck::typedef::TypeVarMeta::Generic, range, .. } + if range.borrow().is_empty() + )); + Ok(Some(extracted_ty)) + } else { + let actual_ty = self + .get_list_elem_type(py, obj, len, unifier, defs, primitives)?; + if let Some(actual_ty) = actual_ty { + unifier.unify(*ty, actual_ty).unwrap(); + Ok(Some(extracted_ty)) + } else { + Ok(None) + } + } + } + (TypeEnum::TTuple { .. }, false) => { + let elements: &PyTuple = obj.cast_as()?; + let types: Result>, _> = elements + .iter() + .map(|elem| self.get_obj_type(py, elem, unifier, defs, primitives)) + .collect(); + let types = types?; + Ok(types.map(|types| unifier.add_ty(TypeEnum::TTuple { ty: types }))) + } + (TypeEnum::TObj { params, fields, .. }, false) => { + let var_map = params + .borrow() + .iter() + .map(|(id_var, ty)| { + if let TypeEnum::TVar { id, range, .. } = &*unifier.get_ty(*ty) { + assert_eq!(*id, *id_var); + (*id, unifier.get_fresh_var_with_range(&range.borrow()).0) + } else { + unreachable!() + } + }) + .collect::>(); + // loop through non-function fields of the class to get the instantiated value + for field in fields.borrow().iter() { + let name: String = (*field.0).into(); + if let TypeEnum::TFunc( .. ) = &*unifier.get_ty(field.1.0) { + continue; + } else { + let field_data = obj.getattr(&name)?; + let ty = self + .get_obj_type(py, field_data, unifier, defs, primitives)? + .unwrap_or(primitives.none); + let field_ty = unifier.subst(field.1.0, &var_map).unwrap_or(field.1.0); + if unifier.unify(ty, field_ty).is_err() { + // field type mismatch + return Ok(None); + } } - fields_ty.insert(field.0, (ty, field.2)); } for (_, ty) in var_map.iter() { // must be concrete type if !unifier.is_concrete(*ty, &[]) { - return Ok(None); + return Ok(None) } } - Ok(Some(unifier.add_ty(TypeEnum::TObj { - obj_id: *object_id, - fields: RefCell::new(fields_ty), - params: RefCell::new(var_map), - }))) - } else { - // only object is supported, functions are not supported - Ok(None) + return Ok(Some(unifier.subst(extracted_ty, &var_map).unwrap_or(extracted_ty))); } - } else { - Ok(None) - } + _ => Ok(Some(extracted_ty)) + }; } fn get_obj_value<'ctx, 'a>( @@ -620,4 +842,4 @@ impl SymbolResolver for Resolver { result }) } -} +} \ No newline at end of file diff --git a/nac3core/src/symbol_resolver.rs b/nac3core/src/symbol_resolver.rs index d7e9d697..3f2d036a 100644 --- a/nac3core/src/symbol_resolver.rs +++ b/nac3core/src/symbol_resolver.rs @@ -138,159 +138,165 @@ pub fn parse_type_annotation( let list_id = ids[6]; let tuple_id = ids[7]; - match &expr.node { - Name { id, .. } => { - if *id == int32_id { - Ok(primitives.int32) - } else if *id == int64_id { - Ok(primitives.int64) - } else if *id == float_id { - Ok(primitives.float) - } else if *id == bool_id { - Ok(primitives.bool) - } else if *id == none_id { - Ok(primitives.none) - } else { - let obj_id = resolver.get_identifier_def(*id); - if let Some(obj_id) = obj_id { - let def = top_level_defs[obj_id.0].read(); - if let TopLevelDef::Class { fields, methods, type_vars, .. } = &*def { - if !type_vars.is_empty() { - return Err(format!( - "Unexpected number of type parameters: expected {} but got 0", - type_vars.len() - )); - } - let fields = RefCell::new( - chain( - fields.iter().map(|(k, v, m)| (*k, (*v, *m))), - methods.iter().map(|(k, v, _)| (*k, (*v, false))), - ) - .collect(), - ); - Ok(unifier.add_ty(TypeEnum::TObj { - obj_id, - fields, - params: Default::default(), - })) - } else { - Err("Cannot use function name as type".into()) + let name_handling = |id: &StrRef, unifier: &mut Unifier| { + if *id == int32_id { + Ok(primitives.int32) + } else if *id == int64_id { + Ok(primitives.int64) + } else if *id == float_id { + Ok(primitives.float) + } else if *id == bool_id { + Ok(primitives.bool) + } else if *id == none_id { + Ok(primitives.none) + } else { + let obj_id = resolver.get_identifier_def(*id); + if let Some(obj_id) = obj_id { + let def = top_level_defs[obj_id.0].read(); + if let TopLevelDef::Class { fields, methods, type_vars, .. } = &*def { + if !type_vars.is_empty() { + return Err(format!( + "Unexpected number of type parameters: expected {} but got 0", + type_vars.len() + )); } + let fields = RefCell::new( + chain( + fields.iter().map(|(k, v, m)| (*k, (*v, *m))), + methods.iter().map(|(k, v, _)| (*k, (*v, false))), + ) + .collect(), + ); + Ok(unifier.add_ty(TypeEnum::TObj { + obj_id, + fields, + params: Default::default(), + })) } else { - // it could be a type variable - let ty = resolver - .get_symbol_type(unifier, top_level_defs, primitives, *id) - .ok_or_else(|| "unknown type variable name".to_owned())?; - if let TypeEnum::TVar { .. } = &*unifier.get_ty(ty) { - Ok(ty) - } else { - Err(format!("Unknown type annotation {}", id)) - } + Err("Cannot use function name as type".into()) + } + } else { + // it could be a type variable + let ty = resolver + .get_symbol_type(unifier, top_level_defs, primitives, *id) + .ok_or_else(|| "unknown type variable name".to_owned())?; + if let TypeEnum::TVar { .. } = &*unifier.get_ty(ty) { + Ok(ty) + } else { + Err(format!("Unknown type annotation {}", id)) } } } - Subscript { value, slice, .. } => { - if let Name { id, .. } = &value.node { - if *id == virtual_id { - let ty = parse_type_annotation( - resolver, - top_level_defs, - unifier, - primitives, - slice, - )?; - Ok(unifier.add_ty(TypeEnum::TVirtual { ty })) - } else if *id == list_id { - let ty = parse_type_annotation( - resolver, - top_level_defs, - unifier, - primitives, - slice, - )?; - Ok(unifier.add_ty(TypeEnum::TList { ty })) - } else if *id == tuple_id { - if let Tuple { elts, .. } = &slice.node { - let ty = elts - .iter() - .map(|elt| { - parse_type_annotation( - resolver, - top_level_defs, - unifier, - primitives, - elt, - ) - }) - .collect::, _>>()?; - Ok(unifier.add_ty(TypeEnum::TTuple { ty })) - } else { - Err("Expected multiple elements for tuple".into()) - } - } else { - let types = if let Tuple { elts, .. } = &slice.node { - elts.iter() - .map(|v| { - parse_type_annotation( - resolver, - top_level_defs, - unifier, - primitives, - v, - ) - }) - .collect::, _>>()? - } else { - vec![parse_type_annotation( + }; + + let subscript_name_handle = |id: &StrRef, slice: &Expr, unifier: &mut Unifier| { + if *id == virtual_id { + let ty = parse_type_annotation( + resolver, + top_level_defs, + unifier, + primitives, + slice, + )?; + Ok(unifier.add_ty(TypeEnum::TVirtual { ty })) + } else if *id == list_id { + let ty = parse_type_annotation( + resolver, + top_level_defs, + unifier, + primitives, + slice, + )?; + Ok(unifier.add_ty(TypeEnum::TList { ty })) + } else if *id == tuple_id { + if let Tuple { elts, .. } = &slice.node { + let ty = elts + .iter() + .map(|elt| { + parse_type_annotation( resolver, top_level_defs, unifier, primitives, - slice, - )?] - }; + elt, + ) + }) + .collect::, _>>()?; + Ok(unifier.add_ty(TypeEnum::TTuple { ty })) + } else { + Err("Expected multiple elements for tuple".into()) + } + } else { + let types = if let Tuple { elts, .. } = &slice.node { + elts.iter() + .map(|v| { + parse_type_annotation( + resolver, + top_level_defs, + unifier, + primitives, + v, + ) + }) + .collect::, _>>()? + } else { + vec![parse_type_annotation( + resolver, + top_level_defs, + unifier, + primitives, + slice, + )?] + }; - let obj_id = resolver - .get_identifier_def(*id) - .ok_or_else(|| format!("Unknown type annotation {}", id))?; - let def = top_level_defs[obj_id.0].read(); - if let TopLevelDef::Class { fields, methods, type_vars, .. } = &*def { - if types.len() != type_vars.len() { - return Err(format!( - "Unexpected number of type parameters: expected {} but got {}", - type_vars.len(), - types.len() - )); - } - let mut subst = HashMap::new(); - for (var, ty) in izip!(type_vars.iter(), types.iter()) { - let id = if let TypeEnum::TVar { id, .. } = &*unifier.get_ty(*var) { - *id - } else { - unreachable!() - }; - subst.insert(id, *ty); - } - let mut fields = fields - .iter() - .map(|(attr, ty, is_mutable)| { - let ty = unifier.subst(*ty, &subst).unwrap_or(*ty); - (*attr, (ty, *is_mutable)) - }) - .collect::>(); - fields.extend(methods.iter().map(|(attr, ty, _)| { - let ty = unifier.subst(*ty, &subst).unwrap_or(*ty); - (*attr, (ty, false)) - })); - Ok(unifier.add_ty(TypeEnum::TObj { - obj_id, - fields: fields.into(), - params: subst.into(), - })) - } else { - Err("Cannot use function name as type".into()) - } + let obj_id = resolver + .get_identifier_def(*id) + .ok_or_else(|| format!("Unknown type annotation {}", id))?; + let def = top_level_defs[obj_id.0].read(); + if let TopLevelDef::Class { fields, methods, type_vars, .. } = &*def { + if types.len() != type_vars.len() { + return Err(format!( + "Unexpected number of type parameters: expected {} but got {}", + type_vars.len(), + types.len() + )); } + let mut subst = HashMap::new(); + for (var, ty) in izip!(type_vars.iter(), types.iter()) { + let id = if let TypeEnum::TVar { id, .. } = &*unifier.get_ty(*var) { + *id + } else { + unreachable!() + }; + subst.insert(id, *ty); + } + let mut fields = fields + .iter() + .map(|(attr, ty, is_mutable)| { + let ty = unifier.subst(*ty, &subst).unwrap_or(*ty); + (*attr, (ty, *is_mutable)) + }) + .collect::>(); + fields.extend(methods.iter().map(|(attr, ty, _)| { + let ty = unifier.subst(*ty, &subst).unwrap_or(*ty); + (*attr, (ty, false)) + })); + Ok(unifier.add_ty(TypeEnum::TObj { + obj_id, + fields: fields.into(), + params: subst.into(), + })) + } else { + Err("Cannot use function name as type".into()) + } + } + }; + + match &expr.node { + Name { id, .. } => name_handling(id, unifier), + Subscript { value, slice, .. } => { + if let Name { id, .. } = &value.node { + subscript_name_handle(id, slice, unifier) } else { Err("unsupported type expression".into()) } diff --git a/nac3core/src/toplevel/composer.rs b/nac3core/src/toplevel/composer.rs index a059362a..50c5344e 100644 --- a/nac3core/src/toplevel/composer.rs +++ b/nac3core/src/toplevel/composer.rs @@ -532,7 +532,7 @@ impl TopLevelComposer { } } - fn extract_def_list(&self) -> Vec>> { + pub fn extract_def_list(&self) -> Vec>> { self.definition_ast_list.iter().map(|(def, ..)| def.clone()).collect_vec() } diff --git a/nac3core/src/toplevel/mod.rs b/nac3core/src/toplevel/mod.rs index f6fc15b0..bcf1e626 100644 --- a/nac3core/src/toplevel/mod.rs +++ b/nac3core/src/toplevel/mod.rs @@ -24,7 +24,7 @@ pub struct DefinitionId(pub usize); pub mod composer; pub mod helper; -mod type_annotation; +pub mod type_annotation; use composer::*; use type_annotation::*; #[cfg(test)] diff --git a/nac3core/src/toplevel/type_annotation.rs b/nac3core/src/toplevel/type_annotation.rs index 307d8794..c4e218fe 100644 --- a/nac3core/src/toplevel/type_annotation.rs +++ b/nac3core/src/toplevel/type_annotation.rs @@ -1,7 +1,6 @@ use std::cell::RefCell; use crate::typecheck::typedef::TypeVarMeta; - use super::*; #[derive(Clone, Debug)] @@ -49,54 +48,121 @@ pub fn parse_ast_to_type_annotation_kinds( primitives: &PrimitiveStore, expr: &ast::Expr, // the key stores the type_var of this topleveldef::class, we only need this field here - mut locked: HashMap>, + locked: HashMap>, ) -> Result { - match &expr.node { - ast::ExprKind::Name { id, .. } => { - if id == &"int32".into() { - Ok(TypeAnnotation::Primitive(primitives.int32)) - } else if id == &"int64".into() { - Ok(TypeAnnotation::Primitive(primitives.int64)) - } else if id == &"float".into() { - Ok(TypeAnnotation::Primitive(primitives.float)) - } else if id == &"bool".into() { - Ok(TypeAnnotation::Primitive(primitives.bool)) - } else if id == &"None".into() { - Ok(TypeAnnotation::Primitive(primitives.none)) - } else if id == &"str".into() { - Ok(TypeAnnotation::Primitive(primitives.str)) - } else if let Some(obj_id) = resolver.get_identifier_def(*id) { - let type_vars = { - let def_read = top_level_defs[obj_id.0].try_read(); - if let Some(def_read) = def_read { - if let TopLevelDef::Class { type_vars, .. } = &*def_read { - type_vars.clone() - } else { - return Err("function cannot be used as a type".into()); - } + let name_handle = |id: &StrRef, unifier: &mut Unifier, locked: HashMap>| { + if id == &"int32".into() { + Ok(TypeAnnotation::Primitive(primitives.int32)) + } else if id == &"int64".into() { + Ok(TypeAnnotation::Primitive(primitives.int64)) + } else if id == &"float".into() { + Ok(TypeAnnotation::Primitive(primitives.float)) + } else if id == &"bool".into() { + Ok(TypeAnnotation::Primitive(primitives.bool)) + } else if id == &"None".into() { + Ok(TypeAnnotation::Primitive(primitives.none)) + } else if id == &"str".into() { + Ok(TypeAnnotation::Primitive(primitives.str)) + } else if let Some(obj_id) = resolver.get_identifier_def(*id) { + let type_vars = { + let def_read = top_level_defs[obj_id.0].try_read(); + if let Some(def_read) = def_read { + if let TopLevelDef::Class { type_vars, .. } = &*def_read { + type_vars.clone() } else { - locked.get(&obj_id).unwrap().clone() + return Err("function cannot be used as a type".into()); } - }; - // check param number here - if !type_vars.is_empty() { - return Err(format!( - "expect {} type variable parameter but got 0", - type_vars.len() - )); - } - Ok(TypeAnnotation::CustomClass { id: obj_id, params: vec![] }) - } else if let Some(ty) = resolver.get_symbol_type(unifier, top_level_defs, primitives, *id) { - if let TypeEnum::TVar { .. } = unifier.get_ty(ty).as_ref() { - Ok(TypeAnnotation::TypeVar(ty)) } else { - Err("not a type variable identifier".into()) + locked.get(&obj_id).unwrap().clone() + } + }; + // check param number here + if !type_vars.is_empty() { + return Err(format!( + "expect {} type variable parameter but got 0", + type_vars.len() + )); + } + Ok(TypeAnnotation::CustomClass { id: obj_id, params: vec![] }) + } else if let Some(ty) = resolver.get_symbol_type(unifier, top_level_defs, primitives, *id) { + if let TypeEnum::TVar { .. } = unifier.get_ty(ty).as_ref() { + Ok(TypeAnnotation::TypeVar(ty)) + } else { + Err("not a type variable identifier".into()) + } + } else { + Err("name cannot be parsed as a type annotation".into()) + } + }; + + let class_name_handle = + |id: &StrRef, slice: &ast::Expr, unifier: &mut Unifier, mut locked: HashMap>| { + if vec!["virtual".into(), "Generic".into(), "list".into(), "tuple".into()] + .contains(id) + { + return Err("keywords cannot be class name".into()); + } + let obj_id = resolver + .get_identifier_def(*id) + .ok_or_else(|| "unknown class name".to_string())?; + let type_vars = { + let def_read = top_level_defs[obj_id.0].try_read(); + if let Some(def_read) = def_read { + if let TopLevelDef::Class { type_vars, .. } = &*def_read { + type_vars.clone() + } else { + unreachable!("must be class here") } } else { - Err("name cannot be parsed as a type annotation".into()) + locked.get(&obj_id).unwrap().clone() } - } - + }; + // we do not check whether the application of type variables are compatible here + let param_type_infos = { + let params_ast = if let ast::ExprKind::Tuple { elts, .. } = &slice.node { + elts.iter().collect_vec() + } else { + vec![slice] + }; + if type_vars.len() != params_ast.len() { + return Err(format!( + "expect {} type parameters but got {}", + type_vars.len(), + params_ast.len() + )); + } + let result = params_ast + .into_iter() + .map(|x| { + parse_ast_to_type_annotation_kinds( + resolver, + top_level_defs, + unifier, + primitives, + x, + { + locked.insert(obj_id, type_vars.clone()); + locked.clone() + }, + ) + }) + .collect::, _>>()?; + // make sure the result do not contain any type vars + let no_type_var = result + .iter() + .all(|x| get_type_var_contained_in_type_annotation(x).is_empty()); + if no_type_var { + result + } else { + return Err("application of type vars to generic class \ + is not currently supported" + .into()); + } + }; + Ok(TypeAnnotation::CustomClass { id: obj_id, params: param_type_infos }) + }; + match &expr.node { + ast::ExprKind::Name { id, .. } => name_handle(id, unifier, locked), // virtual ast::ExprKind::Subscript { value, slice, .. } if { @@ -163,71 +229,7 @@ pub fn parse_ast_to_type_annotation_kinds( // custom class ast::ExprKind::Subscript { value, slice, .. } => { if let ast::ExprKind::Name { id, .. } = &value.node { - if vec!["virtual".into(), "Generic".into(), "list".into(), "tuple".into()] - .contains(id) - { - return Err("keywords cannot be class name".into()); - } - let obj_id = resolver - .get_identifier_def(*id) - .ok_or_else(|| "unknown class name".to_string())?; - let type_vars = { - let def_read = top_level_defs[obj_id.0].try_read(); - if let Some(def_read) = def_read { - if let TopLevelDef::Class { type_vars, .. } = &*def_read { - type_vars.clone() - } else { - unreachable!("must be class here") - } - } else { - locked.get(&obj_id).unwrap().clone() - } - }; - // we do not check whether the application of type variables are compatible here - let param_type_infos = { - let params_ast = if let ast::ExprKind::Tuple { elts, .. } = &slice.node { - elts.iter().collect_vec() - } else { - vec![slice.as_ref()] - }; - if type_vars.len() != params_ast.len() { - return Err(format!( - "expect {} type parameters but got {}", - type_vars.len(), - params_ast.len() - )); - } - let result = params_ast - .into_iter() - .map(|x| { - parse_ast_to_type_annotation_kinds( - resolver, - top_level_defs, - unifier, - primitives, - x, - { - locked.insert(obj_id, type_vars.clone()); - locked.clone() - }, - ) - }) - .collect::, _>>()?; - - // make sure the result do not contain any type vars - let no_type_var = result - .iter() - .all(|x| get_type_var_contained_in_type_annotation(x).is_empty()); - if no_type_var { - result - } else { - return Err("application of type vars to generic class \ - is not currently supported" - .into()); - } - }; - - Ok(TypeAnnotation::CustomClass { id: obj_id, params: param_type_infos }) + class_name_handle(id, slice, unifier, locked) } else { Err("unsupported expression type for class name".into()) } @@ -370,13 +372,7 @@ pub fn get_type_from_type_annotation_kinds( /// But note that here we do not make a duplication of `T`, `V`, we direclty /// use them as they are in the TopLevelDef::Class since those in the /// TopLevelDef::Class.type_vars will be substitute later when seeing applications/instantiations -/// the Type of their fields and methods will also be subst when application/instantiation \ -/// \ -/// Note this implicit self type is different with seeing `A[T, V]` explicitly outside -/// the class def ast body, where it is a new instantiation of the generic class `A`, -/// but equivalent to seeing `A[T, V]` inside the class def body ast, where although we -/// create copies of `T` and `V`, we will find them out as occured type vars in the analyze_class() -/// and unify them with the class generic `T`, `V` +/// the Type of their fields and methods will also be subst when application/instantiation pub fn make_self_type_annotation(type_vars: &[Type], object_id: DefinitionId) -> TypeAnnotation { TypeAnnotation::CustomClass { id: object_id, diff --git a/nac3standalone/src/main.rs b/nac3standalone/src/main.rs index 85b39325..238511c2 100644 --- a/nac3standalone/src/main.rs +++ b/nac3standalone/src/main.rs @@ -3,8 +3,9 @@ use inkwell::{ targets::*, OptimizationLevel, }; -use nac3core::typecheck::type_inferencer::PrimitiveStore; +use nac3core::typecheck::{type_inferencer::PrimitiveStore, typedef::{Type, Unifier}}; use nac3parser::{ast::{Expr, ExprKind, StmtKind}, parser}; +use parking_lot::RwLock; use std::{borrow::Borrow, env}; use std::fs; use std::{collections::HashMap, path::Path, sync::Arc, time::SystemTime}; @@ -15,7 +16,11 @@ use nac3core::{ WorkerRegistry, }, symbol_resolver::SymbolResolver, - toplevel::{composer::TopLevelComposer, TopLevelDef, helper::parse_parameter_default_value}, + toplevel::{ + composer::TopLevelComposer, + TopLevelDef, helper::parse_parameter_default_value, + type_annotation::*, + }, typecheck::typedef::FunSignature, }; @@ -68,25 +73,84 @@ fn main() { for stmt in parser_result.into_iter() { if let StmtKind::Assign { targets, value, .. } = &stmt.node { + fn handle_typevar_definition( + var: &Expr, + resolver: &(dyn SymbolResolver + Send + Sync), + def_list: &[Arc>], + unifier: &mut Unifier, + primitives: &PrimitiveStore, + ) -> Result { + if let ExprKind::Call { func, args, .. } = &var.node { + if matches!(&func.node, ExprKind::Name { id, .. } if id == &"TypeVar".into()) { + let constraints = args + .iter() + .skip(1) + .map(|x| -> Result { + let ty = parse_ast_to_type_annotation_kinds( + resolver, + def_list, + unifier, + primitives, + x, + Default::default(), + )?; + get_type_from_type_annotation_kinds(def_list, unifier, primitives, &ty) + }) + .collect::, _>>()?; + Ok(unifier.get_fresh_var_with_range(&constraints).0) + } else { + Err(format!("expression {:?} cannot be handled as a TypeVar in global scope", var)) + } + } else { + Err(format!("expression {:?} cannot be handled as a TypeVar in global scope", var)) + } + } + fn handle_assignment_pattern( targets: &[Expr], value: &Expr, resolver: &(dyn SymbolResolver + Send + Sync), internal_resolver: &ResolverInternal, + def_list: &[Arc>], + unifier: &mut Unifier, + primitives: &PrimitiveStore, ) -> Result<(), String> { if targets.len() == 1 { match &targets[0].node { ExprKind::Name { id, .. } => { - let val = parse_parameter_default_value(value.borrow(), resolver)?; - internal_resolver.add_module_global(*id, val); - Ok(()) + if let Ok(var) = handle_typevar_definition( + value.borrow(), + resolver, + def_list, + unifier, + primitives, + ) { + internal_resolver.add_id_type(*id, var); + Ok(()) + } else if let Ok(val) = parse_parameter_default_value(value.borrow(), resolver) { + internal_resolver.add_module_global(*id, val); + Ok(()) + } else { + Err(format!("fails to evaluate this expression `{:?}` as a constant or TypeVar at {}", + targets[0].node, + targets[0].location, + )) + } } ExprKind::List { elts, .. } | ExprKind::Tuple { elts, .. } => { - handle_assignment_pattern(elts, value, resolver, internal_resolver)?; + handle_assignment_pattern( + elts, + value, + resolver, + internal_resolver, + def_list, + unifier, + primitives + )?; Ok(()) } - _ => unreachable!("cannot be assigned") + _ => Err(format!("assignment to {:?} is not supported at {}", targets[0], targets[0].location)) } } else { match &value.node { @@ -105,7 +169,10 @@ fn main() { std::slice::from_ref(tar), val, resolver, - internal_resolver + internal_resolver, + def_list, + unifier, + primitives )?; } Ok(()) @@ -115,7 +182,19 @@ fn main() { } } } - if let Err(err) = handle_assignment_pattern(targets, value, resolver.as_ref(), internal_resolver.as_ref()) { + + let def_list = composer.extract_def_list(); + let unifier = &mut composer.unifier; + let primitives = &composer.primitives_ty; + if let Err(err) = handle_assignment_pattern( + targets, + value, + resolver.as_ref(), + internal_resolver.as_ref(), + &def_list, + unifier, + primitives, + ) { eprintln!("{}", err); return; } From 142f82f987b098309d4df2c922fa6ba455598fbe Mon Sep 17 00:00:00 2001 From: Sebastien Bourdeauducq Date: Wed, 1 Dec 2021 22:48:06 +0800 Subject: [PATCH 08/54] remove debug prints --- nac3artiq/src/symbol_resolver.rs | 1 - nac3core/src/typecheck/function_check.rs | 1 - nac3core/src/typecheck/typedef/mod.rs | 1 - 3 files changed, 3 deletions(-) diff --git a/nac3artiq/src/symbol_resolver.rs b/nac3artiq/src/symbol_resolver.rs index f36c146f..92baa592 100644 --- a/nac3artiq/src/symbol_resolver.rs +++ b/nac3artiq/src/symbol_resolver.rs @@ -195,7 +195,6 @@ impl InnerResolver { if let TypeEnum::TVar { id, .. } = &*unifier.get_ty(*x) { (*id, *x) } else { - println!("{}", unifier.default_stringify(*x)); unreachable!() } }).collect() diff --git a/nac3core/src/typecheck/function_check.rs b/nac3core/src/typecheck/function_check.rs index 8b432393..65c810e0 100644 --- a/nac3core/src/typecheck/function_check.rs +++ b/nac3core/src/typecheck/function_check.rs @@ -167,7 +167,6 @@ impl<'a> Inferencer<'a> { } ExprKind::Constant { .. } => {} _ => { - println!("{:?}", expr.node); unimplemented!() } } diff --git a/nac3core/src/typecheck/typedef/mod.rs b/nac3core/src/typecheck/typedef/mod.rs index 0d194b26..8eb62767 100644 --- a/nac3core/src/typecheck/typedef/mod.rs +++ b/nac3core/src/typecheck/typedef/mod.rs @@ -855,7 +855,6 @@ impl Unifier { } } _ => { - println!("{}", ty.get_type_name()); unreachable!("{} not expected", ty.get_type_name()) } } From c683958e4a5ff52bcd58f8d3fcac3384fdf76761 Mon Sep 17 00:00:00 2001 From: Sebastien Bourdeauducq Date: Wed, 1 Dec 2021 22:49:20 +0800 Subject: [PATCH 09/54] nac3artiq: clarify comment about virtual class --- nac3artiq/demo/min_artiq.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/nac3artiq/demo/min_artiq.py b/nac3artiq/demo/min_artiq.py index fcb59ce9..354d2a2a 100644 --- a/nac3artiq/demo/min_artiq.py +++ b/nac3artiq/demo/min_artiq.py @@ -17,8 +17,7 @@ T = TypeVar('T') class KernelInvariant(Generic[T]): pass -# place the `virtual` class infront of the construct of NAC3 object to ensure the -# virtual class is known during the initializing of NAC3 object +# The virtual class must exist before nac3artiq.NAC3 is created. class virtual(Generic[T]): pass From a91b2d602cc4609d4e269df948220a044b35a91d Mon Sep 17 00:00:00 2001 From: Sebastien Bourdeauducq Date: Wed, 1 Dec 2021 22:49:43 +0800 Subject: [PATCH 10/54] flake: switch to nixos- branch --- flake.lock | 8 ++++---- flake.nix | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/flake.lock b/flake.lock index b7922587..70d2b3b8 100644 --- a/flake.lock +++ b/flake.lock @@ -2,16 +2,16 @@ "nodes": { "nixpkgs": { "locked": { - "lastModified": 1637636156, - "narHash": "sha256-E2ym4Vcpqu9JYoQDXJZR48gVD+LPPbaCoYveIk7Xu3Y=", + "lastModified": 1638279546, + "narHash": "sha256-1KCwN7twjp1dBdp0jPgVdYFztDkCR8+roo0B34J9oBY=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "b026e1cf87a108dd06fe521f224fdc72fd0b013d", + "rev": "96b4157790fc96e70d6e6c115e3f34bba7be490f", "type": "github" }, "original": { "owner": "NixOS", - "ref": "release-21.11", + "ref": "nixos-21.11", "repo": "nixpkgs", "type": "github" } diff --git a/flake.nix b/flake.nix index 0e89187f..de5c2871 100644 --- a/flake.nix +++ b/flake.nix @@ -1,7 +1,7 @@ { description = "The third-generation ARTIQ compiler"; - inputs.nixpkgs.url = github:NixOS/nixpkgs/release-21.11; + inputs.nixpkgs.url = github:NixOS/nixpkgs/nixos-21.11; outputs = { self, nixpkgs }: let From 1e6848ab92ef4ccf5eb14f05151fe05c90a0d3cf Mon Sep 17 00:00:00 2001 From: ychenfo Date: Thu, 2 Dec 2021 01:02:42 +0800 Subject: [PATCH 11/54] nac3core: distinguish i64 and i32 in bool conversion --- nac3core/src/toplevel/composer.rs | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/nac3core/src/toplevel/composer.rs b/nac3core/src/toplevel/composer.rs index 50c5344e..137f1871 100644 --- a/nac3core/src/toplevel/composer.rs +++ b/nac3core/src/toplevel/composer.rs @@ -414,7 +414,14 @@ impl TopLevelComposer { let arg = args[0].1; if ctx.unifier.unioned(arg_ty, boolean) { Some(arg) - } else if ctx.unifier.unioned(arg_ty, int32) || ctx.unifier.unioned(arg_ty, int64) { + } else if ctx.unifier.unioned(arg_ty, int32) { + Some(ctx.builder.build_int_compare( + IntPredicate::NE, + ctx.ctx.i32_type().const_zero(), + arg.into_int_value(), + "bool", + ).into()) + } else if ctx.unifier.unioned(arg_ty, int64) { Some(ctx.builder.build_int_compare( IntPredicate::NE, ctx.ctx.i64_type().const_zero(), From 1c31aa6e8edad5fb5dc2786c9399f5c58cf10ff1 Mon Sep 17 00:00:00 2001 From: Sebastien Bourdeauducq Date: Thu, 2 Dec 2021 10:45:46 +0800 Subject: [PATCH 12/54] consistent naming --- nac3core/src/toplevel/builtins.rs | 2 +- nac3core/src/toplevel/composer.rs | 40 +++++++++++++++---------------- nac3core/src/toplevel/test.rs | 6 ++--- 3 files changed, 24 insertions(+), 24 deletions(-) diff --git a/nac3core/src/toplevel/builtins.rs b/nac3core/src/toplevel/builtins.rs index d200aa12..94b598bc 100644 --- a/nac3core/src/toplevel/builtins.rs +++ b/nac3core/src/toplevel/builtins.rs @@ -8,7 +8,7 @@ type BuiltinInfo = ( &'static [&'static str] ); -pub fn get_built_ins(primitives: &mut (PrimitiveStore, Unifier)) -> BuiltinInfo { +pub fn get_builtins(primitives: &mut (PrimitiveStore, Unifier)) -> BuiltinInfo { let int32 = primitives.0.int32; let int64 = primitives.0.int64; let float = primitives.0.float; diff --git a/nac3core/src/toplevel/composer.rs b/nac3core/src/toplevel/composer.rs index c5367845..02b800da 100644 --- a/nac3core/src/toplevel/composer.rs +++ b/nac3core/src/toplevel/composer.rs @@ -24,7 +24,7 @@ pub struct TopLevelComposer { // get the class def id of a class method pub method_class: HashMap, // number of built-in function and classes in the definition list, later skip - pub built_in_num: usize, + pub builtin_num: usize, } impl Default for TopLevelComposer { @@ -40,7 +40,7 @@ impl TopLevelComposer { builtins: Vec<(StrRef, FunSignature, Arc)>, ) -> (Self, HashMap, HashMap) { let mut primitives = Self::make_primitives(); - let (mut definition_ast_list, builtin_name_list) = builtins::get_built_ins(&mut primitives); + let (mut definition_ast_list, builtin_name_list) = builtins::get_builtins(&mut primitives); let primitives_ty = primitives.0; let mut unifier = primitives.1; let mut keyword_list: HashSet = HashSet::from_iter(vec![ @@ -63,8 +63,8 @@ impl TopLevelComposer { let defined_names: HashSet = Default::default(); let method_class: HashMap = Default::default(); - let mut built_in_id: HashMap = Default::default(); - let mut built_in_ty: HashMap = Default::default(); + let mut builtin_id: HashMap = Default::default(); + let mut builtin_ty: HashMap = Default::default(); for (id, name) in builtin_name_list.iter().rev().enumerate() { let name = (**name).into(); @@ -72,8 +72,8 @@ impl TopLevelComposer { let def = definition_ast_list[id].0.read(); if let TopLevelDef::Function { simple_name, signature, .. } = &*def { assert!(name == *simple_name); - built_in_ty.insert(name, *signature); - built_in_id.insert(name, DefinitionId(id)); + builtin_ty.insert(name, *signature); + builtin_id.insert(name, DefinitionId(id)); } else { unreachable!() } @@ -81,8 +81,8 @@ impl TopLevelComposer { for (name, sig, codegen_callback) in builtins { let fun_sig = unifier.add_ty(TypeEnum::TFunc(RefCell::new(sig))); - built_in_ty.insert(name, fun_sig); - built_in_id.insert(name, DefinitionId(definition_ast_list.len())); + builtin_ty.insert(name, fun_sig); + builtin_id.insert(name, DefinitionId(definition_ast_list.len())); definition_ast_list.push(( Arc::new(RwLock::new(TopLevelDef::Function { name: name.into(), @@ -101,7 +101,7 @@ impl TopLevelComposer { ( TopLevelComposer { - built_in_num: definition_ast_list.len(), + builtin_num: definition_ast_list.len(), definition_ast_list, primitives_ty, unifier, @@ -109,8 +109,8 @@ impl TopLevelComposer { defined_names, method_class, }, - built_in_id, - built_in_ty, + builtin_id, + builtin_ty, ) } @@ -320,7 +320,7 @@ impl TopLevelComposer { let primitives_store = &self.primitives_ty; // skip 5 to skip analyzing the primitives - for (class_def, class_ast) in def_list.iter().skip(self.built_in_num) { + for (class_def, class_ast) in def_list.iter().skip(self.builtin_num) { // only deal with class def here let mut class_def = class_def.write(); let (class_bases_ast, class_def_type_vars, class_resolver) = { @@ -426,7 +426,7 @@ impl TopLevelComposer { // first, only push direct parent into the list // skip 5 to skip analyzing the primitives - for (class_def, class_ast) in self.definition_ast_list.iter_mut().skip(self.built_in_num) { + for (class_def, class_ast) in self.definition_ast_list.iter_mut().skip(self.builtin_num) { let mut class_def = class_def.write(); let (class_def_id, class_bases, class_ancestors, class_resolver, class_type_vars) = { if let TopLevelDef::Class { ancestors, resolver, object_id, type_vars, .. } = @@ -490,7 +490,7 @@ impl TopLevelComposer { // second, get all ancestors let mut ancestors_store: HashMap> = Default::default(); // skip 5 to skip analyzing the primitives - for (class_def, _) in self.definition_ast_list.iter().skip(self.built_in_num) { + for (class_def, _) in self.definition_ast_list.iter().skip(self.builtin_num) { let class_def = class_def.read(); let (class_ancestors, class_id) = { if let TopLevelDef::Class { ancestors, object_id, .. } = class_def.deref() { @@ -512,7 +512,7 @@ impl TopLevelComposer { // insert the ancestors to the def list // skip 5 to skip analyzing the primitives - for (class_def, _) in self.definition_ast_list.iter_mut().skip(self.built_in_num) { + for (class_def, _) in self.definition_ast_list.iter_mut().skip(self.builtin_num) { let mut class_def = class_def.write(); let (class_ancestors, class_id, class_type_vars) = { if let TopLevelDef::Class { ancestors, object_id, type_vars, .. } = @@ -545,7 +545,7 @@ impl TopLevelComposer { let mut type_var_to_concrete_def: HashMap = HashMap::new(); // skip 5 to skip analyzing the primitives - for (class_def, class_ast) in def_ast_list.iter().skip(self.built_in_num) { + for (class_def, class_ast) in def_ast_list.iter().skip(self.builtin_num) { if matches!(&*class_def.read(), TopLevelDef::Class { .. }) { Self::analyze_single_class_methods_fields( class_def.clone(), @@ -566,7 +566,7 @@ impl TopLevelComposer { loop { let mut finished = true; - for (class_def, _) in def_ast_list.iter().skip(self.built_in_num) { + for (class_def, _) in def_ast_list.iter().skip(self.builtin_num) { let mut class_def = class_def.write(); if let TopLevelDef::Class { ancestors, .. } = class_def.deref() { // if the length of the ancestor is equal to the current depth @@ -625,7 +625,7 @@ impl TopLevelComposer { let primitives_store = &self.primitives_ty; // skip 5 to skip analyzing the primitives - for (function_def, function_ast) in def_list.iter().skip(self.built_in_num) { + for (function_def, function_ast) in def_list.iter().skip(self.builtin_num) { let mut function_def = function_def.write(); let function_def = function_def.deref_mut(); let function_ast = if let Some(x) = function_ast.as_ref() { @@ -1235,7 +1235,7 @@ impl TopLevelComposer { fn analyze_function_instance(&mut self) -> Result<(), String> { // first get the class contructor type correct for the following type check in function body // also do class field instantiation check - for (def, ast) in self.definition_ast_list.iter().skip(self.built_in_num) { + for (def, ast) in self.definition_ast_list.iter().skip(self.builtin_num) { let class_def = def.read(); if let TopLevelDef::Class { constructor, @@ -1306,7 +1306,7 @@ impl TopLevelComposer { let ctx = Arc::new(self.make_top_level_context()); // type inference inside function body - for (id, (def, ast)) in self.definition_ast_list.iter().enumerate().skip(self.built_in_num) + for (id, (def, ast)) in self.definition_ast_list.iter().enumerate().skip(self.builtin_num) { let mut function_def = def.write(); if let TopLevelDef::Function { diff --git a/nac3core/src/toplevel/test.rs b/nac3core/src/toplevel/test.rs index d1135c7d..81b3e62e 100644 --- a/nac3core/src/toplevel/test.rs +++ b/nac3core/src/toplevel/test.rs @@ -162,7 +162,7 @@ fn test_simple_function_analyze(source: Vec<&str>, tys: Vec<&str>, names: Vec<&s composer.start_analysis(true).unwrap(); - for (i, (def, _)) in composer.definition_ast_list.iter().skip(composer.built_in_num).enumerate() + for (i, (def, _)) in composer.definition_ast_list.iter().skip(composer.builtin_num).enumerate() { let def = &*def.read(); if let TopLevelDef::Function { signature, name, .. } = def { @@ -530,7 +530,7 @@ fn test_analyze(source: Vec<&str>, res: Vec<&str>) { } else { // skip 5 to skip primitives let mut res_vec: Vec = Vec::new(); - for (def, _) in composer.definition_ast_list.iter().skip(composer.built_in_num) { + for (def, _) in composer.definition_ast_list.iter().skip(composer.builtin_num) { let def = &*def.read(); res_vec.push(format!("{}\n", def.to_string(composer.unifier.borrow_mut()))); } @@ -715,7 +715,7 @@ fn test_inference(source: Vec<&str>, res: Vec<&str>) { // skip 5 to skip primitives let mut stringify_folder = TypeToStringFolder { unifier: &mut composer.unifier }; for (_i, (def, _)) in - composer.definition_ast_list.iter().skip(composer.built_in_num).enumerate() + composer.definition_ast_list.iter().skip(composer.builtin_num).enumerate() { let def = &*def.read(); From 8a46032f4cf5d036aeb36c6691d149a2c5e06bb4 Mon Sep 17 00:00:00 2001 From: Sebastien Bourdeauducq Date: Thu, 2 Dec 2021 18:46:04 +0800 Subject: [PATCH 13/54] flake: unbreak llvm-config for cross-compilation of static libs --- flake.nix | 2 +- llvm-unbreak-static-cross.diff | 44 ++++++++++++++++++++++++++++++++++ 2 files changed, 45 insertions(+), 1 deletion(-) create mode 100644 llvm-unbreak-static-cross.diff diff --git a/flake.nix b/flake.nix index 041e7133..a88b7a84 100644 --- a/flake.nix +++ b/flake.nix @@ -10,7 +10,7 @@ nixpkgs-patched = pkgs-orig.applyPatches { name = "nixpkgs"; src = nixpkgs; - patches = [ ./llvm-future-riscv-abi.diff ./llvm-restrict-targets.diff ./llvm-mingw-crosscompile.diff ]; + patches = [ ./llvm-future-riscv-abi.diff ./llvm-restrict-targets.diff ./llvm-mingw-crosscompile.diff ./llvm-unbreak-static-cross.diff ]; }; pkgs = import nixpkgs-patched { system = "x86_64-linux"; }; pkgs-mingw = import nixpkgs-patched { system = "x86_64-linux"; crossSystem = { config = "x86_64-w64-mingw32"; libc = "msvcrt"; }; }; diff --git a/llvm-unbreak-static-cross.diff b/llvm-unbreak-static-cross.diff new file mode 100644 index 00000000..594731de --- /dev/null +++ b/llvm-unbreak-static-cross.diff @@ -0,0 +1,44 @@ +diff --git a/pkgs/development/compilers/llvm/12/llvm/default.nix b/pkgs/development/compilers/llvm/12/llvm/default.nix +index 30a1a7a16df..bb5676b9d48 100644 +--- a/pkgs/development/compilers/llvm/12/llvm/default.nix ++++ b/pkgs/development/compilers/llvm/12/llvm/default.nix +@@ -74,7 +74,7 @@ in stdenv.mkDerivation (rec { + --replace 'set(_install_rpath "@loader_path/../''${CMAKE_INSTALL_LIBDIR}''${LLVM_LIBDIR_SUFFIX}" ''${extra_libdir})' "" + '' + # Patch llvm-config to return correct library path based on --link-{shared,static}. +- + optionalString (enableSharedLibraries) '' ++ + '' + substitute '${./outputs.patch}' ./outputs.patch --subst-var lib + patch -p1 < ./outputs.patch + '' + '' +diff --git a/pkgs/development/compilers/llvm/12/llvm/outputs.patch b/pkgs/development/compilers/llvm/12/llvm/outputs.patch +index 40096fa3497..878460e05b8 100644 +--- a/pkgs/development/compilers/llvm/12/llvm/outputs.patch ++++ b/pkgs/development/compilers/llvm/12/llvm/outputs.patch +@@ -2,23 +2,13 @@ diff --git a/tools/llvm-config/llvm-config.cpp b/tools/llvm-config/llvm-config.c + index 94d426b..37f7794 100644 + --- a/tools/llvm-config/llvm-config.cpp + +++ b/tools/llvm-config/llvm-config.cpp +-@@ -333,6 +333,21 @@ int main(int argc, char **argv) { ++@@ -333,6 +333,11 @@ int main(int argc, char **argv) { + ActiveIncludeOption = "-I" + ActiveIncludeDir; + } + +-+ /// Nix-specific multiple-output handling: override ActiveLibDir if --link-shared +++ /// Nix-specific multiple-output handling: override ActiveLibDir + + if (!IsInDevelopmentTree) { +-+ bool WantShared = true; +-+ for (int i = 1; i < argc; ++i) { +-+ StringRef Arg = argv[i]; +-+ if (Arg == "--link-shared") +-+ WantShared = true; +-+ else if (Arg == "--link-static") +-+ WantShared = false; // the last one wins +-+ } +-+ +-+ if (WantShared) +-+ ActiveLibDir = std::string("@lib@") + "/lib" + LLVM_LIBDIR_SUFFIX; +++ ActiveLibDir = std::string("@lib@") + "/lib" + LLVM_LIBDIR_SUFFIX; + + } + + + /// We only use `shared library` mode in cases where the static library form From a6275fbb57ed1e3850596c957169f593092cdb4e Mon Sep 17 00:00:00 2001 From: Sebastien Bourdeauducq Date: Thu, 2 Dec 2021 19:08:20 +0800 Subject: [PATCH 14/54] flake: add libffi on Windows --- flake.nix | 1 + 1 file changed, 1 insertion(+) diff --git a/flake.nix b/flake.nix index a88b7a84..8147ae76 100644 --- a/flake.nix +++ b/flake.nix @@ -68,6 +68,7 @@ name = "nac3artiq"; src = self; inherit cargoSha256; + buildInputs = [ pkgs-mingw.libffi ]; configurePhase = '' export PYO3_CONFIG_FILE=${pyo3-mingw-config} From aab43b1c07e98576001f6713fc16a0fe41323dd7 Mon Sep 17 00:00:00 2001 From: Sebastien Bourdeauducq Date: Thu, 2 Dec 2021 20:00:45 +0800 Subject: [PATCH 15/54] flake: unbreak Windows library link (WIP) --- flake.nix | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/flake.nix b/flake.nix index 8147ae76..a476cdb9 100644 --- a/flake.nix +++ b/flake.nix @@ -68,12 +68,19 @@ name = "nac3artiq"; src = self; inherit cargoSha256; - buildInputs = [ pkgs-mingw.libffi ]; + buildInputs = [ pkgs-mingw.libffi pkgs-mingw.zlib pkgs-mingw.windows.mcfgthreads ]; configurePhase = '' export PYO3_CONFIG_FILE=${pyo3-mingw-config} + mkdir llvm-cfg - ln -s ${pkgs-mingw.llvm_12.dev}/bin/llvm-config-native llvm-cfg/llvm-config + cat << EOF > llvm-cfg/llvm-config + #!${pkgs.bash}/bin/bash + set -e + # gross hack to work around llvm-config asking for the wrong system libraries + exec ${pkgs-mingw.llvm_12.dev}/bin/llvm-config-native \$@ | ${pkgs.gnused}/bin/sed s/-lrt\ -ldl/-lmcfgthread\ -lz/ + EOF + chmod +x llvm-cfg/llvm-config export PATH=$PATH:`pwd`/llvm-cfg ''; cargoBuildFlags = [ "--package" "nac3artiq" ]; From 998f49261d5a7d800d5bfdceae5064d60bf2fbba Mon Sep 17 00:00:00 2001 From: Sebastien Bourdeauducq Date: Thu, 2 Dec 2021 21:02:48 +0800 Subject: [PATCH 16/54] flake: fix Windows libs further --- flake.nix | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/flake.nix b/flake.nix index a476cdb9..f0fdc431 100644 --- a/flake.nix +++ b/flake.nix @@ -77,8 +77,9 @@ cat << EOF > llvm-cfg/llvm-config #!${pkgs.bash}/bin/bash set -e - # gross hack to work around llvm-config asking for the wrong system libraries - exec ${pkgs-mingw.llvm_12.dev}/bin/llvm-config-native \$@ | ${pkgs.gnused}/bin/sed s/-lrt\ -ldl/-lmcfgthread\ -lz/ + # Gross hack to work around llvm-config asking for the wrong system libraries. + # Also add some other libraries we need here. + exec ${pkgs-mingw.llvm_12.dev}/bin/llvm-config-native \$@ | ${pkgs.gnused}/bin/sed s/-lrt\ -ldl\ -lpthread\ -lm/-lmcfgthread\ -lz\ -luuid\ -lole32/ EOF chmod +x llvm-cfg/llvm-config export PATH=$PATH:`pwd`/llvm-cfg From c0f8d5c6022e8044d439407a20dfa0f7459ef6e0 Mon Sep 17 00:00:00 2001 From: Sebastien Bourdeauducq Date: Thu, 2 Dec 2021 22:01:19 +0800 Subject: [PATCH 17/54] flake: Windows libs working --- flake.nix | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/flake.nix b/flake.nix index f0fdc431..1a6c8d8d 100644 --- a/flake.nix +++ b/flake.nix @@ -68,7 +68,7 @@ name = "nac3artiq"; src = self; inherit cargoSha256; - buildInputs = [ pkgs-mingw.libffi pkgs-mingw.zlib pkgs-mingw.windows.mcfgthreads ]; + buildInputs = [ pkgs-mingw.libffi pkgs-mingw.zlib ]; configurePhase = '' export PYO3_CONFIG_FILE=${pyo3-mingw-config} @@ -78,20 +78,19 @@ #!${pkgs.bash}/bin/bash set -e # Gross hack to work around llvm-config asking for the wrong system libraries. - # Also add some other libraries we need here. - exec ${pkgs-mingw.llvm_12.dev}/bin/llvm-config-native \$@ | ${pkgs.gnused}/bin/sed s/-lrt\ -ldl\ -lpthread\ -lm/-lmcfgthread\ -lz\ -luuid\ -lole32/ + exec ${pkgs-mingw.llvm_12.dev}/bin/llvm-config-native \$@ | ${pkgs.gnused}/bin/sed s/-lrt\ -ldl\ -lpthread\ -lm// EOF chmod +x llvm-cfg/llvm-config export PATH=$PATH:`pwd`/llvm-cfg + + export CARGO_TARGET_X86_64_PC_WINDOWS_GNU_RUSTFLAGS="-C link-arg=-lz -C link-arg=-luuid -C link-arg=-lole32 -C link-arg=-lmcfgthread" ''; cargoBuildFlags = [ "--package" "nac3artiq" ]; doCheck = false; installPhase = '' - TARGET_DIR=$out/${pkgs.python3Packages.python.sitePackages} - mkdir -p $TARGET_DIR - #cp target/x86_64-unknown-linux-gnu/release/libnac3artiq.so $TARGET_DIR/nac3artiq.so - ls target + mkdir -p $out + cp target/x86_64-pc-windows-gnu/release/nac3artiq.dll $out ''; meta.platforms = ["x86_64-windows"]; } From 6315027a8bd6ac899d31dd0789c3b42ae14193ca Mon Sep 17 00:00:00 2001 From: Sebastien Bourdeauducq Date: Thu, 2 Dec 2021 22:24:23 +0800 Subject: [PATCH 18/54] flake: use *.pyd for Windows Python module --- flake.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flake.nix b/flake.nix index 1a6c8d8d..ebbe3437 100644 --- a/flake.nix +++ b/flake.nix @@ -90,7 +90,7 @@ installPhase = '' mkdir -p $out - cp target/x86_64-pc-windows-gnu/release/nac3artiq.dll $out + cp target/x86_64-pc-windows-gnu/release/nac3artiq.dll $out/nac3artiq.pyd ''; meta.platforms = ["x86_64-windows"]; } From 25fc9db66dbdb62032ec51909f556c8b3aa4d077 Mon Sep 17 00:00:00 2001 From: Sebastien Bourdeauducq Date: Thu, 2 Dec 2021 22:24:33 +0800 Subject: [PATCH 19/54] cargo: specify inkwell LLVM target explicitly Windows LLVM linking otherwise breaks on the non-existing targets. --- nac3artiq/Cargo.toml | 7 ++++++- nac3core/Cargo.toml | 7 ++++++- nac3standalone/Cargo.toml | 7 ++++++- 3 files changed, 18 insertions(+), 3 deletions(-) diff --git a/nac3artiq/Cargo.toml b/nac3artiq/Cargo.toml index 96161a9a..b527b92e 100644 --- a/nac3artiq/Cargo.toml +++ b/nac3artiq/Cargo.toml @@ -10,8 +10,13 @@ crate-type = ["cdylib"] [dependencies] pyo3 = { version = "0.14", features = ["extension-module"] } -inkwell = { git = "https://github.com/TheDan64/inkwell", branch = "master", features = ["llvm12-0"] } parking_lot = "0.11" tempfile = "3" nac3parser = { path = "../nac3parser" } nac3core = { path = "../nac3core" } + +[dependencies.inkwell] +git = "https://github.com/TheDan64/inkwell" +branch = "master" +default-features = false +features = ["llvm12-0", "target-x86", "target-arm", "target-riscv"] diff --git a/nac3core/Cargo.toml b/nac3core/Cargo.toml index 24d26b91..5aae208f 100644 --- a/nac3core/Cargo.toml +++ b/nac3core/Cargo.toml @@ -7,13 +7,18 @@ edition = "2018" [dependencies] num-bigint = "0.3" num-traits = "0.2" -inkwell = { git = "https://github.com/TheDan64/inkwell", branch = "master", features = ["llvm12-0"] } itertools = "0.10.1" crossbeam = "0.8.1" parking_lot = "0.11.1" rayon = "1.5.1" nac3parser = { path = "../nac3parser" } +[dependencies.inkwell] +git = "https://github.com/TheDan64/inkwell" +branch = "master" +default-features = false +features = ["llvm12-0", "target-x86", "target-arm", "target-riscv"] + [dev-dependencies] test-case = "1.2.0" indoc = "1.0" diff --git a/nac3standalone/Cargo.toml b/nac3standalone/Cargo.toml index df776f1a..41f33069 100644 --- a/nac3standalone/Cargo.toml +++ b/nac3standalone/Cargo.toml @@ -5,7 +5,12 @@ authors = ["M-Labs"] edition = "2018" [dependencies] -inkwell = { git = "https://github.com/TheDan64/inkwell", branch = "master", features = ["llvm12-0"] } parking_lot = "0.11.1" nac3parser = { path = "../nac3parser" } nac3core = { path = "../nac3core" } + +[dependencies.inkwell] +git = "https://github.com/TheDan64/inkwell" +branch = "master" +default-features = false +features = ["llvm12-0", "target-x86", "target-arm", "target-riscv"] From 22a509e7cea9e82168e5032a3ffe4af18708bd1b Mon Sep 17 00:00:00 2001 From: Sebastien Bourdeauducq Date: Thu, 2 Dec 2021 22:29:42 +0800 Subject: [PATCH 20/54] flake: add Hydra job for Windows build This is a proof-of-concept; it works but requires manual fiddling with DLLs (e.g. copy them from the Nix store into the Windows environment), and LLD is not available on Windows. --- flake.nix | 1 + 1 file changed, 1 insertion(+) diff --git a/flake.nix b/flake.nix index ebbe3437..0b8e61d8 100644 --- a/flake.nix +++ b/flake.nix @@ -115,6 +115,7 @@ hydraJobs = { inherit (packages.x86_64-linux) nac3artiq; + mingw-nac3artiq = packages.x86_64-w64-mingw32.nac3artiq; } // (pkgs.lib.foldr (a: b: {"${pkgs.lib.strings.getName a}" = a;} // b) {} devShell.x86_64-linux.buildInputs); }; From 26e60fca6e9f7837fa6fde4cb6f300284dd83e0e Mon Sep 17 00:00:00 2001 From: Sebastien Bourdeauducq Date: Thu, 2 Dec 2021 22:37:32 +0800 Subject: [PATCH 21/54] flake: cleanup tarball unpacking --- flake.nix | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/flake.nix b/flake.nix index 0b8e61d8..87e5ce6a 100644 --- a/flake.nix +++ b/flake.nix @@ -19,11 +19,17 @@ url = "https://mirror.msys2.org/mingw/mingw64/mingw-w64-x86_64-python-3.9.7-4-any.pkg.tar.zst"; sha256 = "0iwlgbk4b457yn9djwqswid55xhyyi35qymz1lfh42xwdpxdm47c"; }; - msys2-python = pkgs.runCommand "msys2-python" { buildInputs = [ pkgs.gnutar pkgs.zstd ]; } - '' - mkdir $out - tar xvf ${msys2-python-tar} -C $out - ''; + msys2-python = pkgs.stdenvNoCC.mkDerivation { + name = "msys2-python"; + src = msys2-python-tar; + buildInputs = [ pkgs.gnutar pkgs.zstd ]; + phases = [ "installPhase" ]; + installPhase = + '' + mkdir $out + tar xf $src -C $out + ''; + }; pyo3-mingw-config = pkgs.writeTextFile { name = "pyo3-mingw-config"; text = From b0eb7815dacea45e7fcdaeb69dbb1769ee118e4e Mon Sep 17 00:00:00 2001 From: Sebastien Bourdeauducq Date: Thu, 2 Dec 2021 22:37:41 +0800 Subject: [PATCH 22/54] flake: consistent naming --- flake.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flake.nix b/flake.nix index 87e5ce6a..865acad5 100644 --- a/flake.nix +++ b/flake.nix @@ -121,7 +121,7 @@ hydraJobs = { inherit (packages.x86_64-linux) nac3artiq; - mingw-nac3artiq = packages.x86_64-w64-mingw32.nac3artiq; + nac3artiq-mingw = packages.x86_64-w64-mingw32.nac3artiq; } // (pkgs.lib.foldr (a: b: {"${pkgs.lib.strings.getName a}" = a;} // b) {} devShell.x86_64-linux.buildInputs); }; From 0ae2aae645e9ee05fc67b0e82a8fe870cc6d6428 Mon Sep 17 00:00:00 2001 From: Sebastien Bourdeauducq Date: Thu, 2 Dec 2021 22:47:31 +0800 Subject: [PATCH 23/54] flake: publish zipfile with Windows Python module on Hydra --- flake.nix | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/flake.nix b/flake.nix index 865acad5..c72bb8ad 100644 --- a/flake.nix +++ b/flake.nix @@ -74,6 +74,7 @@ name = "nac3artiq"; src = self; inherit cargoSha256; + nativeBuildInputs = [ pkgs.zip ]; buildInputs = [ pkgs-mingw.libffi pkgs-mingw.zlib ]; configurePhase = '' @@ -95,8 +96,10 @@ doCheck = false; installPhase = '' - mkdir -p $out - cp target/x86_64-pc-windows-gnu/release/nac3artiq.dll $out/nac3artiq.pyd + mkdir -p $out $out/nix-support + ln -s target/x86_64-pc-windows-gnu/release/nac3artiq.dll nac3artiq.pyd + zip $out/nac3artiq.zip nac3artiq.pyd + echo file binary-dist $out/nac3artiq.zip >> $out/nix-support/hydra-build-products ''; meta.platforms = ["x86_64-windows"]; } From 8c05d8431d51176c41827b76bb1280c23ed09cb2 Mon Sep 17 00:00:00 2001 From: Sebastien Bourdeauducq Date: Fri, 3 Dec 2021 11:56:54 +0800 Subject: [PATCH 24/54] flake: use upstream nixpkgs patch https://github.com/NixOS/nixpkgs/pull/148367 --- llvm-unbreak-static-cross.diff | 431 ++++++++++++++++++++++++++++++++- 1 file changed, 429 insertions(+), 2 deletions(-) diff --git a/llvm-unbreak-static-cross.diff b/llvm-unbreak-static-cross.diff index 594731de..6ab65020 100644 --- a/llvm-unbreak-static-cross.diff +++ b/llvm-unbreak-static-cross.diff @@ -1,5 +1,93 @@ +diff --git a/pkgs/development/compilers/llvm/10/llvm/default.nix b/pkgs/development/compilers/llvm/10/llvm/default.nix +index 4e590dba4337d..413de2f868a3e 100644 +--- a/pkgs/development/compilers/llvm/10/llvm/default.nix ++++ b/pkgs/development/compilers/llvm/10/llvm/default.nix +@@ -81,7 +81,7 @@ in stdenv.mkDerivation (rec { + --replace 'set(_install_rpath "@loader_path/../''${CMAKE_INSTALL_LIBDIR}''${LLVM_LIBDIR_SUFFIX}" ''${extra_libdir})' "" + '' + # Patch llvm-config to return correct library path based on --link-{shared,static}. +- + optionalString (enableSharedLibraries) '' ++ + '' + substitute '${./outputs.patch}' ./outputs.patch --subst-var lib + patch -p1 < ./outputs.patch + '' + '' +diff --git a/pkgs/development/compilers/llvm/10/llvm/outputs.patch b/pkgs/development/compilers/llvm/10/llvm/outputs.patch +index 40096fa3497fd..878460e05b8af 100644 +--- a/pkgs/development/compilers/llvm/10/llvm/outputs.patch ++++ b/pkgs/development/compilers/llvm/10/llvm/outputs.patch +@@ -2,23 +2,13 @@ diff --git a/tools/llvm-config/llvm-config.cpp b/tools/llvm-config/llvm-config.c + index 94d426b..37f7794 100644 + --- a/tools/llvm-config/llvm-config.cpp + +++ b/tools/llvm-config/llvm-config.cpp +-@@ -333,6 +333,21 @@ int main(int argc, char **argv) { ++@@ -333,6 +333,11 @@ int main(int argc, char **argv) { + ActiveIncludeOption = "-I" + ActiveIncludeDir; + } + +-+ /// Nix-specific multiple-output handling: override ActiveLibDir if --link-shared +++ /// Nix-specific multiple-output handling: override ActiveLibDir + + if (!IsInDevelopmentTree) { +-+ bool WantShared = true; +-+ for (int i = 1; i < argc; ++i) { +-+ StringRef Arg = argv[i]; +-+ if (Arg == "--link-shared") +-+ WantShared = true; +-+ else if (Arg == "--link-static") +-+ WantShared = false; // the last one wins +-+ } +-+ +-+ if (WantShared) +-+ ActiveLibDir = std::string("@lib@") + "/lib" + LLVM_LIBDIR_SUFFIX; +++ ActiveLibDir = std::string("@lib@") + "/lib" + LLVM_LIBDIR_SUFFIX; + + } + + + /// We only use `shared library` mode in cases where the static library form +diff --git a/pkgs/development/compilers/llvm/11/llvm/default.nix b/pkgs/development/compilers/llvm/11/llvm/default.nix +index f0148850dbe8a..32981b9d4dde8 100644 +--- a/pkgs/development/compilers/llvm/11/llvm/default.nix ++++ b/pkgs/development/compilers/llvm/11/llvm/default.nix +@@ -73,7 +73,7 @@ in stdenv.mkDerivation (rec { + --replace 'set(_install_rpath "@loader_path/../''${CMAKE_INSTALL_LIBDIR}''${LLVM_LIBDIR_SUFFIX}" ''${extra_libdir})' "" + '' + # Patch llvm-config to return correct library path based on --link-{shared,static}. +- + optionalString (enableSharedLibraries) '' ++ + '' + substitute '${./outputs.patch}' ./outputs.patch --subst-var lib + patch -p1 < ./outputs.patch + '' + '' +diff --git a/pkgs/development/compilers/llvm/11/llvm/outputs.patch b/pkgs/development/compilers/llvm/11/llvm/outputs.patch +index 40096fa3497fd..878460e05b8af 100644 +--- a/pkgs/development/compilers/llvm/11/llvm/outputs.patch ++++ b/pkgs/development/compilers/llvm/11/llvm/outputs.patch +@@ -2,23 +2,13 @@ diff --git a/tools/llvm-config/llvm-config.cpp b/tools/llvm-config/llvm-config.c + index 94d426b..37f7794 100644 + --- a/tools/llvm-config/llvm-config.cpp + +++ b/tools/llvm-config/llvm-config.cpp +-@@ -333,6 +333,21 @@ int main(int argc, char **argv) { ++@@ -333,6 +333,11 @@ int main(int argc, char **argv) { + ActiveIncludeOption = "-I" + ActiveIncludeDir; + } + +-+ /// Nix-specific multiple-output handling: override ActiveLibDir if --link-shared +++ /// Nix-specific multiple-output handling: override ActiveLibDir + + if (!IsInDevelopmentTree) { +-+ bool WantShared = true; +-+ for (int i = 1; i < argc; ++i) { +-+ StringRef Arg = argv[i]; +-+ if (Arg == "--link-shared") +-+ WantShared = true; +-+ else if (Arg == "--link-static") +-+ WantShared = false; // the last one wins +-+ } +-+ +-+ if (WantShared) +-+ ActiveLibDir = std::string("@lib@") + "/lib" + LLVM_LIBDIR_SUFFIX; +++ ActiveLibDir = std::string("@lib@") + "/lib" + LLVM_LIBDIR_SUFFIX; + + } + + + /// We only use `shared library` mode in cases where the static library form diff --git a/pkgs/development/compilers/llvm/12/llvm/default.nix b/pkgs/development/compilers/llvm/12/llvm/default.nix -index 30a1a7a16df..bb5676b9d48 100644 +index 30a1a7a16df83..bb5676b9d48e4 100644 --- a/pkgs/development/compilers/llvm/12/llvm/default.nix +++ b/pkgs/development/compilers/llvm/12/llvm/default.nix @@ -74,7 +74,7 @@ in stdenv.mkDerivation (rec { @@ -12,7 +100,7 @@ index 30a1a7a16df..bb5676b9d48 100644 patch -p1 < ./outputs.patch '' + '' diff --git a/pkgs/development/compilers/llvm/12/llvm/outputs.patch b/pkgs/development/compilers/llvm/12/llvm/outputs.patch -index 40096fa3497..878460e05b8 100644 +index 40096fa3497fd..878460e05b8af 100644 --- a/pkgs/development/compilers/llvm/12/llvm/outputs.patch +++ b/pkgs/development/compilers/llvm/12/llvm/outputs.patch @@ -2,23 +2,13 @@ diff --git a/tools/llvm-config/llvm-config.cpp b/tools/llvm-config/llvm-config.c @@ -42,3 +130,342 @@ index 40096fa3497..878460e05b8 100644 + } + /// We only use `shared library` mode in cases where the static library form +diff --git a/pkgs/development/compilers/llvm/13/llvm/default.nix b/pkgs/development/compilers/llvm/13/llvm/default.nix +index 957f29e44994a..115b56396e8d8 100644 +--- a/pkgs/development/compilers/llvm/13/llvm/default.nix ++++ b/pkgs/development/compilers/llvm/13/llvm/default.nix +@@ -68,7 +68,7 @@ in stdenv.mkDerivation (rec { + --replace 'set(_install_rpath "@loader_path/../''${CMAKE_INSTALL_LIBDIR}''${LLVM_LIBDIR_SUFFIX}" ''${extra_libdir})' "" + '' + # Patch llvm-config to return correct library path based on --link-{shared,static}. +- + optionalString (enableSharedLibraries) '' ++ + '' + substitute '${./outputs.patch}' ./outputs.patch --subst-var lib + patch -p1 < ./outputs.patch + '' + '' +diff --git a/pkgs/development/compilers/llvm/13/llvm/outputs.patch b/pkgs/development/compilers/llvm/13/llvm/outputs.patch +index 40096fa3497fd..878460e05b8af 100644 +--- a/pkgs/development/compilers/llvm/13/llvm/outputs.patch ++++ b/pkgs/development/compilers/llvm/13/llvm/outputs.patch +@@ -2,23 +2,13 @@ diff --git a/tools/llvm-config/llvm-config.cpp b/tools/llvm-config/llvm-config.c + index 94d426b..37f7794 100644 + --- a/tools/llvm-config/llvm-config.cpp + +++ b/tools/llvm-config/llvm-config.cpp +-@@ -333,6 +333,21 @@ int main(int argc, char **argv) { ++@@ -333,6 +333,11 @@ int main(int argc, char **argv) { + ActiveIncludeOption = "-I" + ActiveIncludeDir; + } + +-+ /// Nix-specific multiple-output handling: override ActiveLibDir if --link-shared +++ /// Nix-specific multiple-output handling: override ActiveLibDir + + if (!IsInDevelopmentTree) { +-+ bool WantShared = true; +-+ for (int i = 1; i < argc; ++i) { +-+ StringRef Arg = argv[i]; +-+ if (Arg == "--link-shared") +-+ WantShared = true; +-+ else if (Arg == "--link-static") +-+ WantShared = false; // the last one wins +-+ } +-+ +-+ if (WantShared) +-+ ActiveLibDir = std::string("@lib@") + "/lib" + LLVM_LIBDIR_SUFFIX; +++ ActiveLibDir = std::string("@lib@") + "/lib" + LLVM_LIBDIR_SUFFIX; + + } + + + /// We only use `shared library` mode in cases where the static library form +diff --git a/pkgs/development/compilers/llvm/5/llvm/default.nix b/pkgs/development/compilers/llvm/5/llvm/default.nix +index 6388cd65fbf47..a4d2fec36193d 100644 +--- a/pkgs/development/compilers/llvm/5/llvm/default.nix ++++ b/pkgs/development/compilers/llvm/5/llvm/default.nix +@@ -74,7 +74,7 @@ stdenv.mkDerivation ({ + --replace 'set(_install_rpath "@loader_path/../''${CMAKE_INSTALL_LIBDIR}" ''${extra_libdir})' "" + '' + # Patch llvm-config to return correct library path based on --link-{shared,static}. +- + optionalString (enableSharedLibraries) '' ++ + '' + substitute '${./outputs.patch}' ./outputs.patch --subst-var lib + patch -p1 < ./outputs.patch + '' + '' +diff --git a/pkgs/development/compilers/llvm/5/llvm/outputs.patch b/pkgs/development/compilers/llvm/5/llvm/outputs.patch +index 40096fa3497fd..878460e05b8af 100644 +--- a/pkgs/development/compilers/llvm/5/llvm/outputs.patch ++++ b/pkgs/development/compilers/llvm/5/llvm/outputs.patch +@@ -2,23 +2,13 @@ diff --git a/tools/llvm-config/llvm-config.cpp b/tools/llvm-config/llvm-config.c + index 94d426b..37f7794 100644 + --- a/tools/llvm-config/llvm-config.cpp + +++ b/tools/llvm-config/llvm-config.cpp +-@@ -333,6 +333,21 @@ int main(int argc, char **argv) { ++@@ -333,6 +333,11 @@ int main(int argc, char **argv) { + ActiveIncludeOption = "-I" + ActiveIncludeDir; + } + +-+ /// Nix-specific multiple-output handling: override ActiveLibDir if --link-shared +++ /// Nix-specific multiple-output handling: override ActiveLibDir + + if (!IsInDevelopmentTree) { +-+ bool WantShared = true; +-+ for (int i = 1; i < argc; ++i) { +-+ StringRef Arg = argv[i]; +-+ if (Arg == "--link-shared") +-+ WantShared = true; +-+ else if (Arg == "--link-static") +-+ WantShared = false; // the last one wins +-+ } +-+ +-+ if (WantShared) +-+ ActiveLibDir = std::string("@lib@") + "/lib" + LLVM_LIBDIR_SUFFIX; +++ ActiveLibDir = std::string("@lib@") + "/lib" + LLVM_LIBDIR_SUFFIX; + + } + + + /// We only use `shared library` mode in cases where the static library form +diff --git a/pkgs/development/compilers/llvm/6/llvm/default.nix b/pkgs/development/compilers/llvm/6/llvm/default.nix +index 0907c89561717..1c010980048cd 100644 +--- a/pkgs/development/compilers/llvm/6/llvm/default.nix ++++ b/pkgs/development/compilers/llvm/6/llvm/default.nix +@@ -72,7 +72,7 @@ stdenv.mkDerivation ({ + --replace 'set(_install_rpath "@loader_path/../''${CMAKE_INSTALL_LIBDIR}" ''${extra_libdir})' "" + '' + # Patch llvm-config to return correct library path based on --link-{shared,static}. +- + optionalString (enableSharedLibraries) '' ++ + '' + substitute '${./outputs.patch}' ./outputs.patch --subst-var lib + patch -p1 < ./outputs.patch + '' + '' +diff --git a/pkgs/development/compilers/llvm/6/llvm/outputs.patch b/pkgs/development/compilers/llvm/6/llvm/outputs.patch +index 40096fa3497fd..878460e05b8af 100644 +--- a/pkgs/development/compilers/llvm/6/llvm/outputs.patch ++++ b/pkgs/development/compilers/llvm/6/llvm/outputs.patch +@@ -2,23 +2,13 @@ diff --git a/tools/llvm-config/llvm-config.cpp b/tools/llvm-config/llvm-config.c + index 94d426b..37f7794 100644 + --- a/tools/llvm-config/llvm-config.cpp + +++ b/tools/llvm-config/llvm-config.cpp +-@@ -333,6 +333,21 @@ int main(int argc, char **argv) { ++@@ -333,6 +333,11 @@ int main(int argc, char **argv) { + ActiveIncludeOption = "-I" + ActiveIncludeDir; + } + +-+ /// Nix-specific multiple-output handling: override ActiveLibDir if --link-shared +++ /// Nix-specific multiple-output handling: override ActiveLibDir + + if (!IsInDevelopmentTree) { +-+ bool WantShared = true; +-+ for (int i = 1; i < argc; ++i) { +-+ StringRef Arg = argv[i]; +-+ if (Arg == "--link-shared") +-+ WantShared = true; +-+ else if (Arg == "--link-static") +-+ WantShared = false; // the last one wins +-+ } +-+ +-+ if (WantShared) +-+ ActiveLibDir = std::string("@lib@") + "/lib" + LLVM_LIBDIR_SUFFIX; +++ ActiveLibDir = std::string("@lib@") + "/lib" + LLVM_LIBDIR_SUFFIX; + + } + + + /// We only use `shared library` mode in cases where the static library form +diff --git a/pkgs/development/compilers/llvm/7/llvm/default.nix b/pkgs/development/compilers/llvm/7/llvm/default.nix +index 4a9b4f5182056..ac6efaccb865e 100644 +--- a/pkgs/development/compilers/llvm/7/llvm/default.nix ++++ b/pkgs/development/compilers/llvm/7/llvm/default.nix +@@ -76,7 +76,7 @@ in stdenv.mkDerivation ({ + --replace 'set(_install_rpath "@loader_path/../''${CMAKE_INSTALL_LIBDIR}" ''${extra_libdir})' "" + '' + # Patch llvm-config to return correct library path based on --link-{shared,static}. +- + optionalString (enableSharedLibraries) '' ++ + '' + substitute '${./outputs.patch}' ./outputs.patch --subst-var lib + patch -p1 < ./outputs.patch + '' + '' +diff --git a/pkgs/development/compilers/llvm/7/llvm/outputs.patch b/pkgs/development/compilers/llvm/7/llvm/outputs.patch +index 40096fa3497fd..878460e05b8af 100644 +--- a/pkgs/development/compilers/llvm/7/llvm/outputs.patch ++++ b/pkgs/development/compilers/llvm/7/llvm/outputs.patch +@@ -2,23 +2,13 @@ diff --git a/tools/llvm-config/llvm-config.cpp b/tools/llvm-config/llvm-config.c + index 94d426b..37f7794 100644 + --- a/tools/llvm-config/llvm-config.cpp + +++ b/tools/llvm-config/llvm-config.cpp +-@@ -333,6 +333,21 @@ int main(int argc, char **argv) { ++@@ -333,6 +333,11 @@ int main(int argc, char **argv) { + ActiveIncludeOption = "-I" + ActiveIncludeDir; + } + +-+ /// Nix-specific multiple-output handling: override ActiveLibDir if --link-shared +++ /// Nix-specific multiple-output handling: override ActiveLibDir + + if (!IsInDevelopmentTree) { +-+ bool WantShared = true; +-+ for (int i = 1; i < argc; ++i) { +-+ StringRef Arg = argv[i]; +-+ if (Arg == "--link-shared") +-+ WantShared = true; +-+ else if (Arg == "--link-static") +-+ WantShared = false; // the last one wins +-+ } +-+ +-+ if (WantShared) +-+ ActiveLibDir = std::string("@lib@") + "/lib" + LLVM_LIBDIR_SUFFIX; +++ ActiveLibDir = std::string("@lib@") + "/lib" + LLVM_LIBDIR_SUFFIX; + + } + + + /// We only use `shared library` mode in cases where the static library form +diff --git a/pkgs/development/compilers/llvm/8/llvm/default.nix b/pkgs/development/compilers/llvm/8/llvm/default.nix +index 83b779b56d827..48a434b24c7cd 100644 +--- a/pkgs/development/compilers/llvm/8/llvm/default.nix ++++ b/pkgs/development/compilers/llvm/8/llvm/default.nix +@@ -79,7 +79,7 @@ in stdenv.mkDerivation ({ + --replace 'set(_install_rpath "@loader_path/../''${CMAKE_INSTALL_LIBDIR}" ''${extra_libdir})' "" + '' + # Patch llvm-config to return correct library path based on --link-{shared,static}. +- + optionalString (enableSharedLibraries) '' ++ + '' + substitute '${./outputs.patch}' ./outputs.patch --subst-var lib + patch -p1 < ./outputs.patch + '' + '' +diff --git a/pkgs/development/compilers/llvm/8/llvm/outputs.patch b/pkgs/development/compilers/llvm/8/llvm/outputs.patch +index 40096fa3497fd..878460e05b8af 100644 +--- a/pkgs/development/compilers/llvm/8/llvm/outputs.patch ++++ b/pkgs/development/compilers/llvm/8/llvm/outputs.patch +@@ -2,23 +2,13 @@ diff --git a/tools/llvm-config/llvm-config.cpp b/tools/llvm-config/llvm-config.c + index 94d426b..37f7794 100644 + --- a/tools/llvm-config/llvm-config.cpp + +++ b/tools/llvm-config/llvm-config.cpp +-@@ -333,6 +333,21 @@ int main(int argc, char **argv) { ++@@ -333,6 +333,11 @@ int main(int argc, char **argv) { + ActiveIncludeOption = "-I" + ActiveIncludeDir; + } + +-+ /// Nix-specific multiple-output handling: override ActiveLibDir if --link-shared +++ /// Nix-specific multiple-output handling: override ActiveLibDir + + if (!IsInDevelopmentTree) { +-+ bool WantShared = true; +-+ for (int i = 1; i < argc; ++i) { +-+ StringRef Arg = argv[i]; +-+ if (Arg == "--link-shared") +-+ WantShared = true; +-+ else if (Arg == "--link-static") +-+ WantShared = false; // the last one wins +-+ } +-+ +-+ if (WantShared) +-+ ActiveLibDir = std::string("@lib@") + "/lib" + LLVM_LIBDIR_SUFFIX; +++ ActiveLibDir = std::string("@lib@") + "/lib" + LLVM_LIBDIR_SUFFIX; + + } + + + /// We only use `shared library` mode in cases where the static library form +diff --git a/pkgs/development/compilers/llvm/9/llvm/default.nix b/pkgs/development/compilers/llvm/9/llvm/default.nix +index 570795824ee58..a569c00071de0 100644 +--- a/pkgs/development/compilers/llvm/9/llvm/default.nix ++++ b/pkgs/development/compilers/llvm/9/llvm/default.nix +@@ -77,7 +77,7 @@ in stdenv.mkDerivation (rec { + --replace 'set(_install_rpath "@loader_path/../''${CMAKE_INSTALL_LIBDIR}" ''${extra_libdir})' "" + '' + # Patch llvm-config to return correct library path based on --link-{shared,static}. +- + optionalString (enableSharedLibraries) '' ++ + '' + substitute '${./outputs.patch}' ./outputs.patch --subst-var lib + patch -p1 < ./outputs.patch + '' + '' +diff --git a/pkgs/development/compilers/llvm/9/llvm/outputs.patch b/pkgs/development/compilers/llvm/9/llvm/outputs.patch +index 40096fa3497fd..878460e05b8af 100644 +--- a/pkgs/development/compilers/llvm/9/llvm/outputs.patch ++++ b/pkgs/development/compilers/llvm/9/llvm/outputs.patch +@@ -2,23 +2,13 @@ diff --git a/tools/llvm-config/llvm-config.cpp b/tools/llvm-config/llvm-config.c + index 94d426b..37f7794 100644 + --- a/tools/llvm-config/llvm-config.cpp + +++ b/tools/llvm-config/llvm-config.cpp +-@@ -333,6 +333,21 @@ int main(int argc, char **argv) { ++@@ -333,6 +333,11 @@ int main(int argc, char **argv) { + ActiveIncludeOption = "-I" + ActiveIncludeDir; + } + +-+ /// Nix-specific multiple-output handling: override ActiveLibDir if --link-shared +++ /// Nix-specific multiple-output handling: override ActiveLibDir + + if (!IsInDevelopmentTree) { +-+ bool WantShared = true; +-+ for (int i = 1; i < argc; ++i) { +-+ StringRef Arg = argv[i]; +-+ if (Arg == "--link-shared") +-+ WantShared = true; +-+ else if (Arg == "--link-static") +-+ WantShared = false; // the last one wins +-+ } +-+ +-+ if (WantShared) +-+ ActiveLibDir = std::string("@lib@") + "/lib" + LLVM_LIBDIR_SUFFIX; +++ ActiveLibDir = std::string("@lib@") + "/lib" + LLVM_LIBDIR_SUFFIX; + + } + + + /// We only use `shared library` mode in cases where the static library form +diff --git a/pkgs/development/compilers/llvm/git/llvm/default.nix b/pkgs/development/compilers/llvm/git/llvm/default.nix +index daf4cfe808b6c..4c895f81dbbac 100644 +--- a/pkgs/development/compilers/llvm/git/llvm/default.nix ++++ b/pkgs/development/compilers/llvm/git/llvm/default.nix +@@ -60,7 +60,7 @@ in stdenv.mkDerivation (rec { + --replace 'set(_install_rpath "@loader_path/../''${CMAKE_INSTALL_LIBDIR}''${LLVM_LIBDIR_SUFFIX}" ''${extra_libdir})' "" + '' + # Patch llvm-config to return correct library path based on --link-{shared,static}. +- + optionalString (enableSharedLibraries) '' ++ + '' + substitute '${./outputs.patch}' ./outputs.patch --subst-var lib + patch -p1 < ./outputs.patch + '' + '' +diff --git a/pkgs/development/compilers/llvm/git/llvm/outputs.patch b/pkgs/development/compilers/llvm/git/llvm/outputs.patch +index 40096fa3497fd..878460e05b8af 100644 +--- a/pkgs/development/compilers/llvm/git/llvm/outputs.patch ++++ b/pkgs/development/compilers/llvm/git/llvm/outputs.patch +@@ -2,23 +2,13 @@ diff --git a/tools/llvm-config/llvm-config.cpp b/tools/llvm-config/llvm-config.c + index 94d426b..37f7794 100644 + --- a/tools/llvm-config/llvm-config.cpp + +++ b/tools/llvm-config/llvm-config.cpp +-@@ -333,6 +333,21 @@ int main(int argc, char **argv) { ++@@ -333,6 +333,11 @@ int main(int argc, char **argv) { + ActiveIncludeOption = "-I" + ActiveIncludeDir; + } + +-+ /// Nix-specific multiple-output handling: override ActiveLibDir if --link-shared +++ /// Nix-specific multiple-output handling: override ActiveLibDir + + if (!IsInDevelopmentTree) { +-+ bool WantShared = true; +-+ for (int i = 1; i < argc; ++i) { +-+ StringRef Arg = argv[i]; +-+ if (Arg == "--link-shared") +-+ WantShared = true; +-+ else if (Arg == "--link-static") +-+ WantShared = false; // the last one wins +-+ } +-+ +-+ if (WantShared) +-+ ActiveLibDir = std::string("@lib@") + "/lib" + LLVM_LIBDIR_SUFFIX; +++ ActiveLibDir = std::string("@lib@") + "/lib" + LLVM_LIBDIR_SUFFIX; + + } + + + /// We only use `shared library` mode in cases where the static library form +diff --git a/pkgs/development/compilers/llvm/rocm/llvm/outputs.patch b/pkgs/development/compilers/llvm/rocm/llvm/outputs.patch +index 40096fa3497fd..878460e05b8af 100644 +--- a/pkgs/development/compilers/llvm/rocm/llvm/outputs.patch ++++ b/pkgs/development/compilers/llvm/rocm/llvm/outputs.patch +@@ -2,23 +2,13 @@ diff --git a/tools/llvm-config/llvm-config.cpp b/tools/llvm-config/llvm-config.c + index 94d426b..37f7794 100644 + --- a/tools/llvm-config/llvm-config.cpp + +++ b/tools/llvm-config/llvm-config.cpp +-@@ -333,6 +333,21 @@ int main(int argc, char **argv) { ++@@ -333,6 +333,11 @@ int main(int argc, char **argv) { + ActiveIncludeOption = "-I" + ActiveIncludeDir; + } + +-+ /// Nix-specific multiple-output handling: override ActiveLibDir if --link-shared +++ /// Nix-specific multiple-output handling: override ActiveLibDir + + if (!IsInDevelopmentTree) { +-+ bool WantShared = true; +-+ for (int i = 1; i < argc; ++i) { +-+ StringRef Arg = argv[i]; +-+ if (Arg == "--link-shared") +-+ WantShared = true; +-+ else if (Arg == "--link-static") +-+ WantShared = false; // the last one wins +-+ } +-+ +-+ if (WantShared) +-+ ActiveLibDir = std::string("@lib@") + "/lib" + LLVM_LIBDIR_SUFFIX; +++ ActiveLibDir = std::string("@lib@") + "/lib" + LLVM_LIBDIR_SUFFIX; + + } + + + /// We only use `shared library` mode in cases where the static library form From 1f3aa48361e03748935cc6083c60223c99ce767d Mon Sep 17 00:00:00 2001 From: ychenfo Date: Thu, 2 Dec 2021 02:54:11 +0800 Subject: [PATCH 25/54] nac3parser: modify parser to handle negative integer edge cases --- nac3parser/src/python.lalrpop | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/nac3parser/src/python.lalrpop b/nac3parser/src/python.lalrpop index 3a7c6b81..7cfdf3a3 100644 --- a/nac3parser/src/python.lalrpop +++ b/nac3parser/src/python.lalrpop @@ -5,7 +5,7 @@ use std::iter::FromIterator; -use crate::ast; +use crate::ast::{self, Constant}; use crate::fstring::parse_located_fstring; use crate::function::{ArgumentList, parse_args, parse_params}; use crate::error::LexicalError; @@ -916,7 +916,17 @@ Factor: ast::Expr = { => ast::Expr { location, custom: (), - node: ast::ExprKind::UnaryOp { operand: Box::new(e), op } + node: { + match (&op, &e.node) { + (ast::Unaryop::USub, ast::ExprKind::Constant { value: Constant::Int(val), kind }) => { + ast::ExprKind::Constant { + value: Constant::Int(-val), + kind: kind.clone() + } + } + _ => ast::ExprKind::UnaryOp { operand: Box::new(e), op } + } + } }, Power, }; From c98f367f90b7b658b3b6476aa111f68676f68f31 Mon Sep 17 00:00:00 2001 From: pca006132 Date: Sat, 4 Dec 2021 17:52:03 +0800 Subject: [PATCH 26/54] nac3artiq: enables inlining --- nac3artiq/src/lib.rs | 135 +++++++++++++++++++++++++------------------ 1 file changed, 80 insertions(+), 55 deletions(-) diff --git a/nac3artiq/src/lib.rs b/nac3artiq/src/lib.rs index ac087994..66660db5 100644 --- a/nac3artiq/src/lib.rs +++ b/nac3artiq/src/lib.rs @@ -4,6 +4,7 @@ use std::process::Command; use std::sync::Arc; use inkwell::{ + memory_buffer::MemoryBuffer, passes::{PassManager, PassManagerBuilder}, targets::*, OptimizationLevel, @@ -301,8 +302,8 @@ impl Nac3 { .call0() .unwrap() .get_item("virtual") - .unwrap(), - )).unwrap() + .unwrap(),)) + .unwrap() .extract() .unwrap(), generic_alias: ( @@ -521,52 +522,17 @@ impl Nac3 { }; let isa = self.isa; let working_directory = self.working_directory.path().to_owned(); - let f = Arc::new(WithCall::new(Box::new(move |module| { - let builder = PassManagerBuilder::create(); - builder.set_optimization_level(OptimizationLevel::Default); - let passes = PassManager::create(()); - builder.populate_module_pass_manager(&passes); - passes.run_on(module); - let (triple, features) = match isa { - Isa::Host => ( - TargetMachine::get_default_triple(), - TargetMachine::get_host_cpu_features().to_string(), - ), - Isa::RiscV32G => ( - TargetTriple::create("riscv32-unknown-linux"), - "+a,+m,+f,+d".to_string(), - ), - Isa::RiscV32IMA => ( - TargetTriple::create("riscv32-unknown-linux"), - "+a,+m".to_string(), - ), - Isa::CortexA9 => ( - TargetTriple::create("armv7-unknown-linux-gnueabihf"), - "+dsp,+fp16,+neon,+vfp3".to_string(), - ), - }; - let target = - Target::from_triple(&triple).expect("couldn't create target from target triple"); - let target_machine = target - .create_target_machine( - &triple, - "", - &features, - OptimizationLevel::Default, - RelocMode::PIC, - CodeModel::Default, - ) - .expect("couldn't create target machine"); - target_machine - .write_to_file( - module, - FileType::Object, - &working_directory.join(&format!("{}.o", module.get_name().to_str().unwrap())), - ) - .expect("couldn't write module to file"); + let membuffers: Arc>>> = Default::default(); + + let membuffer = membuffers.clone(); + + let f = Arc::new(WithCall::new(Box::new(move |module| { + let buffer = module.write_bitcode_to_memory(); + let buffer = buffer.as_slice().into(); + membuffer.lock().push(buffer); }))); - let thread_names: Vec = (0..4).map(|i| format!("module{}", i)).collect(); + let thread_names: Vec = (0..4).map(|_| "main".to_string()).collect(); let threads: Vec<_> = thread_names .iter() .map(|s| Box::new(ArtiqCodeGenerator::new(s.to_string(), self.time_fns))) @@ -578,12 +544,79 @@ impl Nac3 { registry.wait_tasks_complete(handles); }); + let buffers = membuffers.lock(); + let context = inkwell::context::Context::create(); + let main = context + .create_module_from_ir(MemoryBuffer::create_from_memory_range(&buffers[0], "main")) + .unwrap(); + for buffer in buffers.iter().skip(1) { + let other = context + .create_module_from_ir(MemoryBuffer::create_from_memory_range(buffer, "main")) + .unwrap(); + + main.link_in_module(other) + .map_err(|err| exceptions::PyRuntimeError::new_err(err.to_string()))?; + } + + let mut function_iter = main.get_first_function(); + while let Some(func) = function_iter { + if func.count_basic_blocks() > 0 && func.get_name().to_str().unwrap() != "__modinit__" { + func.set_linkage(inkwell::module::Linkage::Private); + } + function_iter = func.get_next_function(); + } + + let builder = PassManagerBuilder::create(); + builder.set_optimization_level(OptimizationLevel::Aggressive); + let passes = PassManager::create(()); + builder.set_inliner_with_threshold(255); + builder.populate_module_pass_manager(&passes); + passes.run_on(&main); + + let (triple, features) = match isa { + Isa::Host => ( + TargetMachine::get_default_triple(), + TargetMachine::get_host_cpu_features().to_string(), + ), + Isa::RiscV32G => ( + TargetTriple::create("riscv32-unknown-linux"), + "+a,+m,+f,+d".to_string(), + ), + Isa::RiscV32IMA => ( + TargetTriple::create("riscv32-unknown-linux"), + "+a,+m".to_string(), + ), + Isa::CortexA9 => ( + TargetTriple::create("armv7-unknown-linux-gnueabihf"), + "+dsp,+fp16,+neon,+vfp3".to_string(), + ), + }; + let target = + Target::from_triple(&triple).expect("couldn't create target from target triple"); + let target_machine = target + .create_target_machine( + &triple, + "", + &features, + OptimizationLevel::Default, + RelocMode::PIC, + CodeModel::Default, + ) + .expect("couldn't create target machine"); + target_machine + .write_to_file(&main, FileType::Object, &working_directory.join("module.o")) + .expect("couldn't write module to file"); + let mut linker_args = vec![ "-shared".to_string(), "--eh-frame-hdr".to_string(), "-x".to_string(), "-o".to_string(), filename.to_string(), + working_directory + .join("module.o") + .to_string_lossy() + .to_string(), ]; if isa != Isa::Host { linker_args.push( @@ -596,15 +629,7 @@ impl Nac3 { .unwrap(), ); } - linker_args.extend(thread_names.iter().map(|name| { - let name_o = name.to_owned() + ".o"; - self.working_directory - .path() - .join(name_o.as_str()) - .to_str() - .unwrap() - .to_string() - })); + if let Ok(linker_status) = Command::new("ld.lld").args(linker_args).status() { if !linker_status.success() { return Err(exceptions::PyRuntimeError::new_err( From 41f88095a5ec6a348114b22de43f0ec9c8906fc3 Mon Sep 17 00:00:00 2001 From: Sebastien Bourdeauducq Date: Sat, 4 Dec 2021 20:35:52 +0800 Subject: [PATCH 27/54] min_artiq: add round64, floor64, ceil64 --- nac3artiq/demo/min_artiq.py | 26 ++++++++++++++++++++++---- 1 file changed, 22 insertions(+), 4 deletions(-) diff --git a/nac3artiq/demo/min_artiq.py b/nac3artiq/demo/min_artiq.py index 354d2a2a..9adf1667 100644 --- a/nac3artiq/demo/min_artiq.py +++ b/nac3artiq/demo/min_artiq.py @@ -3,13 +3,20 @@ from functools import wraps from types import SimpleNamespace from numpy import int32, int64 from typing import Generic, TypeVar +from math import floor, ceil import nac3artiq -__all__ = ["KernelInvariant", "extern", "kernel", "portable", "nac3", - "ms", "us", "ns", - "print_int32", "print_int64", - "Core", "TTLOut", "parallel", "sequential", "virtual"] + +__all__ = [ + "KernelInvariant", "virtual", + "round64", "floor64", "ceil64", + "extern", "kernel", "portable", "nac3", + "ms", "us", "ns", + "print_int32", "print_int64", + "Core", "TTLOut", + "parallel", "sequential" +] T = TypeVar('T') @@ -21,6 +28,17 @@ class KernelInvariant(Generic[T]): class virtual(Generic[T]): pass + +def round64(x): + return round(x) + +def floor64(x): + return floor(x) + +def ceil64(x): + return ceil(x) + + import device_db core_arguments = device_db.device_db["core"]["arguments"] From e4f35372d316c6bfcc58544e8ef3d509eb16b66a Mon Sep 17 00:00:00 2001 From: Sebastien Bourdeauducq Date: Sun, 5 Dec 2021 12:56:47 +0800 Subject: [PATCH 28/54] flake: better shells --- flake.nix | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/flake.nix b/flake.nix index c72bb8ad..62d4d11b 100644 --- a/flake.nix +++ b/flake.nix @@ -106,9 +106,13 @@ ); }; - devShell.x86_64-linux = pkgs.mkShell { + # Use "nix develop" to debug the derivation. + devShell.x86_64-linux = packages.x86_64-linux.nac3artiq; + + # Use "nix shell" to enter a development environment. + defaultPackage.x86_64-linux = pkgs.buildEnv { name = "nac3-dev-shell"; - buildInputs = with pkgs; [ + paths = with pkgs; [ llvm_12 clang_12 lld_12 @@ -125,7 +129,7 @@ hydraJobs = { inherit (packages.x86_64-linux) nac3artiq; nac3artiq-mingw = packages.x86_64-w64-mingw32.nac3artiq; - } // (pkgs.lib.foldr (a: b: {"${pkgs.lib.strings.getName a}" = a;} // b) {} devShell.x86_64-linux.buildInputs); + } // (pkgs.lib.foldr (a: b: {"${pkgs.lib.strings.getName a}" = a;} // b) {} defaultPackage.x86_64-linux.paths); }; nixConfig = { From e8e14994789db51dfaa71e4a4b524eabba5f05b4 Mon Sep 17 00:00:00 2001 From: Sebastien Bourdeauducq Date: Sun, 5 Dec 2021 13:02:57 +0800 Subject: [PATCH 29/54] flake: fix hydraJobs --- flake.nix | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/flake.nix b/flake.nix index 62d4d11b..8157f217 100644 --- a/flake.nix +++ b/flake.nix @@ -110,7 +110,7 @@ devShell.x86_64-linux = packages.x86_64-linux.nac3artiq; # Use "nix shell" to enter a development environment. - defaultPackage.x86_64-linux = pkgs.buildEnv { + defaultPackage.x86_64-linux = pkgs.buildEnv rec { name = "nac3-dev-shell"; paths = with pkgs; [ llvm_12 @@ -124,12 +124,13 @@ clippy (python3.withPackages(ps: [ ps.numpy ])) ]; + passthru.hydrapkgs = paths; }; hydraJobs = { inherit (packages.x86_64-linux) nac3artiq; nac3artiq-mingw = packages.x86_64-w64-mingw32.nac3artiq; - } // (pkgs.lib.foldr (a: b: {"${pkgs.lib.strings.getName a}" = a;} // b) {} defaultPackage.x86_64-linux.paths); + } // (pkgs.lib.foldr (a: b: {"${pkgs.lib.strings.getName a}" = a;} // b) {} defaultPackage.x86_64-linux.hydrapkgs); }; nixConfig = { From 2938eacd1665bf89eb735c9a8b03f7b2c00605b8 Mon Sep 17 00:00:00 2001 From: pca006132 Date: Sat, 4 Dec 2021 22:25:52 +0800 Subject: [PATCH 30/54] nac3artiq: supports running multiple kernels --- nac3artiq/src/lib.rs | 81 +++++++++++++++++++++++++++----------------- 1 file changed, 50 insertions(+), 31 deletions(-) diff --git a/nac3artiq/src/lib.rs b/nac3artiq/src/lib.rs index 66660db5..f27abde4 100644 --- a/nac3artiq/src/lib.rs +++ b/nac3artiq/src/lib.rs @@ -1,6 +1,7 @@ use std::collections::{HashMap, HashSet}; use std::fs; use std::process::Command; +use std::rc::Rc; use std::sync::Arc; use inkwell::{ @@ -10,7 +11,7 @@ use inkwell::{ OptimizationLevel, }; use nac3parser::{ - ast::{self, StrRef}, + ast::{self, Stmt, StrRef}, parser::{self, parse_program}, }; use pyo3::prelude::*; @@ -21,7 +22,7 @@ use parking_lot::{Mutex, RwLock}; use nac3core::{ codegen::{concrete_type::ConcreteTypeStore, CodeGenTask, WithCall, WorkerRegistry}, symbol_resolver::SymbolResolver, - toplevel::{composer::TopLevelComposer, DefinitionId, GenCall, TopLevelContext, TopLevelDef}, + toplevel::{composer::TopLevelComposer, DefinitionId, GenCall, TopLevelDef}, typecheck::typedef::{FunSignature, FuncArg}, typecheck::{type_inferencer::PrimitiveStore, typedef::Type}, }; @@ -62,6 +63,13 @@ pub struct PrimitivePythonId { virtual_id: u64, } +type TopLevelComponent = ( + Stmt, + Arc, + String, + Rc>, +); + // TopLevelComposer is unsendable as it holds the unification table, which is // unsendable due to Rc. Arc would cause a performance hit. #[pyclass(unsendable, name = "NAC3")] @@ -69,15 +77,15 @@ struct Nac3 { isa: Isa, time_fns: &'static (dyn TimeFns + Sync), primitive: PrimitiveStore, + builtins: Vec<(StrRef, FunSignature, Arc)>, builtins_ty: HashMap, builtins_def: HashMap, pyid_to_def: Arc>>, pyid_to_type: Arc>>, - composer: TopLevelComposer, - top_level: Option>, primitive_ids: PrimitivePythonId, global_value_ids: Arc>>, working_directory: TempDir, + top_levels: Vec, } impl Nac3 { @@ -134,8 +142,7 @@ impl Nac3 { module: module.clone(), helper, }))) as Arc; - let mut name_to_def = HashMap::new(); - let mut name_to_type = HashMap::new(); + let name_to_pyid = Rc::new(name_to_pyid); for mut stmt in parser_result.into_iter() { let include = match stmt.node { @@ -202,24 +209,14 @@ impl Nac3 { }; if include { - let (name, def_id, ty) = self - .composer - .register_top_level(stmt, Some(resolver.clone()), module_name.clone()) - .unwrap(); - name_to_def.insert(name, def_id); - if let Some(ty) = ty { - name_to_type.insert(name, ty); - } + self.top_levels.push(( + stmt, + resolver.clone(), + module_name.clone(), + name_to_pyid.clone(), + )); } } - let mut map = self.pyid_to_def.write(); - for (name, def) in name_to_def.into_iter() { - map.insert(*name_to_pyid.get(&name).unwrap(), def); - } - let mut map = self.pyid_to_type.write(); - for (name, ty) in name_to_type.into_iter() { - map.insert(*name_to_pyid.get(&name).unwrap(), ty); - } Ok(()) } } @@ -287,7 +284,7 @@ impl Nac3 { }))), ), ]; - let (composer, builtins_def, builtins_ty) = TopLevelComposer::new(builtins); + let (_, builtins_def, builtins_ty) = TopLevelComposer::new(builtins.clone()); let builtins_mod = PyModule::import(py, "builtins").unwrap(); let id_fn = builtins_mod.getattr("id").unwrap(); @@ -376,11 +373,11 @@ impl Nac3 { isa, time_fns, primitive, + builtins, builtins_ty, builtins_def, - composer, primitive_ids, - top_level: None, + top_levels: Default::default(), pyid_to_def: Default::default(), pyid_to_type: Default::default(), global_value_ids: Default::default(), @@ -423,6 +420,30 @@ impl Nac3 { filename: &str, py: Python, ) -> PyResult<()> { + let (mut composer, _, _) = TopLevelComposer::new(self.builtins.clone()); + let mut id_to_def = HashMap::new(); + let mut id_to_type = HashMap::new(); + for (stmt, resolver, path, name_to_pyid) in self.top_levels.iter() { + let (name, def_id, ty) = composer + .register_top_level(stmt.clone(), Some(resolver.clone()), path.clone()) + .unwrap(); + let id = *name_to_pyid.get(&name).unwrap(); + id_to_def.insert(id, def_id); + if let Some(ty) = ty { + id_to_type.insert(id, ty); + } + } + { + let mut map = self.pyid_to_def.write(); + for (id, def) in id_to_def.into_iter() { + map.insert(id, def); + } + let mut map = self.pyid_to_type.write(); + for (id, ty) in id_to_type.into_iter() { + map.insert(id, ty); + } + } + let id_fun = PyModule::import(py, "builtins")?.getattr("id")?; let mut name_to_pyid: HashMap = HashMap::new(); let module = PyModule::new(py, "tmp")?; @@ -466,8 +487,7 @@ impl Nac3 { module: module.to_object(py), helper, }))) as Arc; - let (_, def_id, _) = self - .composer + let (_, def_id, _) = composer .register_top_level( synthesized.pop().unwrap(), Some(resolver.clone()), @@ -483,16 +503,15 @@ impl Nac3 { let mut store = ConcreteTypeStore::new(); let mut cache = HashMap::new(); let signature = store.from_signature( - &mut self.composer.unifier, + &mut composer.unifier, &self.primitive, &signature, &mut cache, ); let signature = store.add_cty(signature); - self.composer.start_analysis(true).unwrap(); - self.top_level = Some(Arc::new(self.composer.make_top_level_context())); - let top_level = self.top_level.as_ref().unwrap(); + composer.start_analysis(true).unwrap(); + let top_level = Arc::new(composer.make_top_level_context()); let instance = { let defs = top_level.definitions.read(); let mut definition = defs[def_id.0].write(); From 65bc1e5fa446e7ff664f6567bf96e0ec54416ef6 Mon Sep 17 00:00:00 2001 From: pca006132 Date: Sat, 4 Dec 2021 22:57:48 +0800 Subject: [PATCH 31/54] nac3artiq: handle name_to_pyid in compilation python variables can change between kernel invocations --- nac3artiq/src/lib.rs | 116 ++++++++++++++++--------------- nac3artiq/src/symbol_resolver.rs | 1 + 2 files changed, 60 insertions(+), 57 deletions(-) diff --git a/nac3artiq/src/lib.rs b/nac3artiq/src/lib.rs index f27abde4..46ee93aa 100644 --- a/nac3artiq/src/lib.rs +++ b/nac3artiq/src/lib.rs @@ -15,7 +15,7 @@ use nac3parser::{ parser::{self, parse_program}, }; use pyo3::prelude::*; -use pyo3::{exceptions, types::PyBytes, types::PyList, types::PySet}; +use pyo3::{exceptions, types::PyBytes, types::PyDict, types::PySet}; use parking_lot::{Mutex, RwLock}; @@ -63,12 +63,7 @@ pub struct PrimitivePythonId { virtual_id: u64, } -type TopLevelComponent = ( - Stmt, - Arc, - String, - Rc>, -); +type TopLevelComponent = (Stmt, String, PyObject); // TopLevelComposer is unsendable as it holds the unification table, which is // unsendable due to Rc. Arc would cause a performance hit. @@ -94,35 +89,13 @@ impl Nac3 { module: PyObject, registered_class_ids: &HashSet, ) -> PyResult<()> { - let mut name_to_pyid: HashMap = HashMap::new(); - let (module_name, source_file, helper) = - Python::with_gil(|py| -> PyResult<(String, String, PythonHelper)> { - let module: &PyAny = module.extract(py)?; - let builtins = PyModule::import(py, "builtins")?; - let id_fn = builtins.getattr("id")?; - let members: &PyList = PyModule::import(py, "inspect")? - .getattr("getmembers")? - .call1((module,))? - .cast_as()?; - for member in members.iter() { - let key: &str = member.get_item(0)?.extract()?; - let val = id_fn.call1((member.get_item(1)?,))?.extract()?; - name_to_pyid.insert(key.into(), val); - } - let typings = PyModule::import(py, "typing")?; - let helper = PythonHelper { - id_fn: builtins.getattr("id").unwrap().to_object(py), - len_fn: builtins.getattr("len").unwrap().to_object(py), - type_fn: builtins.getattr("type").unwrap().to_object(py), - origin_ty_fn: typings.getattr("get_origin").unwrap().to_object(py), - args_ty_fn: typings.getattr("get_args").unwrap().to_object(py), - }; - Ok(( - module.getattr("__name__")?.extract()?, - module.getattr("__file__")?.extract()?, - helper, - )) - })?; + let (module_name, source_file) = Python::with_gil(|py| -> PyResult<(String, String)> { + let module: &PyAny = module.extract(py)?; + Ok(( + module.getattr("__name__")?.extract()?, + module.getattr("__file__")?.extract()?, + )) + })?; let source = fs::read_to_string(source_file).map_err(|e| { exceptions::PyIOError::new_err(format!("failed to read input file: {}", e)) @@ -130,20 +103,6 @@ impl Nac3 { let parser_result = parser::parse_program(&source) .map_err(|e| exceptions::PySyntaxError::new_err(format!("parse error: {}", e)))?; - let resolver = Arc::new(Resolver(Arc::new(InnerResolver { - id_to_type: self.builtins_ty.clone().into(), - id_to_def: self.builtins_def.clone().into(), - pyid_to_def: self.pyid_to_def.clone(), - pyid_to_type: self.pyid_to_type.clone(), - primitive_ids: self.primitive_ids.clone(), - global_value_ids: self.global_value_ids.clone(), - class_names: Default::default(), - name_to_pyid: name_to_pyid.clone(), - module: module.clone(), - helper, - }))) as Arc; - let name_to_pyid = Rc::new(name_to_pyid); - for mut stmt in parser_result.into_iter() { let include = match stmt.node { ast::StmtKind::ClassDef { @@ -209,12 +168,8 @@ impl Nac3 { }; if include { - self.top_levels.push(( - stmt, - resolver.clone(), - module_name.clone(), - name_to_pyid.clone(), - )); + self.top_levels + .push((stmt, module_name.clone(), module.clone())); } } Ok(()) @@ -423,7 +378,54 @@ impl Nac3 { let (mut composer, _, _) = TopLevelComposer::new(self.builtins.clone()); let mut id_to_def = HashMap::new(); let mut id_to_type = HashMap::new(); - for (stmt, resolver, path, name_to_pyid) in self.top_levels.iter() { + + let builtins = PyModule::import(py, "builtins")?; + let typings = PyModule::import(py, "typing")?; + let id_fn = builtins.getattr("id")?; + let helper = PythonHelper { + id_fn: builtins.getattr("id").unwrap().to_object(py), + len_fn: builtins.getattr("len").unwrap().to_object(py), + type_fn: builtins.getattr("type").unwrap().to_object(py), + origin_ty_fn: typings.getattr("get_origin").unwrap().to_object(py), + args_ty_fn: typings.getattr("get_args").unwrap().to_object(py), + }; + let mut module_to_resolver_cache: HashMap = HashMap::new(); + + for (stmt, path, module) in self.top_levels.iter() { + let py_module: &PyAny = module.extract(py)?; + let module_id: u64 = id_fn.call1((py_module,))?.extract()?; + let helper = helper.clone(); + let (name_to_pyid, resolver) = module_to_resolver_cache + .get(&module_id) + .cloned() + .unwrap_or_else(|| { + let mut name_to_pyid: HashMap = HashMap::new(); + let members: &PyDict = + py_module.getattr("__dict__").unwrap().cast_as().unwrap(); + for (key, val) in members.iter() { + let key: &str = key.extract().unwrap(); + let val = id_fn.call1((val,)).unwrap().extract().unwrap(); + name_to_pyid.insert(key.into(), val); + } + let resolver = Arc::new(Resolver(Arc::new(InnerResolver { + id_to_type: self.builtins_ty.clone().into(), + id_to_def: self.builtins_def.clone().into(), + pyid_to_def: self.pyid_to_def.clone(), + pyid_to_type: self.pyid_to_type.clone(), + primitive_ids: self.primitive_ids.clone(), + global_value_ids: self.global_value_ids.clone(), + class_names: Default::default(), + name_to_pyid: name_to_pyid.clone(), + module: module.clone(), + helper, + }))) + as Arc; + let name_to_pyid = Rc::new(name_to_pyid); + module_to_resolver_cache + .insert(module_id, (name_to_pyid.clone(), resolver.clone())); + (name_to_pyid, resolver) + }); + let (name, def_id, ty) = composer .register_top_level(stmt.clone(), Some(resolver.clone()), path.clone()) .unwrap(); diff --git a/nac3artiq/src/symbol_resolver.rs b/nac3artiq/src/symbol_resolver.rs index 92baa592..0eeedf25 100644 --- a/nac3artiq/src/symbol_resolver.rs +++ b/nac3artiq/src/symbol_resolver.rs @@ -39,6 +39,7 @@ pub struct InnerResolver { pub struct Resolver(pub Arc); +#[derive(Clone)] pub struct PythonHelper { pub type_fn: PyObject, pub len_fn: PyObject, From 9ee2168932db3836794f5a881229a8bf18c0787d Mon Sep 17 00:00:00 2001 From: Sebastien Bourdeauducq Date: Sun, 5 Dec 2021 14:35:25 +0800 Subject: [PATCH 32/54] Revert "flake: fix hydraJobs" This reverts commit e8e14994789db51dfaa71e4a4b524eabba5f05b4. --- flake.nix | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/flake.nix b/flake.nix index 8157f217..62d4d11b 100644 --- a/flake.nix +++ b/flake.nix @@ -110,7 +110,7 @@ devShell.x86_64-linux = packages.x86_64-linux.nac3artiq; # Use "nix shell" to enter a development environment. - defaultPackage.x86_64-linux = pkgs.buildEnv rec { + defaultPackage.x86_64-linux = pkgs.buildEnv { name = "nac3-dev-shell"; paths = with pkgs; [ llvm_12 @@ -124,13 +124,12 @@ clippy (python3.withPackages(ps: [ ps.numpy ])) ]; - passthru.hydrapkgs = paths; }; hydraJobs = { inherit (packages.x86_64-linux) nac3artiq; nac3artiq-mingw = packages.x86_64-w64-mingw32.nac3artiq; - } // (pkgs.lib.foldr (a: b: {"${pkgs.lib.strings.getName a}" = a;} // b) {} defaultPackage.x86_64-linux.hydrapkgs); + } // (pkgs.lib.foldr (a: b: {"${pkgs.lib.strings.getName a}" = a;} // b) {} defaultPackage.x86_64-linux.paths); }; nixConfig = { From d2919b9620c88756b73aa0cb13f9f014703b67c4 Mon Sep 17 00:00:00 2001 From: Sebastien Bourdeauducq Date: Sun, 5 Dec 2021 14:35:27 +0800 Subject: [PATCH 33/54] Revert "flake: better shells" llvm-config/llvm-sys hates pkgs.buildEnv. This reverts commit e4f35372d316c6bfcc58544e8ef3d509eb16b66a. --- flake.nix | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/flake.nix b/flake.nix index 62d4d11b..c72bb8ad 100644 --- a/flake.nix +++ b/flake.nix @@ -106,13 +106,9 @@ ); }; - # Use "nix develop" to debug the derivation. - devShell.x86_64-linux = packages.x86_64-linux.nac3artiq; - - # Use "nix shell" to enter a development environment. - defaultPackage.x86_64-linux = pkgs.buildEnv { + devShell.x86_64-linux = pkgs.mkShell { name = "nac3-dev-shell"; - paths = with pkgs; [ + buildInputs = with pkgs; [ llvm_12 clang_12 lld_12 @@ -129,7 +125,7 @@ hydraJobs = { inherit (packages.x86_64-linux) nac3artiq; nac3artiq-mingw = packages.x86_64-w64-mingw32.nac3artiq; - } // (pkgs.lib.foldr (a: b: {"${pkgs.lib.strings.getName a}" = a;} // b) {} defaultPackage.x86_64-linux.paths); + } // (pkgs.lib.foldr (a: b: {"${pkgs.lib.strings.getName a}" = a;} // b) {} devShell.x86_64-linux.buildInputs); }; nixConfig = { From baa713a3cabb90e2b3519f1d323f09c2f881c605 Mon Sep 17 00:00:00 2001 From: Sebastien Bourdeauducq Date: Sun, 5 Dec 2021 14:40:10 +0800 Subject: [PATCH 34/54] flake: don't attempt to fixup Windows build --- flake.nix | 1 + 1 file changed, 1 insertion(+) diff --git a/flake.nix b/flake.nix index c72bb8ad..a4d84b97 100644 --- a/flake.nix +++ b/flake.nix @@ -101,6 +101,7 @@ zip $out/nac3artiq.zip nac3artiq.pyd echo file binary-dist $out/nac3artiq.zip >> $out/nix-support/hydra-build-products ''; + dontFixup = true; meta.platforms = ["x86_64-windows"]; } ); From 6d00d4dabb54f6944a9c97981a1d3416093a3df9 Mon Sep 17 00:00:00 2001 From: pca006132 Date: Sun, 5 Dec 2021 20:30:03 +0800 Subject: [PATCH 35/54] nac3artiq: cache python data if possible --- nac3artiq/src/lib.rs | 17 +- nac3artiq/src/symbol_resolver.rs | 384 ++++++++++++++++++++----------- 2 files changed, 255 insertions(+), 146 deletions(-) diff --git a/nac3artiq/src/lib.rs b/nac3artiq/src/lib.rs index 46ee93aa..405645d8 100644 --- a/nac3artiq/src/lib.rs +++ b/nac3artiq/src/lib.rs @@ -78,7 +78,7 @@ struct Nac3 { pyid_to_def: Arc>>, pyid_to_type: Arc>>, primitive_ids: PrimitivePythonId, - global_value_ids: Arc>>, + global_value_ids: Arc>>, working_directory: TempDir, top_levels: Vec, } @@ -417,6 +417,9 @@ impl Nac3 { class_names: Default::default(), name_to_pyid: name_to_pyid.clone(), module: module.clone(), + id_to_pyval: Default::default(), + id_to_primitive: Default::default(), + field_to_val: Default::default(), helper, }))) as Arc; @@ -468,15 +471,6 @@ impl Nac3 { ) }; let mut synthesized = parse_program(&synthesized).unwrap(); - let builtins = PyModule::import(py, "builtins")?; - let typings = PyModule::import(py, "typing")?; - let helper = PythonHelper { - id_fn: builtins.getattr("id").unwrap().to_object(py), - len_fn: builtins.getattr("len").unwrap().to_object(py), - type_fn: builtins.getattr("type").unwrap().to_object(py), - origin_ty_fn: typings.getattr("get_origin").unwrap().to_object(py), - args_ty_fn: typings.getattr("get_args").unwrap().to_object(py), - }; let resolver = Arc::new(Resolver(Arc::new(InnerResolver { id_to_type: self.builtins_ty.clone().into(), id_to_def: self.builtins_def.clone().into(), @@ -485,6 +479,9 @@ impl Nac3 { primitive_ids: self.primitive_ids.clone(), global_value_ids: self.global_value_ids.clone(), class_names: Default::default(), + id_to_pyval: Default::default(), + id_to_primitive: Default::default(), + field_to_val: Default::default(), name_to_pyid, module: module.to_object(py), helper, diff --git a/nac3artiq/src/symbol_resolver.rs b/nac3artiq/src/symbol_resolver.rs index 0eeedf25..e42db752 100644 --- a/nac3artiq/src/symbol_resolver.rs +++ b/nac3artiq/src/symbol_resolver.rs @@ -12,7 +12,7 @@ use nac3core::{ use nac3parser::ast::{self, StrRef}; use parking_lot::{Mutex, RwLock}; use pyo3::{ - types::{PyList, PyModule, PyTuple}, + types::{PyDict, PyTuple}, PyAny, PyObject, PyResult, Python, }; use std::{ @@ -23,10 +23,20 @@ use std::{ use crate::PrimitivePythonId; +pub enum PrimitiveValue { + I32(i32), + I64(i64), + F64(f64), + Bool(bool), +} + pub struct InnerResolver { - pub id_to_type: Mutex>, - pub id_to_def: Mutex>, - pub global_value_ids: Arc>>, + pub id_to_type: RwLock>, + pub id_to_def: RwLock>, + pub id_to_pyval: RwLock>, + pub id_to_primitive: RwLock>, + pub field_to_val: RwLock>>, + pub global_value_ids: Arc>>, pub class_names: Mutex>, pub pyid_to_def: Arc>>, pub pyid_to_type: Arc>>, @@ -63,6 +73,20 @@ impl StaticValue for PythonValue { &self, ctx: &mut CodeGenContext<'ctx, 'a>, ) -> BasicValueEnum<'ctx> { + if let Some(val) = self.resolver.id_to_primitive.read().get(&self.id) { + return match val { + PrimitiveValue::I32(val) => ctx.ctx.i32_type().const_int(*val as u64, false).into(), + PrimitiveValue::I64(val) => ctx.ctx.i64_type().const_int(*val as u64, false).into(), + PrimitiveValue::F64(val) => ctx.ctx.f64_type().const_float(*val).into(), + PrimitiveValue::Bool(val) => { + ctx.ctx.bool_type().const_int(*val as u64, false).into() + } + }; + } + if let Some(global) = ctx.module.get_global(&self.id.to_string()) { + return global.as_pointer_value().into(); + } + Python::with_gil(|py| -> PyResult> { self.resolver .get_obj_value(py, self.value.as_ref(py), ctx) @@ -76,34 +100,48 @@ impl StaticValue for PythonValue { name: StrRef, ctx: &mut CodeGenContext<'ctx, 'a>, ) -> Option> { - Python::with_gil(|py| -> PyResult>> { - let helper = &self.resolver.helper; - let ty = helper.type_fn.call1(py, (&self.value,))?; - let ty_id: u64 = helper.id_fn.call1(py, (ty,))?.extract(py)?; - let def_id = { *self.resolver.pyid_to_def.read().get(&ty_id).unwrap() }; - let mut mutable = true; - let defs = ctx.top_level.definitions.read(); - if let TopLevelDef::Class { fields, .. } = &*defs[def_id.0].read() { - for (field_name, _, is_mutable) in fields.iter() { - if field_name == &name { - mutable = *is_mutable; - break; + { + let field_to_val = self.resolver.field_to_val.read(); + field_to_val.get(&(self.id, name)).cloned() + } + .unwrap_or_else(|| { + Python::with_gil(|py| -> PyResult> { + let helper = &self.resolver.helper; + let ty = helper.type_fn.call1(py, (&self.value,))?; + let ty_id: u64 = helper.id_fn.call1(py, (ty,))?.extract(py)?; + let def_id = { *self.resolver.pyid_to_def.read().get(&ty_id).unwrap() }; + let mut mutable = true; + let defs = ctx.top_level.definitions.read(); + if let TopLevelDef::Class { fields, .. } = &*defs[def_id.0].read() { + for (field_name, _, is_mutable) in fields.iter() { + if field_name == &name { + mutable = *is_mutable; + break; + } } } - } - Ok(if mutable { - None - } else { - let obj = self.value.getattr(py, &name.to_string())?; - let id = self.resolver.helper.id_fn.call1(py, (&obj,))?.extract(py)?; - Some(ValueEnum::Static(Arc::new(PythonValue { - id, - value: obj, - resolver: self.resolver.clone(), - }))) + let result = if mutable { + None + } else { + let obj = self.value.getattr(py, &name.to_string())?; + let id = self.resolver.helper.id_fn.call1(py, (&obj,))?.extract(py)?; + Some((id, obj)) + }; + self.resolver + .field_to_val + .write() + .insert((self.id, name), result.clone()); + Ok(result) }) + .unwrap() + }) + .map(|(id, obj)| { + ValueEnum::Static(Arc::new(PythonValue { + id, + value: obj, + resolver: self.resolver.clone(), + })) }) - .unwrap() } } @@ -148,11 +186,7 @@ impl InnerResolver { defs: &[Arc>], primitives: &PrimitiveStore, ) -> PyResult> { - let ty_id: u64 = self - .helper - .id_fn - .call1(py, (pyty,))? - .extract(py)?; + let ty_id: u64 = self.helper.id_fn.call1(py, (pyty,))?.extract(py)?; let ty_ty_id: u64 = self .helper .id_fn @@ -198,7 +232,8 @@ impl InnerResolver { } else { unreachable!() } - }).collect() + }) + .collect() }), fields: RefCell::new({ let mut res = methods @@ -207,7 +242,7 @@ impl InnerResolver { .collect::>(); res.extend(fields.clone().into_iter().map(|x| (x.0, (x.1, x.2)))); res - }) + }), }; // here also false, later instantiation use python object to check compatible (unifier.add_ty(ty), false) @@ -232,10 +267,10 @@ impl InnerResolver { "the {}th constraint of TypeVar `{}` is not concrete", i + 1, pyty.getattr("__name__")?.extract::()? - ))) + ))); } - }, - Err(err) => return Ok(Err(err)) + } + Err(err) => return Ok(Err(err)), } }) } else { @@ -246,31 +281,45 @@ impl InnerResolver { }; let res = unifier.get_fresh_var_with_range(&constraint_types).0; Ok(Ok((res, true))) - } else if ty_ty_id == self.primitive_ids.generic_alias.0 || ty_ty_id == self.primitive_ids.generic_alias.1 { + } else if ty_ty_id == self.primitive_ids.generic_alias.0 + || ty_ty_id == self.primitive_ids.generic_alias.1 + { let origin = self.helper.origin_ty_fn.call1(py, (pyty,))?; let args = self.helper.args_ty_fn.call1(py, (pyty,))?; let args: &PyTuple = args.cast_as(py)?; - let origin_ty = match self.get_pyty_obj_type(py, origin.as_ref(py), unifier, defs, primitives)? { - Ok((ty, false)) => ty, - Ok((_, true)) => return Ok(Err("instantiated type does not take type parameters".into())), - Err(err) => return Ok(Err(err)) - }; + let origin_ty = + match self.get_pyty_obj_type(py, origin.as_ref(py), unifier, defs, primitives)? { + Ok((ty, false)) => ty, + Ok((_, true)) => { + return Ok(Err("instantiated type does not take type parameters".into())) + } + Err(err) => return Ok(Err(err)), + }; match &*unifier.get_ty(origin_ty) { TypeEnum::TList { .. } => { if args.len() == 1 { - let ty = match self.get_pyty_obj_type(py, args.get_item(0), unifier, defs, primitives)? { + let ty = match self.get_pyty_obj_type( + py, + args.get_item(0), + unifier, + defs, + primitives, + )? { Ok(ty) => ty, - Err(err) => return Ok(Err(err)) + Err(err) => return Ok(Err(err)), }; if !unifier.is_concrete(ty.0, &[]) && !ty.1 { panic!("type list should take concrete parameters in type var ranges") } Ok(Ok((unifier.add_ty(TypeEnum::TList { ty: ty.0 }), true))) } else { - return Ok(Err(format!("type list needs exactly 1 type parameters, found {}", args.len()))) + return Ok(Err(format!( + "type list needs exactly 1 type parameters, found {}", + args.len() + ))); } - }, + } TypeEnum::TTuple { .. } => { let args = match args .iter() @@ -291,7 +340,7 @@ impl InnerResolver { _ => return Ok(Err("tuple type needs at least 1 type parameters".to_string())) }; Ok(Ok((unifier.add_ty(TypeEnum::TTuple { ty: args }), true))) - }, + } TypeEnum::TObj { params, obj_id, .. } => { let subst = { let params = &*params.borrow(); @@ -301,7 +350,7 @@ impl InnerResolver { obj_id.0, params.len(), args.len(), - ))) + ))); } let args = match args .iter() @@ -326,29 +375,48 @@ impl InnerResolver { .map(|((id, _), ty)| (*id, *ty)) .collect::>() }; - Ok(Ok((unifier.subst(origin_ty, &subst).unwrap_or(origin_ty), true))) - }, + Ok(Ok(( + unifier.subst(origin_ty, &subst).unwrap_or(origin_ty), + true, + ))) + } TypeEnum::TVirtual { .. } => { if args.len() == 1 { - let ty = match self.get_pyty_obj_type(py, args.get_item(0), unifier, defs, primitives)? { + let ty = match self.get_pyty_obj_type( + py, + args.get_item(0), + unifier, + defs, + primitives, + )? { Ok(ty) => ty, - Err(err) => return Ok(Err(err)) + Err(err) => return Ok(Err(err)), }; if !unifier.is_concrete(ty.0, &[]) && !ty.1 { - panic!("virtual class should take concrete parameters in type var ranges") + panic!( + "virtual class should take concrete parameters in type var ranges" + ) } Ok(Ok((unifier.add_ty(TypeEnum::TVirtual { ty: ty.0 }), true))) } else { - return Ok(Err(format!("virtual class needs exactly 1 type parameters, found {}", args.len()))) + return Ok(Err(format!( + "virtual class needs exactly 1 type parameters, found {}", + args.len() + ))); } } - _ => unimplemented!() + _ => unimplemented!(), } } else if ty_id == self.primitive_ids.virtual_id { - Ok(Ok(({ - let ty = TypeEnum::TVirtual { ty: unifier.get_fresh_var().0 }; - unifier.add_ty(ty) - }, false))) + Ok(Ok(( + { + let ty = TypeEnum::TVirtual { + ty: unifier.get_fresh_var().0, + }; + unifier.add_ty(ty) + }, + false, + ))) } else { Ok(Err("unknown type".into())) } @@ -366,10 +434,18 @@ impl InnerResolver { let (extracted_ty, inst_check) = match self.get_pyty_obj_type( py, { - if [self.primitive_ids.typevar, + if [ + self.primitive_ids.typevar, self.primitive_ids.generic_alias.0, - self.primitive_ids.generic_alias.1 - ].contains(&self.helper.id_fn.call1(py, (ty.clone(),))?.extract::(py)?) { + self.primitive_ids.generic_alias.1, + ] + .contains( + &self + .helper + .id_fn + .call1(py, (ty.clone(),))? + .extract::(py)?, + ) { obj } else { ty.as_ref(py) @@ -377,10 +453,10 @@ impl InnerResolver { }, unifier, defs, - primitives + primitives, )? { Ok(s) => s, - Err(_) => return Ok(None) + Err(_) => return Ok(None), }; return match (&*unifier.get_ty(extracted_ty), inst_check) { // do the instantiation for these three types @@ -394,8 +470,8 @@ impl InnerResolver { )); Ok(Some(extracted_ty)) } else { - let actual_ty = self - .get_list_elem_type(py, obj, len, unifier, defs, primitives)?; + let actual_ty = + self.get_list_elem_type(py, obj, len, unifier, defs, primitives)?; if let Some(actual_ty) = actual_ty { unifier.unify(*ty, actual_ty).unwrap(); Ok(Some(extracted_ty)) @@ -429,14 +505,14 @@ impl InnerResolver { // loop through non-function fields of the class to get the instantiated value for field in fields.borrow().iter() { let name: String = (*field.0).into(); - if let TypeEnum::TFunc( .. ) = &*unifier.get_ty(field.1.0) { + if let TypeEnum::TFunc(..) = &*unifier.get_ty(field.1 .0) { continue; } else { let field_data = obj.getattr(&name)?; let ty = self .get_obj_type(py, field_data, unifier, defs, primitives)? .unwrap_or(primitives.none); - let field_ty = unifier.subst(field.1.0, &var_map).unwrap_or(field.1.0); + let field_ty = unifier.subst(field.1 .0, &var_map).unwrap_or(field.1 .0); if unifier.unify(ty, field_ty).is_err() { // field type mismatch return Ok(None); @@ -446,12 +522,16 @@ impl InnerResolver { for (_, ty) in var_map.iter() { // must be concrete type if !unifier.is_concrete(*ty, &[]) { - return Ok(None) + return Ok(None); } } - return Ok(Some(unifier.subst(extracted_ty, &var_map).unwrap_or(extracted_ty))); + return Ok(Some( + unifier + .subst(extracted_ty, &var_map) + .unwrap_or(extracted_ty), + )); } - _ => Ok(Some(extracted_ty)) + _ => Ok(Some(extracted_ty)), }; } @@ -466,23 +546,40 @@ impl InnerResolver { .id_fn .call1(py, (self.helper.type_fn.call1(py, (obj,))?,))? .extract(py)?; + let id: u64 = self.helper.id_fn.call1(py, (obj,))?.extract(py)?; if ty_id == self.primitive_ids.int || ty_id == self.primitive_ids.int32 { let val: i32 = obj.extract()?; + self.id_to_primitive + .write() + .insert(id, PrimitiveValue::I32(val)); Ok(Some(ctx.ctx.i32_type().const_int(val as u64, false).into())) } else if ty_id == self.primitive_ids.int64 { let val: i64 = obj.extract()?; + self.id_to_primitive + .write() + .insert(id, PrimitiveValue::I64(val)); Ok(Some(ctx.ctx.i64_type().const_int(val as u64, false).into())) } else if ty_id == self.primitive_ids.bool { let val: bool = obj.extract()?; + self.id_to_primitive + .write() + .insert(id, PrimitiveValue::Bool(val)); Ok(Some( ctx.ctx.bool_type().const_int(val as u64, false).into(), )) } else if ty_id == self.primitive_ids.float { let val: f64 = obj.extract()?; + self.id_to_primitive + .write() + .insert(id, PrimitiveValue::F64(val)); Ok(Some(ctx.ctx.f64_type().const_float(val).into())) } else if ty_id == self.primitive_ids.list { - let id: u64 = self.helper.id_fn.call1(py, (obj,))?.extract(py)?; let id_str = id.to_string(); + + if let Some(global) = ctx.module.get_global(&id_str) { + return Ok(Some(global.as_pointer_value().into())); + } + let len: usize = self.helper.len_fn.call1(py, (obj,))?.extract(py)?; let ty = if len == 0 { ctx.primitives.int32 @@ -507,15 +604,14 @@ impl InnerResolver { ); { - let mut global_value_ids = self.global_value_ids.lock(); - if global_value_ids.contains(&id) { + if self.global_value_ids.read().contains(&id) { let global = ctx.module.get_global(&id_str).unwrap_or_else(|| { ctx.module .add_global(arr_ty, Some(AddressSpace::Generic), &id_str) }); return Ok(Some(global.as_pointer_value().into())); } else { - global_value_ids.insert(id); + self.global_value_ids.write().insert(id); } } @@ -583,8 +679,12 @@ impl InnerResolver { Ok(Some(global.as_pointer_value().into())) } else if ty_id == self.primitive_ids.tuple { - let id: u64 = self.helper.id_fn.call1(py, (obj,))?.extract(py)?; let id_str = id.to_string(); + + if let Some(global) = ctx.module.get_global(&id_str) { + return Ok(Some(global.as_pointer_value().into())); + } + let elements: &PyTuple = obj.cast_as()?; let types: Result>, _> = elements .iter() @@ -603,15 +703,14 @@ impl InnerResolver { let ty = ctx.ctx.struct_type(&types, false); { - let mut global_value_ids = self.global_value_ids.lock(); - if global_value_ids.contains(&id) { + if self.global_value_ids.read().contains(&id) { let global = ctx.module.get_global(&id_str).unwrap_or_else(|| { ctx.module .add_global(ty, Some(AddressSpace::Generic), &id_str) }); return Ok(Some(global.as_pointer_value().into())); } else { - global_value_ids.insert(id); + self.global_value_ids.write().insert(id); } } @@ -627,8 +726,12 @@ impl InnerResolver { global.set_initializer(&val); Ok(Some(global.as_pointer_value().into())) } else { - let id: u64 = self.helper.id_fn.call1(py, (obj,))?.extract(py)?; let id_str = id.to_string(); + + if let Some(global) = ctx.module.get_global(&id_str) { + return Ok(Some(global.as_pointer_value().into())); + } + let top_level_defs = ctx.top_level.definitions.read(); let ty = self .get_obj_type(py, obj, &mut ctx.unifier, &top_level_defs, &ctx.primitives)? @@ -639,16 +742,16 @@ impl InnerResolver { .get_element_type() .into_struct_type() .as_basic_type_enum(); + { - let mut global_value_ids = self.global_value_ids.lock(); - if global_value_ids.contains(&id) { + if self.global_value_ids.read().contains(&id) { let global = ctx.module.get_global(&id_str).unwrap_or_else(|| { ctx.module .add_global(ty, Some(AddressSpace::Generic), &id_str) }); return Ok(Some(global.as_pointer_value().into())); } else { - global_value_ids.insert(id); + self.global_value_ids.write().insert(id); } } // should be classes @@ -727,14 +830,10 @@ impl SymbolResolver for Resolver { ast::ExprKind::Name { id, .. } => { Python::with_gil(|py| -> PyResult> { let obj: &PyAny = self.0.module.extract(py)?; - let members: &PyList = PyModule::import(py, "inspect")? - .getattr("getmembers")? - .call1((obj,))? - .cast_as()?; + let members: &PyDict = obj.getattr("__dict__").unwrap().cast_as().unwrap(); let mut sym_value = None; - for member in members.iter() { - let key: &str = member.get_item(0)?.extract()?; - let val = member.get_item(1)?; + for (key, val) in members.iter() { + let key: &str = key.extract()?; if key == id.to_string() { sym_value = Some( self.0 @@ -760,38 +859,40 @@ impl SymbolResolver for Resolver { primitives: &PrimitiveStore, str: StrRef, ) -> Option { - let mut id_to_type = self.0.id_to_type.lock(); - id_to_type.get(&str).cloned().or_else(|| { + { + let id_to_type = self.0.id_to_type.read(); + id_to_type.get(&str).cloned() + } + .or_else(|| { let py_id = self.0.name_to_pyid.get(&str); let result = py_id.and_then(|id| { - self.0.pyid_to_type.read().get(id).copied().or_else(|| { - Python::with_gil(|py| -> PyResult> { + { + let pyid_to_type = self.0.pyid_to_type.read(); + pyid_to_type.get(id).copied() + } + .or_else(|| { + let result = Python::with_gil(|py| -> PyResult> { let obj: &PyAny = self.0.module.extract(py)?; - let members: &PyList = PyModule::import(py, "inspect")? - .getattr("getmembers")? - .call1((obj,))? - .cast_as()?; let mut sym_ty = None; - for member in members.iter() { - let key: &str = member.get_item(0)?.extract()?; + let members: &PyDict = obj.getattr("__dict__").unwrap().cast_as().unwrap(); + for (key, val) in members.iter() { + let key: &str = key.extract()?; if key == str.to_string() { - sym_ty = self.0.get_obj_type( - py, - member.get_item(1)?, - unifier, - defs, - primitives, - )?; + sym_ty = self.0.get_obj_type(py, val, unifier, defs, primitives)?; break; } } Ok(sym_ty) }) - .unwrap() + .unwrap(); + if let Some(result) = result { + self.0.pyid_to_type.write().insert(*id, result); + } + result }) }); if let Some(result) = &result { - id_to_type.insert(str, *result); + self.0.id_to_type.write().insert(str, *result); } result }) @@ -802,29 +903,37 @@ impl SymbolResolver for Resolver { id: StrRef, _: &mut CodeGenContext<'ctx, 'a>, ) -> Option> { - Python::with_gil(|py| -> PyResult>> { - let obj: &PyAny = self.0.module.extract(py)?; - let members: &PyList = PyModule::import(py, "inspect")? - .getattr("getmembers")? - .call1((obj,))? - .cast_as()?; - let mut sym_value = None; - for member in members.iter() { - let key: &str = member.get_item(0)?.extract()?; - let val = member.get_item(1)?; - if key == id.to_string() { - let id = self.0.helper.id_fn.call1(py, (val,))?.extract(py)?; - sym_value = Some(PythonValue { - id, - value: val.extract()?, - resolver: self.0.clone(), - }); - break; + let sym_value = { + let id_to_val = self.0.id_to_pyval.read(); + id_to_val.get(&id).cloned() + } + .or_else(|| { + Python::with_gil(|py| -> PyResult> { + let obj: &PyAny = self.0.module.extract(py)?; + let mut sym_value: Option<(u64, PyObject)> = None; + let members: &PyDict = obj.getattr("__dict__").unwrap().cast_as().unwrap(); + for (key, val) in members.iter() { + let key: &str = key.extract()?; + if key == id.to_string() { + let id = self.0.helper.id_fn.call1(py, (val,))?.extract(py)?; + sym_value = Some((id, val.extract()?)); + break; + } } - } - Ok(sym_value.map(|v| ValueEnum::Static(Arc::new(v)))) + if let Some((pyid, val)) = &sym_value { + self.0.id_to_pyval.write().insert(id, (*pyid, val.clone())); + } + Ok(sym_value) + }) + .unwrap() + }); + sym_value.map(|(id, v)| { + ValueEnum::Static(Arc::new(PythonValue { + id, + value: v, + resolver: self.0.clone(), + })) }) - .unwrap() } fn get_symbol_location(&self, _: StrRef) -> Option { @@ -832,14 +941,17 @@ impl SymbolResolver for Resolver { } fn get_identifier_def(&self, id: StrRef) -> Option { - let mut id_to_def = self.0.id_to_def.lock(); - id_to_def.get(&id).cloned().or_else(|| { + { + let id_to_def = self.0.id_to_def.read(); + id_to_def.get(&id).cloned() + } + .or_else(|| { let py_id = self.0.name_to_pyid.get(&id); let result = py_id.and_then(|id| self.0.pyid_to_def.read().get(id).copied()); if let Some(result) = &result { - id_to_def.insert(id, *result); + self.0.id_to_def.write().insert(id, *result); } result }) } -} \ No newline at end of file +} From ddb4c548ae8d6a2bb6a8c6e69515cb4903c2eb4c Mon Sep 17 00:00:00 2001 From: Sebastien Bourdeauducq Date: Wed, 8 Dec 2021 16:55:25 +0800 Subject: [PATCH 36/54] add and use local copy of LLVM Nix files Modifications accumulate and many are not suitable for nixpkgs upstream. Based on nixpkgs 3f629e3dd5293bd3c211c4950c418f7cfb4501af --- flake.nix | 27 +- llvm-future-riscv-abi.diff | 61 ---- llvm-mingw-crosscompile.diff | 35 --- llvm-restrict-targets.diff | 12 - llvm-unbreak-static-cross.diff | 471 ------------------------------ llvm/default.nix | 242 +++++++++++++++ llvm/gnu-install-dirs-polly.patch | 105 +++++++ llvm/gnu-install-dirs.patch | 417 ++++++++++++++++++++++++++ llvm/llvm-future-riscv-abi.diff | 28 ++ llvm/outputs.patch | 16 + 10 files changed, 818 insertions(+), 596 deletions(-) delete mode 100644 llvm-future-riscv-abi.diff delete mode 100644 llvm-mingw-crosscompile.diff delete mode 100644 llvm-restrict-targets.diff delete mode 100644 llvm-unbreak-static-cross.diff create mode 100644 llvm/default.nix create mode 100644 llvm/gnu-install-dirs-polly.patch create mode 100644 llvm/gnu-install-dirs.patch create mode 100644 llvm/llvm-future-riscv-abi.diff create mode 100644 llvm/outputs.patch diff --git a/flake.nix b/flake.nix index a4d84b97..7b7198d9 100644 --- a/flake.nix +++ b/flake.nix @@ -5,15 +5,8 @@ outputs = { self, nixpkgs }: let - # We can't use overlays because llvm dependencies are handled internally in llvmPackages_xx - pkgs-orig = import nixpkgs { system = "x86_64-linux"; }; - nixpkgs-patched = pkgs-orig.applyPatches { - name = "nixpkgs"; - src = nixpkgs; - patches = [ ./llvm-future-riscv-abi.diff ./llvm-restrict-targets.diff ./llvm-mingw-crosscompile.diff ./llvm-unbreak-static-cross.diff ]; - }; - pkgs = import nixpkgs-patched { system = "x86_64-linux"; }; - pkgs-mingw = import nixpkgs-patched { system = "x86_64-linux"; crossSystem = { config = "x86_64-w64-mingw32"; libc = "msvcrt"; }; }; + pkgs = import nixpkgs { system = "x86_64-linux"; }; + pkgs-mingw = import nixpkgs { system = "x86_64-linux"; crossSystem = { config = "x86_64-w64-mingw32"; libc = "msvcrt"; }; }; cargoSha256 = "sha256-otKLhr58HYMjVXAof6AdObNpggPnvK6qOl7I+4LWIP8="; msys2-python-tar = pkgs.fetchurl { url = "https://mirror.msys2.org/mingw/mingw64/mingw-w64-x86_64-python-3.9.7-4-any.pkg.tar.zst"; @@ -46,16 +39,15 @@ ''; }; in rec { - inherit nixpkgs-patched; - - packages.x86_64-linux = { + packages.x86_64-linux = rec { + llvm-nac3 = pkgs.callPackage "${self}/llvm" {}; nac3artiq = pkgs.python3Packages.toPythonModule ( pkgs.rustPlatform.buildRustPackage { name = "nac3artiq"; src = self; inherit cargoSha256; - nativeBuildInputs = [ pkgs.python3 pkgs.llvm_12 ]; - buildInputs = [ pkgs.python3 pkgs.libffi pkgs.libxml2 pkgs.llvm_12 ]; + nativeBuildInputs = [ pkgs.python3 llvm-nac3 ]; + buildInputs = [ pkgs.python3 pkgs.libffi pkgs.libxml2 llvm-nac3 ]; cargoBuildFlags = [ "--package" "nac3artiq" ]; cargoTestFlags = [ "--package" "nac3ast" "--package" "nac3parser" "--package" "nac3core" "--package" "nac3artiq" ]; installPhase = @@ -68,7 +60,8 @@ ); }; - packages.x86_64-w64-mingw32 = { + packages.x86_64-w64-mingw32 = rec { + llvm-nac3 = pkgs-mingw.callPackage "${self}/llvm" { inherit (pkgs) llvmPackages_12; }; nac3artiq = pkgs-mingw.python3Packages.toPythonModule ( pkgs-mingw.rustPlatform.buildRustPackage { name = "nac3artiq"; @@ -85,7 +78,7 @@ #!${pkgs.bash}/bin/bash set -e # Gross hack to work around llvm-config asking for the wrong system libraries. - exec ${pkgs-mingw.llvm_12.dev}/bin/llvm-config-native \$@ | ${pkgs.gnused}/bin/sed s/-lrt\ -ldl\ -lpthread\ -lm// + exec ${llvm-nac3.dev}/bin/llvm-config-native \$@ | ${pkgs.gnused}/bin/sed s/-lrt\ -ldl\ -lpthread\ -lm// EOF chmod +x llvm-cfg/llvm-config export PATH=$PATH:`pwd`/llvm-cfg @@ -110,7 +103,7 @@ devShell.x86_64-linux = pkgs.mkShell { name = "nac3-dev-shell"; buildInputs = with pkgs; [ - llvm_12 + packages.x86_64-linux.llvm-nac3 clang_12 lld_12 cargo diff --git a/llvm-future-riscv-abi.diff b/llvm-future-riscv-abi.diff deleted file mode 100644 index ffe9ecb2..00000000 --- a/llvm-future-riscv-abi.diff +++ /dev/null @@ -1,61 +0,0 @@ -commit 6e2dea56207b4e52ade9d1eee6a4f198336dd0a6 -Author: Sebastien Bourdeauducq -Date: Thu Nov 11 23:32:13 2021 +0800 - - llvm: switch RISC-V ABI when FPU is present - -diff --git a/pkgs/development/compilers/llvm/12/llvm/default.nix b/pkgs/development/compilers/llvm/12/llvm/default.nix -index 30a1a7a16df..41b7211b2a5 100644 ---- a/pkgs/development/compilers/llvm/12/llvm/default.nix -+++ b/pkgs/development/compilers/llvm/12/llvm/default.nix -@@ -66,6 +66,7 @@ in stdenv.mkDerivation (rec { - sha256 = "sha256:12s8vr6ibri8b48h2z38f3afhwam10arfiqfy4yg37bmc054p5hi"; - stripLen = 1; - }) -+ ./llvm-future-riscv-abi.diff - ] ++ lib.optional enablePolly ./gnu-install-dirs-polly.patch; - - postPatch = optionalString stdenv.isDarwin '' -@@ -183,7 +184,7 @@ in stdenv.mkDerivation (rec { - cp NATIVE/bin/llvm-config $dev/bin/llvm-config-native - ''; - -- doCheck = stdenv.isLinux && (!stdenv.isx86_32) && (!stdenv.hostPlatform.isMusl); -+ doCheck = false; # the ABI change breaks RISC-V FP tests - - checkTarget = "check-all"; - -diff --git a/pkgs/development/compilers/llvm/12/llvm/llvm-future-riscv-abi.diff b/pkgs/development/compilers/llvm/12/llvm/llvm-future-riscv-abi.diff -new file mode 100644 -index 00000000000..2427ed0e02c ---- /dev/null -+++ b/pkgs/development/compilers/llvm/12/llvm/llvm-future-riscv-abi.diff -@@ -0,0 +1,28 @@ -+diff --git a/lib/Target/RISCV/MCTargetDesc/RISCVBaseInfo.cpp b/lib/Target/RISCV/MCTargetDesc/RISCVBaseInfo.cpp -+index 0aba18b20..9bb75e7f4 100644 -+--- a/lib/Target/RISCV/MCTargetDesc/RISCVBaseInfo.cpp -++++ b/lib/Target/RISCV/MCTargetDesc/RISCVBaseInfo.cpp -+@@ -33,6 +33,8 @@ ABI computeTargetABI(const Triple &TT, FeatureBitset FeatureBits, -+ auto TargetABI = getTargetABI(ABIName); -+ bool IsRV64 = TT.isArch64Bit(); -+ bool IsRV32E = FeatureBits[RISCV::FeatureRV32E]; -++ bool IsRV32D = FeatureBits[RISCV::FeatureStdExtD]; -++ bool IsRV32F = FeatureBits[RISCV::FeatureStdExtF]; -+ -+ if (!ABIName.empty() && TargetABI == ABI_Unknown) { -+ errs() -+@@ -56,10 +58,10 @@ ABI computeTargetABI(const Triple &TT, FeatureBitset FeatureBits, -+ if (TargetABI != ABI_Unknown) -+ return TargetABI; -+ -+- // For now, default to the ilp32/ilp32e/lp64 ABI if no explicit ABI is given -+- // or an invalid/unrecognised string is given. In the future, it might be -+- // worth changing this to default to ilp32f/lp64f and ilp32d/lp64d when -+- // hardware support for floating point is present. -++ if (IsRV32D) -++ return ABI_ILP32D; -++ if (IsRV32F) -++ return ABI_ILP32F; -+ if (IsRV32E) -+ return ABI_ILP32E; -+ if (IsRV64) diff --git a/llvm-mingw-crosscompile.diff b/llvm-mingw-crosscompile.diff deleted file mode 100644 index 09fc6599..00000000 --- a/llvm-mingw-crosscompile.diff +++ /dev/null @@ -1,35 +0,0 @@ -diff --git a/pkgs/development/compilers/llvm/12/llvm/default.nix b/pkgs/development/compilers/llvm/12/llvm/default.nix -index 30a1a7a16df..4f9435d1819 100644 ---- a/pkgs/development/compilers/llvm/12/llvm/default.nix -+++ b/pkgs/development/compilers/llvm/12/llvm/default.nix -@@ -15,10 +15,11 @@ - , buildLlvmTools - , debugVersion ? false - , enableManpages ? false --, enableSharedLibraries ? !stdenv.hostPlatform.isStatic -+, enableSharedLibraries ? (!stdenv.hostPlatform.isStatic && !stdenv.targetPlatform.isMinGW) - , enablePFM ? !(stdenv.isDarwin - || stdenv.isAarch64 # broken for Ampere eMAG 8180 (c2.large.arm on Packet) #56245 - || stdenv.isAarch32 # broken for the armv7l builder -+ || stdenv.targetPlatform.isMinGW - ) - , enablePolly ? false - }: -@@ -120,7 +121,7 @@ in stdenv.mkDerivation (rec { - "-DLLVM_INSTALL_CMAKE_DIR=${placeholder "dev"}/lib/cmake/llvm/" - "-DCMAKE_BUILD_TYPE=${if debugVersion then "Debug" else "Release"}" - "-DLLVM_INSTALL_UTILS=ON" # Needed by rustc -- "-DLLVM_BUILD_TESTS=ON" -+ "-DLLVM_BUILD_TESTS=${if stdenv.targetPlatform.isMinGW then "OFF" else "ON"}" - "-DLLVM_ENABLE_FFI=ON" - "-DLLVM_ENABLE_RTTI=ON" - "-DLLVM_HOST_TRIPLE=${stdenv.hostPlatform.config}" -@@ -134,7 +135,7 @@ in stdenv.mkDerivation (rec { - "-DSPHINX_OUTPUT_MAN=ON" - "-DSPHINX_OUTPUT_HTML=OFF" - "-DSPHINX_WARNINGS_AS_ERRORS=OFF" -- ] ++ optionals (!isDarwin) [ -+ ] ++ optionals (!isDarwin && !stdenv.targetPlatform.isMinGW) [ - "-DLLVM_BINUTILS_INCDIR=${libbfd.dev}/include" - ] ++ optionals isDarwin [ - "-DLLVM_ENABLE_LIBCXX=ON" diff --git a/llvm-restrict-targets.diff b/llvm-restrict-targets.diff deleted file mode 100644 index 6500c9c5..00000000 --- a/llvm-restrict-targets.diff +++ /dev/null @@ -1,12 +0,0 @@ -diff --git a/pkgs/development/compilers/llvm/12/llvm/default.nix b/pkgs/development/compilers/llvm/12/llvm/default.nix -index 41b7211b2a5..dfc707f034d 100644 ---- a/pkgs/development/compilers/llvm/12/llvm/default.nix -+++ b/pkgs/development/compilers/llvm/12/llvm/default.nix -@@ -127,6 +127,7 @@ in stdenv.mkDerivation (rec { - "-DLLVM_HOST_TRIPLE=${stdenv.hostPlatform.config}" - "-DLLVM_DEFAULT_TARGET_TRIPLE=${stdenv.hostPlatform.config}" - "-DLLVM_ENABLE_DUMP=ON" -+ "-DLLVM_TARGETS_TO_BUILD=X86;ARM;RISCV" - ] ++ optionals enableSharedLibraries [ - "-DLLVM_LINK_LLVM_DYLIB=ON" - ] ++ optionals enableManpages [ diff --git a/llvm-unbreak-static-cross.diff b/llvm-unbreak-static-cross.diff deleted file mode 100644 index 6ab65020..00000000 --- a/llvm-unbreak-static-cross.diff +++ /dev/null @@ -1,471 +0,0 @@ -diff --git a/pkgs/development/compilers/llvm/10/llvm/default.nix b/pkgs/development/compilers/llvm/10/llvm/default.nix -index 4e590dba4337d..413de2f868a3e 100644 ---- a/pkgs/development/compilers/llvm/10/llvm/default.nix -+++ b/pkgs/development/compilers/llvm/10/llvm/default.nix -@@ -81,7 +81,7 @@ in stdenv.mkDerivation (rec { - --replace 'set(_install_rpath "@loader_path/../''${CMAKE_INSTALL_LIBDIR}''${LLVM_LIBDIR_SUFFIX}" ''${extra_libdir})' "" - '' - # Patch llvm-config to return correct library path based on --link-{shared,static}. -- + optionalString (enableSharedLibraries) '' -+ + '' - substitute '${./outputs.patch}' ./outputs.patch --subst-var lib - patch -p1 < ./outputs.patch - '' + '' -diff --git a/pkgs/development/compilers/llvm/10/llvm/outputs.patch b/pkgs/development/compilers/llvm/10/llvm/outputs.patch -index 40096fa3497fd..878460e05b8af 100644 ---- a/pkgs/development/compilers/llvm/10/llvm/outputs.patch -+++ b/pkgs/development/compilers/llvm/10/llvm/outputs.patch -@@ -2,23 +2,13 @@ diff --git a/tools/llvm-config/llvm-config.cpp b/tools/llvm-config/llvm-config.c - index 94d426b..37f7794 100644 - --- a/tools/llvm-config/llvm-config.cpp - +++ b/tools/llvm-config/llvm-config.cpp --@@ -333,6 +333,21 @@ int main(int argc, char **argv) { -+@@ -333,6 +333,11 @@ int main(int argc, char **argv) { - ActiveIncludeOption = "-I" + ActiveIncludeDir; - } - --+ /// Nix-specific multiple-output handling: override ActiveLibDir if --link-shared -++ /// Nix-specific multiple-output handling: override ActiveLibDir - + if (!IsInDevelopmentTree) { --+ bool WantShared = true; --+ for (int i = 1; i < argc; ++i) { --+ StringRef Arg = argv[i]; --+ if (Arg == "--link-shared") --+ WantShared = true; --+ else if (Arg == "--link-static") --+ WantShared = false; // the last one wins --+ } --+ --+ if (WantShared) --+ ActiveLibDir = std::string("@lib@") + "/lib" + LLVM_LIBDIR_SUFFIX; -++ ActiveLibDir = std::string("@lib@") + "/lib" + LLVM_LIBDIR_SUFFIX; - + } - + - /// We only use `shared library` mode in cases where the static library form -diff --git a/pkgs/development/compilers/llvm/11/llvm/default.nix b/pkgs/development/compilers/llvm/11/llvm/default.nix -index f0148850dbe8a..32981b9d4dde8 100644 ---- a/pkgs/development/compilers/llvm/11/llvm/default.nix -+++ b/pkgs/development/compilers/llvm/11/llvm/default.nix -@@ -73,7 +73,7 @@ in stdenv.mkDerivation (rec { - --replace 'set(_install_rpath "@loader_path/../''${CMAKE_INSTALL_LIBDIR}''${LLVM_LIBDIR_SUFFIX}" ''${extra_libdir})' "" - '' - # Patch llvm-config to return correct library path based on --link-{shared,static}. -- + optionalString (enableSharedLibraries) '' -+ + '' - substitute '${./outputs.patch}' ./outputs.patch --subst-var lib - patch -p1 < ./outputs.patch - '' + '' -diff --git a/pkgs/development/compilers/llvm/11/llvm/outputs.patch b/pkgs/development/compilers/llvm/11/llvm/outputs.patch -index 40096fa3497fd..878460e05b8af 100644 ---- a/pkgs/development/compilers/llvm/11/llvm/outputs.patch -+++ b/pkgs/development/compilers/llvm/11/llvm/outputs.patch -@@ -2,23 +2,13 @@ diff --git a/tools/llvm-config/llvm-config.cpp b/tools/llvm-config/llvm-config.c - index 94d426b..37f7794 100644 - --- a/tools/llvm-config/llvm-config.cpp - +++ b/tools/llvm-config/llvm-config.cpp --@@ -333,6 +333,21 @@ int main(int argc, char **argv) { -+@@ -333,6 +333,11 @@ int main(int argc, char **argv) { - ActiveIncludeOption = "-I" + ActiveIncludeDir; - } - --+ /// Nix-specific multiple-output handling: override ActiveLibDir if --link-shared -++ /// Nix-specific multiple-output handling: override ActiveLibDir - + if (!IsInDevelopmentTree) { --+ bool WantShared = true; --+ for (int i = 1; i < argc; ++i) { --+ StringRef Arg = argv[i]; --+ if (Arg == "--link-shared") --+ WantShared = true; --+ else if (Arg == "--link-static") --+ WantShared = false; // the last one wins --+ } --+ --+ if (WantShared) --+ ActiveLibDir = std::string("@lib@") + "/lib" + LLVM_LIBDIR_SUFFIX; -++ ActiveLibDir = std::string("@lib@") + "/lib" + LLVM_LIBDIR_SUFFIX; - + } - + - /// We only use `shared library` mode in cases where the static library form -diff --git a/pkgs/development/compilers/llvm/12/llvm/default.nix b/pkgs/development/compilers/llvm/12/llvm/default.nix -index 30a1a7a16df83..bb5676b9d48e4 100644 ---- a/pkgs/development/compilers/llvm/12/llvm/default.nix -+++ b/pkgs/development/compilers/llvm/12/llvm/default.nix -@@ -74,7 +74,7 @@ in stdenv.mkDerivation (rec { - --replace 'set(_install_rpath "@loader_path/../''${CMAKE_INSTALL_LIBDIR}''${LLVM_LIBDIR_SUFFIX}" ''${extra_libdir})' "" - '' - # Patch llvm-config to return correct library path based on --link-{shared,static}. -- + optionalString (enableSharedLibraries) '' -+ + '' - substitute '${./outputs.patch}' ./outputs.patch --subst-var lib - patch -p1 < ./outputs.patch - '' + '' -diff --git a/pkgs/development/compilers/llvm/12/llvm/outputs.patch b/pkgs/development/compilers/llvm/12/llvm/outputs.patch -index 40096fa3497fd..878460e05b8af 100644 ---- a/pkgs/development/compilers/llvm/12/llvm/outputs.patch -+++ b/pkgs/development/compilers/llvm/12/llvm/outputs.patch -@@ -2,23 +2,13 @@ diff --git a/tools/llvm-config/llvm-config.cpp b/tools/llvm-config/llvm-config.c - index 94d426b..37f7794 100644 - --- a/tools/llvm-config/llvm-config.cpp - +++ b/tools/llvm-config/llvm-config.cpp --@@ -333,6 +333,21 @@ int main(int argc, char **argv) { -+@@ -333,6 +333,11 @@ int main(int argc, char **argv) { - ActiveIncludeOption = "-I" + ActiveIncludeDir; - } - --+ /// Nix-specific multiple-output handling: override ActiveLibDir if --link-shared -++ /// Nix-specific multiple-output handling: override ActiveLibDir - + if (!IsInDevelopmentTree) { --+ bool WantShared = true; --+ for (int i = 1; i < argc; ++i) { --+ StringRef Arg = argv[i]; --+ if (Arg == "--link-shared") --+ WantShared = true; --+ else if (Arg == "--link-static") --+ WantShared = false; // the last one wins --+ } --+ --+ if (WantShared) --+ ActiveLibDir = std::string("@lib@") + "/lib" + LLVM_LIBDIR_SUFFIX; -++ ActiveLibDir = std::string("@lib@") + "/lib" + LLVM_LIBDIR_SUFFIX; - + } - + - /// We only use `shared library` mode in cases where the static library form -diff --git a/pkgs/development/compilers/llvm/13/llvm/default.nix b/pkgs/development/compilers/llvm/13/llvm/default.nix -index 957f29e44994a..115b56396e8d8 100644 ---- a/pkgs/development/compilers/llvm/13/llvm/default.nix -+++ b/pkgs/development/compilers/llvm/13/llvm/default.nix -@@ -68,7 +68,7 @@ in stdenv.mkDerivation (rec { - --replace 'set(_install_rpath "@loader_path/../''${CMAKE_INSTALL_LIBDIR}''${LLVM_LIBDIR_SUFFIX}" ''${extra_libdir})' "" - '' - # Patch llvm-config to return correct library path based on --link-{shared,static}. -- + optionalString (enableSharedLibraries) '' -+ + '' - substitute '${./outputs.patch}' ./outputs.patch --subst-var lib - patch -p1 < ./outputs.patch - '' + '' -diff --git a/pkgs/development/compilers/llvm/13/llvm/outputs.patch b/pkgs/development/compilers/llvm/13/llvm/outputs.patch -index 40096fa3497fd..878460e05b8af 100644 ---- a/pkgs/development/compilers/llvm/13/llvm/outputs.patch -+++ b/pkgs/development/compilers/llvm/13/llvm/outputs.patch -@@ -2,23 +2,13 @@ diff --git a/tools/llvm-config/llvm-config.cpp b/tools/llvm-config/llvm-config.c - index 94d426b..37f7794 100644 - --- a/tools/llvm-config/llvm-config.cpp - +++ b/tools/llvm-config/llvm-config.cpp --@@ -333,6 +333,21 @@ int main(int argc, char **argv) { -+@@ -333,6 +333,11 @@ int main(int argc, char **argv) { - ActiveIncludeOption = "-I" + ActiveIncludeDir; - } - --+ /// Nix-specific multiple-output handling: override ActiveLibDir if --link-shared -++ /// Nix-specific multiple-output handling: override ActiveLibDir - + if (!IsInDevelopmentTree) { --+ bool WantShared = true; --+ for (int i = 1; i < argc; ++i) { --+ StringRef Arg = argv[i]; --+ if (Arg == "--link-shared") --+ WantShared = true; --+ else if (Arg == "--link-static") --+ WantShared = false; // the last one wins --+ } --+ --+ if (WantShared) --+ ActiveLibDir = std::string("@lib@") + "/lib" + LLVM_LIBDIR_SUFFIX; -++ ActiveLibDir = std::string("@lib@") + "/lib" + LLVM_LIBDIR_SUFFIX; - + } - + - /// We only use `shared library` mode in cases where the static library form -diff --git a/pkgs/development/compilers/llvm/5/llvm/default.nix b/pkgs/development/compilers/llvm/5/llvm/default.nix -index 6388cd65fbf47..a4d2fec36193d 100644 ---- a/pkgs/development/compilers/llvm/5/llvm/default.nix -+++ b/pkgs/development/compilers/llvm/5/llvm/default.nix -@@ -74,7 +74,7 @@ stdenv.mkDerivation ({ - --replace 'set(_install_rpath "@loader_path/../''${CMAKE_INSTALL_LIBDIR}" ''${extra_libdir})' "" - '' - # Patch llvm-config to return correct library path based on --link-{shared,static}. -- + optionalString (enableSharedLibraries) '' -+ + '' - substitute '${./outputs.patch}' ./outputs.patch --subst-var lib - patch -p1 < ./outputs.patch - '' + '' -diff --git a/pkgs/development/compilers/llvm/5/llvm/outputs.patch b/pkgs/development/compilers/llvm/5/llvm/outputs.patch -index 40096fa3497fd..878460e05b8af 100644 ---- a/pkgs/development/compilers/llvm/5/llvm/outputs.patch -+++ b/pkgs/development/compilers/llvm/5/llvm/outputs.patch -@@ -2,23 +2,13 @@ diff --git a/tools/llvm-config/llvm-config.cpp b/tools/llvm-config/llvm-config.c - index 94d426b..37f7794 100644 - --- a/tools/llvm-config/llvm-config.cpp - +++ b/tools/llvm-config/llvm-config.cpp --@@ -333,6 +333,21 @@ int main(int argc, char **argv) { -+@@ -333,6 +333,11 @@ int main(int argc, char **argv) { - ActiveIncludeOption = "-I" + ActiveIncludeDir; - } - --+ /// Nix-specific multiple-output handling: override ActiveLibDir if --link-shared -++ /// Nix-specific multiple-output handling: override ActiveLibDir - + if (!IsInDevelopmentTree) { --+ bool WantShared = true; --+ for (int i = 1; i < argc; ++i) { --+ StringRef Arg = argv[i]; --+ if (Arg == "--link-shared") --+ WantShared = true; --+ else if (Arg == "--link-static") --+ WantShared = false; // the last one wins --+ } --+ --+ if (WantShared) --+ ActiveLibDir = std::string("@lib@") + "/lib" + LLVM_LIBDIR_SUFFIX; -++ ActiveLibDir = std::string("@lib@") + "/lib" + LLVM_LIBDIR_SUFFIX; - + } - + - /// We only use `shared library` mode in cases where the static library form -diff --git a/pkgs/development/compilers/llvm/6/llvm/default.nix b/pkgs/development/compilers/llvm/6/llvm/default.nix -index 0907c89561717..1c010980048cd 100644 ---- a/pkgs/development/compilers/llvm/6/llvm/default.nix -+++ b/pkgs/development/compilers/llvm/6/llvm/default.nix -@@ -72,7 +72,7 @@ stdenv.mkDerivation ({ - --replace 'set(_install_rpath "@loader_path/../''${CMAKE_INSTALL_LIBDIR}" ''${extra_libdir})' "" - '' - # Patch llvm-config to return correct library path based on --link-{shared,static}. -- + optionalString (enableSharedLibraries) '' -+ + '' - substitute '${./outputs.patch}' ./outputs.patch --subst-var lib - patch -p1 < ./outputs.patch - '' + '' -diff --git a/pkgs/development/compilers/llvm/6/llvm/outputs.patch b/pkgs/development/compilers/llvm/6/llvm/outputs.patch -index 40096fa3497fd..878460e05b8af 100644 ---- a/pkgs/development/compilers/llvm/6/llvm/outputs.patch -+++ b/pkgs/development/compilers/llvm/6/llvm/outputs.patch -@@ -2,23 +2,13 @@ diff --git a/tools/llvm-config/llvm-config.cpp b/tools/llvm-config/llvm-config.c - index 94d426b..37f7794 100644 - --- a/tools/llvm-config/llvm-config.cpp - +++ b/tools/llvm-config/llvm-config.cpp --@@ -333,6 +333,21 @@ int main(int argc, char **argv) { -+@@ -333,6 +333,11 @@ int main(int argc, char **argv) { - ActiveIncludeOption = "-I" + ActiveIncludeDir; - } - --+ /// Nix-specific multiple-output handling: override ActiveLibDir if --link-shared -++ /// Nix-specific multiple-output handling: override ActiveLibDir - + if (!IsInDevelopmentTree) { --+ bool WantShared = true; --+ for (int i = 1; i < argc; ++i) { --+ StringRef Arg = argv[i]; --+ if (Arg == "--link-shared") --+ WantShared = true; --+ else if (Arg == "--link-static") --+ WantShared = false; // the last one wins --+ } --+ --+ if (WantShared) --+ ActiveLibDir = std::string("@lib@") + "/lib" + LLVM_LIBDIR_SUFFIX; -++ ActiveLibDir = std::string("@lib@") + "/lib" + LLVM_LIBDIR_SUFFIX; - + } - + - /// We only use `shared library` mode in cases where the static library form -diff --git a/pkgs/development/compilers/llvm/7/llvm/default.nix b/pkgs/development/compilers/llvm/7/llvm/default.nix -index 4a9b4f5182056..ac6efaccb865e 100644 ---- a/pkgs/development/compilers/llvm/7/llvm/default.nix -+++ b/pkgs/development/compilers/llvm/7/llvm/default.nix -@@ -76,7 +76,7 @@ in stdenv.mkDerivation ({ - --replace 'set(_install_rpath "@loader_path/../''${CMAKE_INSTALL_LIBDIR}" ''${extra_libdir})' "" - '' - # Patch llvm-config to return correct library path based on --link-{shared,static}. -- + optionalString (enableSharedLibraries) '' -+ + '' - substitute '${./outputs.patch}' ./outputs.patch --subst-var lib - patch -p1 < ./outputs.patch - '' + '' -diff --git a/pkgs/development/compilers/llvm/7/llvm/outputs.patch b/pkgs/development/compilers/llvm/7/llvm/outputs.patch -index 40096fa3497fd..878460e05b8af 100644 ---- a/pkgs/development/compilers/llvm/7/llvm/outputs.patch -+++ b/pkgs/development/compilers/llvm/7/llvm/outputs.patch -@@ -2,23 +2,13 @@ diff --git a/tools/llvm-config/llvm-config.cpp b/tools/llvm-config/llvm-config.c - index 94d426b..37f7794 100644 - --- a/tools/llvm-config/llvm-config.cpp - +++ b/tools/llvm-config/llvm-config.cpp --@@ -333,6 +333,21 @@ int main(int argc, char **argv) { -+@@ -333,6 +333,11 @@ int main(int argc, char **argv) { - ActiveIncludeOption = "-I" + ActiveIncludeDir; - } - --+ /// Nix-specific multiple-output handling: override ActiveLibDir if --link-shared -++ /// Nix-specific multiple-output handling: override ActiveLibDir - + if (!IsInDevelopmentTree) { --+ bool WantShared = true; --+ for (int i = 1; i < argc; ++i) { --+ StringRef Arg = argv[i]; --+ if (Arg == "--link-shared") --+ WantShared = true; --+ else if (Arg == "--link-static") --+ WantShared = false; // the last one wins --+ } --+ --+ if (WantShared) --+ ActiveLibDir = std::string("@lib@") + "/lib" + LLVM_LIBDIR_SUFFIX; -++ ActiveLibDir = std::string("@lib@") + "/lib" + LLVM_LIBDIR_SUFFIX; - + } - + - /// We only use `shared library` mode in cases where the static library form -diff --git a/pkgs/development/compilers/llvm/8/llvm/default.nix b/pkgs/development/compilers/llvm/8/llvm/default.nix -index 83b779b56d827..48a434b24c7cd 100644 ---- a/pkgs/development/compilers/llvm/8/llvm/default.nix -+++ b/pkgs/development/compilers/llvm/8/llvm/default.nix -@@ -79,7 +79,7 @@ in stdenv.mkDerivation ({ - --replace 'set(_install_rpath "@loader_path/../''${CMAKE_INSTALL_LIBDIR}" ''${extra_libdir})' "" - '' - # Patch llvm-config to return correct library path based on --link-{shared,static}. -- + optionalString (enableSharedLibraries) '' -+ + '' - substitute '${./outputs.patch}' ./outputs.patch --subst-var lib - patch -p1 < ./outputs.patch - '' + '' -diff --git a/pkgs/development/compilers/llvm/8/llvm/outputs.patch b/pkgs/development/compilers/llvm/8/llvm/outputs.patch -index 40096fa3497fd..878460e05b8af 100644 ---- a/pkgs/development/compilers/llvm/8/llvm/outputs.patch -+++ b/pkgs/development/compilers/llvm/8/llvm/outputs.patch -@@ -2,23 +2,13 @@ diff --git a/tools/llvm-config/llvm-config.cpp b/tools/llvm-config/llvm-config.c - index 94d426b..37f7794 100644 - --- a/tools/llvm-config/llvm-config.cpp - +++ b/tools/llvm-config/llvm-config.cpp --@@ -333,6 +333,21 @@ int main(int argc, char **argv) { -+@@ -333,6 +333,11 @@ int main(int argc, char **argv) { - ActiveIncludeOption = "-I" + ActiveIncludeDir; - } - --+ /// Nix-specific multiple-output handling: override ActiveLibDir if --link-shared -++ /// Nix-specific multiple-output handling: override ActiveLibDir - + if (!IsInDevelopmentTree) { --+ bool WantShared = true; --+ for (int i = 1; i < argc; ++i) { --+ StringRef Arg = argv[i]; --+ if (Arg == "--link-shared") --+ WantShared = true; --+ else if (Arg == "--link-static") --+ WantShared = false; // the last one wins --+ } --+ --+ if (WantShared) --+ ActiveLibDir = std::string("@lib@") + "/lib" + LLVM_LIBDIR_SUFFIX; -++ ActiveLibDir = std::string("@lib@") + "/lib" + LLVM_LIBDIR_SUFFIX; - + } - + - /// We only use `shared library` mode in cases where the static library form -diff --git a/pkgs/development/compilers/llvm/9/llvm/default.nix b/pkgs/development/compilers/llvm/9/llvm/default.nix -index 570795824ee58..a569c00071de0 100644 ---- a/pkgs/development/compilers/llvm/9/llvm/default.nix -+++ b/pkgs/development/compilers/llvm/9/llvm/default.nix -@@ -77,7 +77,7 @@ in stdenv.mkDerivation (rec { - --replace 'set(_install_rpath "@loader_path/../''${CMAKE_INSTALL_LIBDIR}" ''${extra_libdir})' "" - '' - # Patch llvm-config to return correct library path based on --link-{shared,static}. -- + optionalString (enableSharedLibraries) '' -+ + '' - substitute '${./outputs.patch}' ./outputs.patch --subst-var lib - patch -p1 < ./outputs.patch - '' + '' -diff --git a/pkgs/development/compilers/llvm/9/llvm/outputs.patch b/pkgs/development/compilers/llvm/9/llvm/outputs.patch -index 40096fa3497fd..878460e05b8af 100644 ---- a/pkgs/development/compilers/llvm/9/llvm/outputs.patch -+++ b/pkgs/development/compilers/llvm/9/llvm/outputs.patch -@@ -2,23 +2,13 @@ diff --git a/tools/llvm-config/llvm-config.cpp b/tools/llvm-config/llvm-config.c - index 94d426b..37f7794 100644 - --- a/tools/llvm-config/llvm-config.cpp - +++ b/tools/llvm-config/llvm-config.cpp --@@ -333,6 +333,21 @@ int main(int argc, char **argv) { -+@@ -333,6 +333,11 @@ int main(int argc, char **argv) { - ActiveIncludeOption = "-I" + ActiveIncludeDir; - } - --+ /// Nix-specific multiple-output handling: override ActiveLibDir if --link-shared -++ /// Nix-specific multiple-output handling: override ActiveLibDir - + if (!IsInDevelopmentTree) { --+ bool WantShared = true; --+ for (int i = 1; i < argc; ++i) { --+ StringRef Arg = argv[i]; --+ if (Arg == "--link-shared") --+ WantShared = true; --+ else if (Arg == "--link-static") --+ WantShared = false; // the last one wins --+ } --+ --+ if (WantShared) --+ ActiveLibDir = std::string("@lib@") + "/lib" + LLVM_LIBDIR_SUFFIX; -++ ActiveLibDir = std::string("@lib@") + "/lib" + LLVM_LIBDIR_SUFFIX; - + } - + - /// We only use `shared library` mode in cases where the static library form -diff --git a/pkgs/development/compilers/llvm/git/llvm/default.nix b/pkgs/development/compilers/llvm/git/llvm/default.nix -index daf4cfe808b6c..4c895f81dbbac 100644 ---- a/pkgs/development/compilers/llvm/git/llvm/default.nix -+++ b/pkgs/development/compilers/llvm/git/llvm/default.nix -@@ -60,7 +60,7 @@ in stdenv.mkDerivation (rec { - --replace 'set(_install_rpath "@loader_path/../''${CMAKE_INSTALL_LIBDIR}''${LLVM_LIBDIR_SUFFIX}" ''${extra_libdir})' "" - '' - # Patch llvm-config to return correct library path based on --link-{shared,static}. -- + optionalString (enableSharedLibraries) '' -+ + '' - substitute '${./outputs.patch}' ./outputs.patch --subst-var lib - patch -p1 < ./outputs.patch - '' + '' -diff --git a/pkgs/development/compilers/llvm/git/llvm/outputs.patch b/pkgs/development/compilers/llvm/git/llvm/outputs.patch -index 40096fa3497fd..878460e05b8af 100644 ---- a/pkgs/development/compilers/llvm/git/llvm/outputs.patch -+++ b/pkgs/development/compilers/llvm/git/llvm/outputs.patch -@@ -2,23 +2,13 @@ diff --git a/tools/llvm-config/llvm-config.cpp b/tools/llvm-config/llvm-config.c - index 94d426b..37f7794 100644 - --- a/tools/llvm-config/llvm-config.cpp - +++ b/tools/llvm-config/llvm-config.cpp --@@ -333,6 +333,21 @@ int main(int argc, char **argv) { -+@@ -333,6 +333,11 @@ int main(int argc, char **argv) { - ActiveIncludeOption = "-I" + ActiveIncludeDir; - } - --+ /// Nix-specific multiple-output handling: override ActiveLibDir if --link-shared -++ /// Nix-specific multiple-output handling: override ActiveLibDir - + if (!IsInDevelopmentTree) { --+ bool WantShared = true; --+ for (int i = 1; i < argc; ++i) { --+ StringRef Arg = argv[i]; --+ if (Arg == "--link-shared") --+ WantShared = true; --+ else if (Arg == "--link-static") --+ WantShared = false; // the last one wins --+ } --+ --+ if (WantShared) --+ ActiveLibDir = std::string("@lib@") + "/lib" + LLVM_LIBDIR_SUFFIX; -++ ActiveLibDir = std::string("@lib@") + "/lib" + LLVM_LIBDIR_SUFFIX; - + } - + - /// We only use `shared library` mode in cases where the static library form -diff --git a/pkgs/development/compilers/llvm/rocm/llvm/outputs.patch b/pkgs/development/compilers/llvm/rocm/llvm/outputs.patch -index 40096fa3497fd..878460e05b8af 100644 ---- a/pkgs/development/compilers/llvm/rocm/llvm/outputs.patch -+++ b/pkgs/development/compilers/llvm/rocm/llvm/outputs.patch -@@ -2,23 +2,13 @@ diff --git a/tools/llvm-config/llvm-config.cpp b/tools/llvm-config/llvm-config.c - index 94d426b..37f7794 100644 - --- a/tools/llvm-config/llvm-config.cpp - +++ b/tools/llvm-config/llvm-config.cpp --@@ -333,6 +333,21 @@ int main(int argc, char **argv) { -+@@ -333,6 +333,11 @@ int main(int argc, char **argv) { - ActiveIncludeOption = "-I" + ActiveIncludeDir; - } - --+ /// Nix-specific multiple-output handling: override ActiveLibDir if --link-shared -++ /// Nix-specific multiple-output handling: override ActiveLibDir - + if (!IsInDevelopmentTree) { --+ bool WantShared = true; --+ for (int i = 1; i < argc; ++i) { --+ StringRef Arg = argv[i]; --+ if (Arg == "--link-shared") --+ WantShared = true; --+ else if (Arg == "--link-static") --+ WantShared = false; // the last one wins --+ } --+ --+ if (WantShared) --+ ActiveLibDir = std::string("@lib@") + "/lib" + LLVM_LIBDIR_SUFFIX; -++ ActiveLibDir = std::string("@lib@") + "/lib" + LLVM_LIBDIR_SUFFIX; - + } - + - /// We only use `shared library` mode in cases where the static library form diff --git a/llvm/default.nix b/llvm/default.nix new file mode 100644 index 00000000..8f65dfda --- /dev/null +++ b/llvm/default.nix @@ -0,0 +1,242 @@ +{ lib, stdenv +, pkgsBuildBuild +, fetchurl +, fetchpatch +, cmake +, python3 +, libffi +, libbfd +, libpfm +, libxml2 +, ncurses +, zlib +, llvmPackages_12 +, debugVersion ? false +, enableManpages ? false +, enableSharedLibraries ? (!stdenv.hostPlatform.isStatic && !stdenv.targetPlatform.isMinGW) +, enablePFM ? !(stdenv.isDarwin + || stdenv.isAarch64 # broken for Ampere eMAG 8180 (c2.large.arm on Packet) #56245 + || stdenv.isAarch32 # broken for the armv7l builder + || stdenv.targetPlatform.isMinGW +) +, enablePolly ? false +}: + +let + inherit (lib) optional optionals optionalString; + + release_version = "12.0.1"; + candidate = ""; # empty or "rcN" + dash-candidate = lib.optionalString (candidate != "") "-${candidate}"; + version = "${release_version}${dash-candidate}"; # differentiating these (variables) is important for RCs + fetch = name: sha256: fetchurl { + url = "https://github.com/llvm/llvm-project/releases/download/llvmorg-${version}/${name}-${release_version}${candidate}.src.tar.xz"; + inherit sha256; + }; + + # Used when creating a version-suffixed symlink of libLLVM.dylib + shortVersion = with lib; + concatStringsSep "." (take 1 (splitString "." release_version)); + +in stdenv.mkDerivation (rec { + pname = "llvm"; + inherit version; + + src = fetch pname "1pzx9zrmd7r3481sbhwvkms68fwhffpp4mmz45dgrkjpyl2q96kx"; + polly_src = fetch "polly" "1yfm9ixda4a2sx7ak5vswijx4ydk5lv1c1xh39xmd2kh299y4m12"; + + unpackPhase = '' + unpackFile $src + mv llvm-${release_version}* llvm + sourceRoot=$PWD/llvm + '' + optionalString enablePolly '' + unpackFile $polly_src + mv polly-* $sourceRoot/tools/polly + ''; + + outputs = [ "out" "lib" "dev" "python" ]; + + nativeBuildInputs = [ cmake python3 ] + ++ optionals enableManpages [ python3.pkgs.sphinx python3.pkgs.recommonmark ]; + + buildInputs = [ libxml2 libffi ] + ++ optional enablePFM libpfm; # exegesis + + propagatedBuildInputs = optionals (stdenv.buildPlatform == stdenv.hostPlatform) [ ncurses ] + ++ [ zlib ]; + + patches = [ + ./gnu-install-dirs.patch + # On older CPUs (e.g. Hydra/wendy) we'd be getting an error in this test. + (fetchpatch { + name = "uops-CMOV16rm-noreg.diff"; + url = "https://github.com/llvm/llvm-project/commit/9e9f991ac033.diff"; + sha256 = "sha256:12s8vr6ibri8b48h2z38f3afhwam10arfiqfy4yg37bmc054p5hi"; + stripLen = 1; + }) + ./llvm-future-riscv-abi.diff + ] ++ lib.optional enablePolly ./gnu-install-dirs-polly.patch; + + postPatch = optionalString stdenv.isDarwin '' + substituteInPlace cmake/modules/AddLLVM.cmake \ + --replace 'set(_install_name_dir INSTALL_NAME_DIR "@rpath")' "set(_install_name_dir)" \ + --replace 'set(_install_rpath "@loader_path/../''${CMAKE_INSTALL_LIBDIR}''${LLVM_LIBDIR_SUFFIX}" ''${extra_libdir})' "" + '' + # Patch llvm-config to return correct library path based on --link-{shared,static}. + + '' + substitute '${./outputs.patch}' ./outputs.patch --subst-var lib + patch -p1 < ./outputs.patch + '' + '' + # FileSystem permissions tests fail with various special bits + substituteInPlace unittests/Support/CMakeLists.txt \ + --replace "Path.cpp" "" + rm unittests/Support/Path.cpp + substituteInPlace unittests/IR/CMakeLists.txt \ + --replace "PassBuilderCallbacksTest.cpp" "" + rm unittests/IR/PassBuilderCallbacksTest.cpp + # TODO: Fix failing tests: + rm test/DebugInfo/X86/vla-multi.ll + '' + optionalString stdenv.hostPlatform.isMusl '' + patch -p1 -i ${../../TLI-musl.patch} + substituteInPlace unittests/Support/CMakeLists.txt \ + --replace "add_subdirectory(DynamicLibrary)" "" + rm unittests/Support/DynamicLibrary/DynamicLibraryTest.cpp + # valgrind unhappy with musl or glibc, but fails w/musl only + rm test/CodeGen/AArch64/wineh4.mir + '' + optionalString stdenv.hostPlatform.isAarch32 '' + # skip failing X86 test cases on 32-bit ARM + rm test/DebugInfo/X86/convert-debugloc.ll + rm test/DebugInfo/X86/convert-inlined.ll + rm test/DebugInfo/X86/convert-linked.ll + rm test/tools/dsymutil/X86/op-convert.test + '' + optionalString (stdenv.hostPlatform.system == "armv6l-linux") '' + # Seems to require certain floating point hardware (NEON?) + rm test/ExecutionEngine/frem.ll + '' + '' + patchShebangs test/BugPoint/compile-custom.ll.py + ''; + + # hacky fix: created binaries need to be run before installation + preBuild = '' + mkdir -p $out/ + ln -sv $PWD/lib $out + ''; + + # E.g. mesa.drivers use the build-id as a cache key (see #93946): + LDFLAGS = optionalString (enableSharedLibraries && !stdenv.isDarwin) "-Wl,--build-id=sha1"; + + cmakeFlags = with stdenv; [ + "-DLLVM_INSTALL_CMAKE_DIR=${placeholder "dev"}/lib/cmake/llvm/" + "-DCMAKE_BUILD_TYPE=${if debugVersion then "Debug" else "Release"}" + "-DLLVM_INSTALL_UTILS=ON" # Needed by rustc + "-DLLVM_BUILD_TESTS=${if stdenv.targetPlatform.isMinGW then "OFF" else "ON"}" + "-DLLVM_ENABLE_FFI=ON" + "-DLLVM_ENABLE_RTTI=ON" + "-DLLVM_HOST_TRIPLE=${stdenv.hostPlatform.config}" + "-DLLVM_DEFAULT_TARGET_TRIPLE=${stdenv.hostPlatform.config}" + "-DLLVM_ENABLE_DUMP=ON" + "-DLLVM_TARGETS_TO_BUILD=X86;ARM;RISCV" + ] ++ optionals enableSharedLibraries [ + "-DLLVM_LINK_LLVM_DYLIB=ON" + ] ++ optionals enableManpages [ + "-DLLVM_BUILD_DOCS=ON" + "-DLLVM_ENABLE_SPHINX=ON" + "-DSPHINX_OUTPUT_MAN=ON" + "-DSPHINX_OUTPUT_HTML=OFF" + "-DSPHINX_WARNINGS_AS_ERRORS=OFF" + ] ++ optionals (!isDarwin && !stdenv.targetPlatform.isMinGW) [ + "-DLLVM_BINUTILS_INCDIR=${libbfd.dev}/include" + ] ++ optionals isDarwin [ + "-DLLVM_ENABLE_LIBCXX=ON" + "-DCAN_TARGET_i386=false" + ] ++ optionals (stdenv.hostPlatform != stdenv.buildPlatform) [ + "-DCMAKE_CROSSCOMPILING=True" + "-DLLVM_TABLEGEN=${llvmPackages_12.tools.llvm}/bin/llvm-tblgen" + ( + let + nativeCC = pkgsBuildBuild.targetPackages.stdenv.cc; + nativeBintools = nativeCC.bintools.bintools; + nativeToolchainFlags = [ + "-DCMAKE_C_COMPILER=${nativeCC}/bin/${nativeCC.targetPrefix}cc" + "-DCMAKE_CXX_COMPILER=${nativeCC}/bin/${nativeCC.targetPrefix}c++" + "-DCMAKE_AR=${nativeBintools}/bin/${nativeBintools.targetPrefix}ar" + "-DCMAKE_STRIP=${nativeBintools}/bin/${nativeBintools.targetPrefix}strip" + "-DCMAKE_RANLIB=${nativeBintools}/bin/${nativeBintools.targetPrefix}ranlib" + ]; + in "-DCROSS_TOOLCHAIN_FLAGS_NATIVE:list=${lib.concatStringsSep ";" nativeToolchainFlags}" + ) + ]; + + postBuild = '' + rm -fR $out + ''; + + preCheck = '' + export LD_LIBRARY_PATH=$LD_LIBRARY_PATH''${LD_LIBRARY_PATH:+:}$PWD/lib + ''; + + postInstall = '' + mkdir -p $python/share + mv $out/share/opt-viewer $python/share/opt-viewer + moveToOutput "bin/llvm-config*" "$dev" + substituteInPlace "$dev/lib/cmake/llvm/LLVMExports-${if debugVersion then "debug" else "release"}.cmake" \ + --replace "\''${_IMPORT_PREFIX}/lib/lib" "$lib/lib/lib" \ + --replace "$out/bin/llvm-config" "$dev/bin/llvm-config" + substituteInPlace "$dev/lib/cmake/llvm/LLVMConfig.cmake" \ + --replace 'set(LLVM_BINARY_DIR "''${LLVM_INSTALL_PREFIX}")' 'set(LLVM_BINARY_DIR "''${LLVM_INSTALL_PREFIX}'"$lib"'")' + '' + + optionalString (stdenv.isDarwin && enableSharedLibraries) '' + ln -s $lib/lib/libLLVM.dylib $lib/lib/libLLVM-${shortVersion}.dylib + ln -s $lib/lib/libLLVM.dylib $lib/lib/libLLVM-${release_version}.dylib + '' + + optionalString (stdenv.buildPlatform != stdenv.hostPlatform) '' + cp NATIVE/bin/llvm-config $dev/bin/llvm-config-native + ''; + + doCheck = false; # the ABI change breaks RISC-V FP tests + + checkTarget = "check-all"; + + requiredSystemFeatures = [ "big-parallel" ]; + meta = { + homepage = "https://llvm.org/"; + description = "A collection of modular and reusable compiler and toolchain technologies"; + longDescription = '' + The LLVM Project is a collection of modular and reusable compiler and + toolchain technologies. Despite its name, LLVM has little to do with + traditional virtual machines. The name "LLVM" itself is not an acronym; it + is the full name of the project. + LLVM began as a research project at the University of Illinois, with the + goal of providing a modern, SSA-based compilation strategy capable of + supporting both static and dynamic compilation of arbitrary programming + languages. Since then, LLVM has grown to be an umbrella project consisting + of a number of subprojects, many of which are being used in production by + a wide variety of commercial and open source projects as well as being + widely used in academic research. Code in the LLVM project is licensed + under the "Apache 2.0 License with LLVM exceptions". + ''; + }; +} // lib.optionalAttrs enableManpages { + pname = "llvm-manpages"; + + buildPhase = '' + make docs-llvm-man + ''; + + propagatedBuildInputs = []; + + installPhase = '' + make -C docs install + ''; + + postPatch = null; + postInstall = null; + + outputs = [ "out" ]; + + doCheck = false; + + meta = { + description = "man pages for LLVM ${version}"; + }; +}) diff --git a/llvm/gnu-install-dirs-polly.patch b/llvm/gnu-install-dirs-polly.patch new file mode 100644 index 00000000..68f3c453 --- /dev/null +++ b/llvm/gnu-install-dirs-polly.patch @@ -0,0 +1,105 @@ +diff --git a/tools/polly/CMakeLists.txt b/tools/polly/CMakeLists.txt +index ca7c04c565bb..6ed5db5dd4f8 100644 +--- a/tools/polly/CMakeLists.txt ++++ b/tools/polly/CMakeLists.txt +@@ -2,7 +2,11 @@ + if (NOT DEFINED LLVM_MAIN_SRC_DIR) + project(Polly) + cmake_minimum_required(VERSION 3.13.4) ++endif() ++ ++include(GNUInstallDirs) + ++if (NOT DEFINED LLVM_MAIN_SRC_DIR) + # Where is LLVM installed? + find_package(LLVM CONFIG REQUIRED) + set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} ${LLVM_CMAKE_DIR}) +@@ -122,13 +126,13 @@ include_directories( + + if (NOT LLVM_INSTALL_TOOLCHAIN_ONLY) + install(DIRECTORY include/ +- DESTINATION include ++ DESTINATION ${CMAKE_INSTALL_INCLUDEDIR} + FILES_MATCHING + PATTERN "*.h" + ) + + install(DIRECTORY ${POLLY_BINARY_DIR}/include/ +- DESTINATION include ++ DESTINATION ${CMAKE_INSTALL_INCLUDEDIR} + FILES_MATCHING + PATTERN "*.h" + PATTERN "CMakeFiles" EXCLUDE +diff --git a/tools/polly/cmake/CMakeLists.txt b/tools/polly/cmake/CMakeLists.txt +index 7cc129ba2e90..137be25e4b80 100644 +--- a/tools/polly/cmake/CMakeLists.txt ++++ b/tools/polly/cmake/CMakeLists.txt +@@ -79,18 +79,18 @@ file(GENERATE + + # Generate PollyConfig.cmake for the install tree. + unset(POLLY_EXPORTS) +-set(POLLY_INSTALL_PREFIX "${CMAKE_INSTALL_PREFIX}") ++set(POLLY_INSTALL_PREFIX "") + set(POLLY_CONFIG_LLVM_CMAKE_DIR "${LLVM_BINARY_DIR}/${LLVM_INSTALL_PACKAGE_DIR}") +-set(POLLY_CONFIG_CMAKE_DIR "${POLLY_INSTALL_PREFIX}/${POLLY_INSTALL_PACKAGE_DIR}") +-set(POLLY_CONFIG_LIBRARY_DIRS "${POLLY_INSTALL_PREFIX}/lib${LLVM_LIBDIR_SUFFIX}") ++set(POLLY_CONFIG_CMAKE_DIR "${POLLY_INSTALL_PREFIX}${CMAKE_INSTALL_PREFIX}/${POLLY_INSTALL_PACKAGE_DIR}") ++set(POLLY_CONFIG_LIBRARY_DIRS "${POLLY_INSTALL_PREFIX}${CMAKE_INSTALL_FULL_LIBDIR}${LLVM_LIBDIR_SUFFIX}") + if (POLLY_BUNDLED_ISL) + set(POLLY_CONFIG_INCLUDE_DIRS +- "${POLLY_INSTALL_PREFIX}/include" +- "${POLLY_INSTALL_PREFIX}/include/polly" ++ "${POLLY_INSTALL_PREFIX}${CMAKE_INSTALL_FULL_LIBDIR}" ++ "${POLLY_INSTALL_PREFIX}${CMAKE_INSTALL_FULL_LIBDIR}/polly" + ) + else() + set(POLLY_CONFIG_INCLUDE_DIRS +- "${POLLY_INSTALL_PREFIX}/include" ++ "${POLLY_INSTALL_PREFIX}${CMAKE_INSTALL_FULL_INCLUDEDIR}" + ${ISL_INCLUDE_DIRS} + ) + endif() +@@ -100,12 +100,12 @@ endif() + foreach(tgt IN LISTS POLLY_CONFIG_EXPORTED_TARGETS) + get_target_property(tgt_type ${tgt} TYPE) + if (tgt_type STREQUAL "EXECUTABLE") +- set(tgt_prefix "bin/") ++ set(tgt_prefix "${CMAKE_INSTALL_BINDIR}/") + else() +- set(tgt_prefix "lib/") ++ set(tgt_prefix "${CMAKE_INSTALL_LIBDIR}/") + endif() + +- set(tgt_path "${CMAKE_INSTALL_PREFIX}/${tgt_prefix}$") ++ set(tgt_path "${tgt_prefix}$") + file(RELATIVE_PATH tgt_path ${POLLY_CONFIG_CMAKE_DIR} ${tgt_path}) + + if (NOT tgt_type STREQUAL "INTERFACE_LIBRARY") +diff --git a/tools/polly/cmake/polly_macros.cmake b/tools/polly/cmake/polly_macros.cmake +index 518a09b45a42..bd9d6f5542ad 100644 +--- a/tools/polly/cmake/polly_macros.cmake ++++ b/tools/polly/cmake/polly_macros.cmake +@@ -44,8 +44,8 @@ macro(add_polly_library name) + if (NOT LLVM_INSTALL_TOOLCHAIN_ONLY OR ${name} STREQUAL "LLVMPolly") + install(TARGETS ${name} + EXPORT LLVMExports +- LIBRARY DESTINATION lib${LLVM_LIBDIR_SUFFIX} +- ARCHIVE DESTINATION lib${LLVM_LIBDIR_SUFFIX}) ++ LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR}${LLVM_LIBDIR_SUFFIX} ++ ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR}${LLVM_LIBDIR_SUFFIX}) + endif() + set_property(GLOBAL APPEND PROPERTY LLVM_EXPORTS ${name}) + endmacro(add_polly_library) +diff --git a/tools/polly/lib/External/CMakeLists.txt b/tools/polly/lib/External/CMakeLists.txt +index 8991094d92c7..178d8ad606bb 100644 +--- a/tools/polly/lib/External/CMakeLists.txt ++++ b/tools/polly/lib/External/CMakeLists.txt +@@ -275,7 +275,7 @@ if (POLLY_BUNDLED_ISL) + install(DIRECTORY + ${ISL_SOURCE_DIR}/include/ + ${ISL_BINARY_DIR}/include/ +- DESTINATION include/polly ++ DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/polly + FILES_MATCHING + PATTERN "*.h" + PATTERN "CMakeFiles" EXCLUDE diff --git a/llvm/gnu-install-dirs.patch b/llvm/gnu-install-dirs.patch new file mode 100644 index 00000000..da8dc144 --- /dev/null +++ b/llvm/gnu-install-dirs.patch @@ -0,0 +1,417 @@ +diff --git a/CMakeLists.txt b/CMakeLists.txt +index 277d0fe54d7b..af69c8be8745 100644 +--- a/CMakeLists.txt ++++ b/CMakeLists.txt +@@ -256,15 +256,21 @@ if (CMAKE_BUILD_TYPE AND + message(FATAL_ERROR "Invalid value for CMAKE_BUILD_TYPE: ${CMAKE_BUILD_TYPE}") + endif() + ++include(GNUInstallDirs) ++ + set(LLVM_LIBDIR_SUFFIX "" CACHE STRING "Define suffix of library directory name (32/64)" ) + +-set(LLVM_TOOLS_INSTALL_DIR "bin" CACHE STRING "Path for binary subdirectory (defaults to 'bin')") ++set(LLVM_TOOLS_INSTALL_DIR "${CMAKE_INSTALL_BINDIR}" CACHE STRING ++ "Path for binary subdirectory (defaults to 'bin')") + mark_as_advanced(LLVM_TOOLS_INSTALL_DIR) + + set(LLVM_UTILS_INSTALL_DIR "${LLVM_TOOLS_INSTALL_DIR}" CACHE STRING + "Path to install LLVM utilities (enabled by LLVM_INSTALL_UTILS=ON) (defaults to LLVM_TOOLS_INSTALL_DIR)") + mark_as_advanced(LLVM_UTILS_INSTALL_DIR) + ++set(LLVM_INSTALL_CMAKE_DIR "${CMAKE_INSTALL_LIBDIR}${LLVM_LIBDIR_SUFFIX}/cmake/llvm" CACHE STRING ++ "Path for CMake subdirectory (defaults to lib/cmake/llvm)" ) ++ + # They are used as destination of target generators. + set(LLVM_RUNTIME_OUTPUT_INTDIR ${CMAKE_CURRENT_BINARY_DIR}/${CMAKE_CFG_INTDIR}/bin) + set(LLVM_LIBRARY_OUTPUT_INTDIR ${CMAKE_CURRENT_BINARY_DIR}/${CMAKE_CFG_INTDIR}/lib${LLVM_LIBDIR_SUFFIX}) +@@ -567,9 +573,9 @@ option (LLVM_ENABLE_SPHINX "Use Sphinx to generate llvm documentation." OFF) + option (LLVM_ENABLE_OCAMLDOC "Build OCaml bindings documentation." ON) + option (LLVM_ENABLE_BINDINGS "Build bindings." ON) + +-set(LLVM_INSTALL_DOXYGEN_HTML_DIR "share/doc/llvm/doxygen-html" ++set(LLVM_INSTALL_DOXYGEN_HTML_DIR "${CMAKE_INSTALL_DOCDIR}/${project}/doxygen-html" + CACHE STRING "Doxygen-generated HTML documentation install directory") +-set(LLVM_INSTALL_OCAMLDOC_HTML_DIR "share/doc/llvm/ocaml-html" ++set(LLVM_INSTALL_OCAMLDOC_HTML_DIR "${CMAKE_INSTALL_DOCDIR}/${project}/ocaml-html" + CACHE STRING "OCamldoc-generated HTML documentation install directory") + + option (LLVM_BUILD_EXTERNAL_COMPILER_RT +@@ -1027,7 +1033,7 @@ endif() + + if (NOT LLVM_INSTALL_TOOLCHAIN_ONLY) + install(DIRECTORY include/llvm include/llvm-c +- DESTINATION include ++ DESTINATION ${CMAKE_INSTALL_INCLUDEDIR} + COMPONENT llvm-headers + FILES_MATCHING + PATTERN "*.def" +@@ -1038,7 +1044,7 @@ if (NOT LLVM_INSTALL_TOOLCHAIN_ONLY) + ) + + install(DIRECTORY ${LLVM_INCLUDE_DIR}/llvm ${LLVM_INCLUDE_DIR}/llvm-c +- DESTINATION include ++ DESTINATION ${CMAKE_INSTALL_INCLUDEDIR} + COMPONENT llvm-headers + FILES_MATCHING + PATTERN "*.def" +@@ -1052,13 +1058,13 @@ if (NOT LLVM_INSTALL_TOOLCHAIN_ONLY) + + if (LLVM_INSTALL_MODULEMAPS) + install(DIRECTORY include/llvm include/llvm-c +- DESTINATION include ++ DESTINATION ${CMAKE_INSTALL_INCLUDEDIR} + COMPONENT llvm-headers + FILES_MATCHING + PATTERN "module.modulemap" + ) + install(FILES include/llvm/module.install.modulemap +- DESTINATION include/llvm ++ DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/llvm + COMPONENT llvm-headers + RENAME "module.extern.modulemap" + ) +diff --git a/cmake/modules/AddLLVM.cmake b/cmake/modules/AddLLVM.cmake +index 97c9980c7de3..409e8b615f75 100644 +--- a/cmake/modules/AddLLVM.cmake ++++ b/cmake/modules/AddLLVM.cmake +@@ -804,9 +804,9 @@ macro(add_llvm_library name) + + install(TARGETS ${name} + ${export_to_llvmexports} +- LIBRARY DESTINATION lib${LLVM_LIBDIR_SUFFIX} COMPONENT ${name} +- ARCHIVE DESTINATION lib${LLVM_LIBDIR_SUFFIX} COMPONENT ${name} +- RUNTIME DESTINATION bin COMPONENT ${name}) ++ LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR}${LLVM_LIBDIR_SUFFIX} COMPONENT ${name} ++ ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR}${LLVM_LIBDIR_SUFFIX} COMPONENT ${name} ++ RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR} COMPONENT ${name}) + + if (NOT LLVM_ENABLE_IDE) + add_llvm_install_targets(install-${name} +@@ -1022,7 +1022,7 @@ function(process_llvm_pass_plugins) + "set(LLVM_STATIC_EXTENSIONS ${LLVM_STATIC_EXTENSIONS})") + install(FILES + ${llvm_cmake_builddir}/LLVMConfigExtensions.cmake +- DESTINATION ${LLVM_INSTALL_PACKAGE_DIR} ++ DESTINATION ${LLVM_INSTALL_CMAKE_DIR} + COMPONENT cmake-exports) + + set(ExtensionDef "${LLVM_BINARY_DIR}/include/llvm/Support/Extension.def") +@@ -1242,7 +1242,7 @@ macro(add_llvm_example name) + endif() + add_llvm_executable(${name} ${ARGN}) + if( LLVM_BUILD_EXAMPLES ) +- install(TARGETS ${name} RUNTIME DESTINATION examples) ++ install(TARGETS ${name} RUNTIME DESTINATION ${CMAKE_INSTALL_DOCDIR}/examples) + endif() + set_target_properties(${name} PROPERTIES FOLDER "Examples") + endmacro(add_llvm_example name) +@@ -1854,7 +1854,7 @@ function(llvm_install_library_symlink name dest type) + set(full_name ${CMAKE_${type}_LIBRARY_PREFIX}${name}${CMAKE_${type}_LIBRARY_SUFFIX}) + set(full_dest ${CMAKE_${type}_LIBRARY_PREFIX}${dest}${CMAKE_${type}_LIBRARY_SUFFIX}) + +- set(output_dir lib${LLVM_LIBDIR_SUFFIX}) ++ set(output_dir ${CMAKE_INSTALL_FULL_LIBDIR}${LLVM_LIBDIR_SUFFIX}) + if(WIN32 AND "${type}" STREQUAL "SHARED") + set(output_dir bin) + endif() +@@ -1871,7 +1871,7 @@ function(llvm_install_library_symlink name dest type) + endif() + endfunction() + +-function(llvm_install_symlink name dest) ++function(llvm_install_symlink name dest output_dir) + cmake_parse_arguments(ARG "ALWAYS_GENERATE" "COMPONENT" "" ${ARGN}) + foreach(path ${CMAKE_MODULE_PATH}) + if(EXISTS ${path}/LLVMInstallSymlink.cmake) +@@ -1894,7 +1894,7 @@ function(llvm_install_symlink name dest) + set(full_dest ${dest}${CMAKE_EXECUTABLE_SUFFIX}) + + install(SCRIPT ${INSTALL_SYMLINK} +- CODE "install_symlink(${full_name} ${full_dest} ${LLVM_TOOLS_INSTALL_DIR})" ++ CODE "install_symlink(${full_name} ${full_dest} ${output_dir})" + COMPONENT ${component}) + + if (NOT LLVM_ENABLE_IDE AND NOT ARG_ALWAYS_GENERATE) +@@ -1977,7 +1977,8 @@ function(add_llvm_tool_symlink link_name target) + endif() + + if ((TOOL_IS_TOOLCHAIN OR NOT LLVM_INSTALL_TOOLCHAIN_ONLY) AND LLVM_BUILD_TOOLS) +- llvm_install_symlink(${link_name} ${target}) ++ GNUInstallDirs_get_absolute_install_dir(output_dir LLVM_TOOLS_INSTALL_DIR) ++ llvm_install_symlink(${link_name} ${target} ${output_dir}) + endif() + endif() + endfunction() +@@ -2100,9 +2101,9 @@ function(llvm_setup_rpath name) + + if (APPLE) + set(_install_name_dir INSTALL_NAME_DIR "@rpath") +- set(_install_rpath "@loader_path/../lib${LLVM_LIBDIR_SUFFIX}" ${extra_libdir}) ++ set(_install_rpath "@loader_path/../${CMAKE_INSTALL_LIBDIR}${LLVM_LIBDIR_SUFFIX}" ${extra_libdir}) + elseif(UNIX) +- set(_install_rpath "\$ORIGIN/../lib${LLVM_LIBDIR_SUFFIX}" ${extra_libdir}) ++ set(_install_rpath "\$ORIGIN/../${CMAKE_INSTALL_LIBDIR}${LLVM_LIBDIR_SUFFIX}" ${extra_libdir}) + if(${CMAKE_SYSTEM_NAME} MATCHES "(FreeBSD|DragonFly)") + set_property(TARGET ${name} APPEND_STRING PROPERTY + LINK_FLAGS " -Wl,-z,origin ") +diff --git a/cmake/modules/AddOCaml.cmake b/cmake/modules/AddOCaml.cmake +index 554046b20edf..4d1ad980641e 100644 +--- a/cmake/modules/AddOCaml.cmake ++++ b/cmake/modules/AddOCaml.cmake +@@ -144,9 +144,9 @@ function(add_ocaml_library name) + endforeach() + + if( APPLE ) +- set(ocaml_rpath "@executable_path/../../../lib${LLVM_LIBDIR_SUFFIX}") ++ set(ocaml_rpath "@executable_path/../../../${CMAKE_INSTALL_LIBDIR}${LLVM_LIBDIR_SUFFIX}") + elseif( UNIX ) +- set(ocaml_rpath "\\$ORIGIN/../../../lib${LLVM_LIBDIR_SUFFIX}") ++ set(ocaml_rpath "\\$ORIGIN/../../../${CMAKE_INSTALL_LIBDIR}${LLVM_LIBDIR_SUFFIX}") + endif() + list(APPEND ocaml_flags "-ldopt" "-Wl,-rpath,${ocaml_rpath}") + +diff --git a/cmake/modules/AddSphinxTarget.cmake b/cmake/modules/AddSphinxTarget.cmake +index e80c3b5c1cac..482f6d715ef5 100644 +--- a/cmake/modules/AddSphinxTarget.cmake ++++ b/cmake/modules/AddSphinxTarget.cmake +@@ -90,7 +90,7 @@ function (add_sphinx_target builder project) + endif() + elseif (builder STREQUAL html) + string(TOUPPER "${project}" project_upper) +- set(${project_upper}_INSTALL_SPHINX_HTML_DIR "share/doc/${project}/html" ++ set(${project_upper}_INSTALL_SPHINX_HTML_DIR "${CMAKE_INSTALL_DOCDIR}/${project}/html" + CACHE STRING "HTML documentation install directory for ${project}") + + # '/.' indicates: copy the contents of the directory directly into +diff --git a/cmake/modules/CMakeLists.txt b/cmake/modules/CMakeLists.txt +index 505dc9a29d70..36e6c63af3f4 100644 +--- a/cmake/modules/CMakeLists.txt ++++ b/cmake/modules/CMakeLists.txt +@@ -1,4 +1,4 @@ +-set(LLVM_INSTALL_PACKAGE_DIR lib${LLVM_LIBDIR_SUFFIX}/cmake/llvm) ++set(LLVM_INSTALL_PACKAGE_DIR ${LLVM_INSTALL_CMAKE_DIR} CACHE STRING "Path for CMake subdirectory (defaults to 'cmake/llvm')") + set(llvm_cmake_builddir "${LLVM_BINARY_DIR}/${LLVM_INSTALL_PACKAGE_DIR}") + + # First for users who use an installed LLVM, create the LLVMExports.cmake file. +@@ -107,13 +107,13 @@ foreach(p ${_count}) + set(LLVM_CONFIG_CODE "${LLVM_CONFIG_CODE} + get_filename_component(LLVM_INSTALL_PREFIX \"\${LLVM_INSTALL_PREFIX}\" PATH)") + endforeach(p) +-set(LLVM_CONFIG_INCLUDE_DIRS "\${LLVM_INSTALL_PREFIX}/include") ++set(LLVM_CONFIG_INCLUDE_DIRS "\${LLVM_INSTALL_PREFIX}/${CMAKE_INSTALL_INCLUDEDIR}") + set(LLVM_CONFIG_INCLUDE_DIR "${LLVM_CONFIG_INCLUDE_DIRS}") + set(LLVM_CONFIG_MAIN_INCLUDE_DIR "${LLVM_CONFIG_INCLUDE_DIRS}") +-set(LLVM_CONFIG_LIBRARY_DIRS "\${LLVM_INSTALL_PREFIX}/lib\${LLVM_LIBDIR_SUFFIX}") ++set(LLVM_CONFIG_LIBRARY_DIRS "\${LLVM_INSTALL_PREFIX}/${CMAKE_INSTALL_LIBDIR}\${LLVM_LIBDIR_SUFFIX}") + set(LLVM_CONFIG_CMAKE_DIR "\${LLVM_INSTALL_PREFIX}/${LLVM_INSTALL_PACKAGE_DIR}") + set(LLVM_CONFIG_BINARY_DIR "\${LLVM_INSTALL_PREFIX}") +-set(LLVM_CONFIG_TOOLS_BINARY_DIR "\${LLVM_INSTALL_PREFIX}/bin") ++set(LLVM_CONFIG_TOOLS_BINARY_DIR "\${LLVM_INSTALL_PREFIX}/${CMAKE_INSTALL_BINDIR}") + + # Generate a default location for lit + if (LLVM_INSTALL_UTILS AND LLVM_BUILD_UTILS) +diff --git a/cmake/modules/LLVMInstallSymlink.cmake b/cmake/modules/LLVMInstallSymlink.cmake +index 09fed8085c23..aa79f192abf0 100644 +--- a/cmake/modules/LLVMInstallSymlink.cmake ++++ b/cmake/modules/LLVMInstallSymlink.cmake +@@ -10,7 +10,7 @@ function(install_symlink name target outdir) + set(LINK_OR_COPY copy) + endif() + +- set(bindir "${DESTDIR}${CMAKE_INSTALL_PREFIX}/${outdir}/") ++ set(bindir "${DESTDIR}${outdir}/") + + message(STATUS "Creating ${name}") + +diff --git a/docs/CMake.rst b/docs/CMake.rst +index bb821b417ad9..6a528f7c2ad3 100644 +--- a/docs/CMake.rst ++++ b/docs/CMake.rst +@@ -196,7 +196,7 @@ CMake manual, or execute ``cmake --help-variable VARIABLE_NAME``. + **LLVM_LIBDIR_SUFFIX**:STRING + Extra suffix to append to the directory where libraries are to be + installed. On a 64-bit architecture, one could use ``-DLLVM_LIBDIR_SUFFIX=64`` +- to install libraries to ``/usr/lib64``. ++ to install libraries to ``/usr/lib64``. See also ``CMAKE_INSTALL_LIBDIR``. + + **CMAKE_C_FLAGS**:STRING + Extra flags to use when compiling C source files. +@@ -550,8 +550,8 @@ LLVM-specific variables + + **LLVM_INSTALL_DOXYGEN_HTML_DIR**:STRING + The path to install Doxygen-generated HTML documentation to. This path can +- either be absolute or relative to the CMAKE_INSTALL_PREFIX. Defaults to +- `share/doc/llvm/doxygen-html`. ++ either be absolute or relative to the ``CMAKE_INSTALL_PREFIX``. Defaults to ++ `${CMAKE_INSTALL_DOCDIR}/${project}/doxygen-html`. + + **LLVM_ENABLE_SPHINX**:BOOL + If specified, CMake will search for the ``sphinx-build`` executable and will make +@@ -582,13 +582,33 @@ LLVM-specific variables + + **LLVM_INSTALL_SPHINX_HTML_DIR**:STRING + The path to install Sphinx-generated HTML documentation to. This path can +- either be absolute or relative to the CMAKE_INSTALL_PREFIX. Defaults to +- `share/doc/llvm/html`. ++ either be absolute or relative to the ``CMAKE_INSTALL_PREFIX``. Defaults to ++ `${CMAKE_INSTALL_DOCDIR}/${project}/html`. + + **LLVM_INSTALL_OCAMLDOC_HTML_DIR**:STRING + The path to install OCamldoc-generated HTML documentation to. This path can +- either be absolute or relative to the CMAKE_INSTALL_PREFIX. Defaults to +- `share/doc/llvm/ocaml-html`. ++ either be absolute or relative to the ``CMAKE_INSTALL_PREFIX``. Defaults to ++ `${CMAKE_INSTALL_DOCDIR}/${project}/ocaml-html`. ++ ++**CMAKE_INSTALL_BINDIR**:STRING ++ The path to install binary tools, relative to the ``CMAKE_INSTALL_PREFIX``. ++ Defaults to `bin`. ++ ++**CMAKE_INSTALL_LIBDIR**:STRING ++ The path to install libraries, relative to the ``CMAKE_INSTALL_PREFIX``. ++ Defaults to `lib`. ++ ++**CMAKE_INSTALL_INCLUDEDIR**:STRING ++ The path to install header files, relative to the ``CMAKE_INSTALL_PREFIX``. ++ Defaults to `include`. ++ ++**CMAKE_INSTALL_DOCDIR**:STRING ++ The path to install documentation, relative to the ``CMAKE_INSTALL_PREFIX``. ++ Defaults to `share/doc`. ++ ++**CMAKE_INSTALL_MANDIR**:STRING ++ The path to install manpage files, relative to the ``CMAKE_INSTALL_PREFIX``. ++ Defaults to `share/man`. + + **LLVM_CREATE_XCODE_TOOLCHAIN**:BOOL + macOS Only: If enabled CMake will generate a target named +@@ -786,9 +806,11 @@ the ``cmake`` command or by setting it directly in ``ccmake`` or ``cmake-gui``). + + This file is available in two different locations. + +-* ``/lib/cmake/llvm/LLVMConfig.cmake`` where +- ```` is the install prefix of an installed version of LLVM. +- On Linux typically this is ``/usr/lib/cmake/llvm/LLVMConfig.cmake``. ++* ``LLVMConfig.cmake`` where ++ ```` is the location where LLVM CMake modules are ++ installed as part of an installed version of LLVM. This is typically ++ ``cmake/llvm/`` within the lib directory. On Linux, this is typically ++ ``/usr/lib/cmake/llvm/LLVMConfig.cmake``. + + * ``/lib/cmake/llvm/LLVMConfig.cmake`` where + ```` is the root of the LLVM build tree. **Note: this is only +diff --git a/examples/Bye/CMakeLists.txt b/examples/Bye/CMakeLists.txt +index bb96edb4b4bf..678c22fb43c8 100644 +--- a/examples/Bye/CMakeLists.txt ++++ b/examples/Bye/CMakeLists.txt +@@ -14,6 +14,6 @@ if (NOT WIN32) + BUILDTREE_ONLY + ) + +- install(TARGETS ${name} RUNTIME DESTINATION examples) ++ install(TARGETS ${name} RUNTIME DESTINATION ${CMAKE_INSTALL_DOCDIR}/examples) + set_target_properties(${name} PROPERTIES FOLDER "Examples") + endif() +diff --git a/include/llvm/CMakeLists.txt b/include/llvm/CMakeLists.txt +index b46319f24fc8..2feabd1954e4 100644 +--- a/include/llvm/CMakeLists.txt ++++ b/include/llvm/CMakeLists.txt +@@ -5,5 +5,5 @@ add_subdirectory(Frontend) + # If we're doing an out-of-tree build, copy a module map for generated + # header files into the build area. + if (NOT "${CMAKE_SOURCE_DIR}" STREQUAL "${CMAKE_BINARY_DIR}") +- configure_file(module.modulemap.build module.modulemap COPYONLY) ++ configure_file(module.modulemap.build ${LLVM_INCLUDE_DIR}/module.modulemap COPYONLY) + endif (NOT "${CMAKE_SOURCE_DIR}" STREQUAL "${CMAKE_BINARY_DIR}") +diff --git a/tools/llvm-config/BuildVariables.inc.in b/tools/llvm-config/BuildVariables.inc.in +index ebe5b73a5c65..70c497be12f5 100644 +--- a/tools/llvm-config/BuildVariables.inc.in ++++ b/tools/llvm-config/BuildVariables.inc.in +@@ -23,6 +23,10 @@ + #define LLVM_CXXFLAGS "@LLVM_CXXFLAGS@" + #define LLVM_BUILDMODE "@LLVM_BUILDMODE@" + #define LLVM_LIBDIR_SUFFIX "@LLVM_LIBDIR_SUFFIX@" ++#define LLVM_INSTALL_BINDIR "@CMAKE_INSTALL_BINDIR@" ++#define LLVM_INSTALL_LIBDIR "@CMAKE_INSTALL_LIBDIR@" ++#define LLVM_INSTALL_INCLUDEDIR "@CMAKE_INSTALL_INCLUDEDIR@" ++#define LLVM_INSTALL_CMAKEDIR "@LLVM_INSTALL_CMAKE_DIR@" + #define LLVM_TARGETS_BUILT "@LLVM_TARGETS_BUILT@" + #define LLVM_SYSTEM_LIBS "@LLVM_SYSTEM_LIBS@" + #define LLVM_BUILD_SYSTEM "@LLVM_BUILD_SYSTEM@" +diff --git a/tools/llvm-config/llvm-config.cpp b/tools/llvm-config/llvm-config.cpp +index 1a2f04552d13..44fa7d3eec6b 100644 +--- a/tools/llvm-config/llvm-config.cpp ++++ b/tools/llvm-config/llvm-config.cpp +@@ -357,12 +357,26 @@ int main(int argc, char **argv) { + ("-I" + ActiveIncludeDir + " " + "-I" + ActiveObjRoot + "/include"); + } else { + ActivePrefix = CurrentExecPrefix; +- ActiveIncludeDir = ActivePrefix + "/include"; +- SmallString<256> path(StringRef(LLVM_TOOLS_INSTALL_DIR)); +- sys::fs::make_absolute(ActivePrefix, path); +- ActiveBinDir = std::string(path.str()); +- ActiveLibDir = ActivePrefix + "/lib" + LLVM_LIBDIR_SUFFIX; +- ActiveCMakeDir = ActiveLibDir + "/cmake/llvm"; ++ { ++ SmallString<256> path(StringRef(LLVM_INSTALL_INCLUDEDIR)); ++ sys::fs::make_absolute(ActivePrefix, path); ++ ActiveIncludeDir = std::string(path.str()); ++ } ++ { ++ SmallString<256> path(StringRef(LLVM_INSTALL_BINDIR)); ++ sys::fs::make_absolute(ActivePrefix, path); ++ ActiveBinDir = std::string(path.str()); ++ } ++ { ++ SmallString<256> path(StringRef(LLVM_INSTALL_LIBDIR LLVM_LIBDIR_SUFFIX)); ++ sys::fs::make_absolute(ActivePrefix, path); ++ ActiveLibDir = std::string(path.str()); ++ } ++ { ++ SmallString<256> path(StringRef(LLVM_INSTALL_CMAKEDIR)); ++ sys::fs::make_absolute(ActivePrefix, path); ++ ActiveCMakeDir = std::string(path.str()); ++ } + ActiveIncludeOption = "-I" + ActiveIncludeDir; + } + +diff --git a/tools/lto/CMakeLists.txt b/tools/lto/CMakeLists.txt +index 2963f97cad88..69d66c9c9ca1 100644 +--- a/tools/lto/CMakeLists.txt ++++ b/tools/lto/CMakeLists.txt +@@ -25,7 +25,7 @@ add_llvm_library(LTO SHARED INSTALL_WITH_TOOLCHAIN ${SOURCES} DEPENDS + intrinsics_gen) + + install(FILES ${LLVM_MAIN_INCLUDE_DIR}/llvm-c/lto.h +- DESTINATION include/llvm-c ++ DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/llvm-c + COMPONENT LTO) + + if (APPLE) +diff --git a/tools/opt-viewer/CMakeLists.txt b/tools/opt-viewer/CMakeLists.txt +index ead73ec13a8f..250362021f17 100644 +--- a/tools/opt-viewer/CMakeLists.txt ++++ b/tools/opt-viewer/CMakeLists.txt +@@ -8,7 +8,7 @@ set (files + + foreach (file ${files}) + install(PROGRAMS ${file} +- DESTINATION share/opt-viewer ++ DESTINATION ${CMAKE_INSTALL_DATADIR}/opt-viewer + COMPONENT opt-viewer) + endforeach (file) + +diff --git a/tools/remarks-shlib/CMakeLists.txt b/tools/remarks-shlib/CMakeLists.txt +index 865436247270..ce1daa62f6ab 100644 +--- a/tools/remarks-shlib/CMakeLists.txt ++++ b/tools/remarks-shlib/CMakeLists.txt +@@ -19,7 +19,7 @@ if(LLVM_ENABLE_PIC) + endif() + + install(FILES ${LLVM_MAIN_INCLUDE_DIR}/llvm-c/Remarks.h +- DESTINATION include/llvm-c ++ DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/llvm-c + COMPONENT Remarks) + + if (APPLE) diff --git a/llvm/llvm-future-riscv-abi.diff b/llvm/llvm-future-riscv-abi.diff new file mode 100644 index 00000000..2427ed0e --- /dev/null +++ b/llvm/llvm-future-riscv-abi.diff @@ -0,0 +1,28 @@ +diff --git a/lib/Target/RISCV/MCTargetDesc/RISCVBaseInfo.cpp b/lib/Target/RISCV/MCTargetDesc/RISCVBaseInfo.cpp +index 0aba18b20..9bb75e7f4 100644 +--- a/lib/Target/RISCV/MCTargetDesc/RISCVBaseInfo.cpp ++++ b/lib/Target/RISCV/MCTargetDesc/RISCVBaseInfo.cpp +@@ -33,6 +33,8 @@ ABI computeTargetABI(const Triple &TT, FeatureBitset FeatureBits, + auto TargetABI = getTargetABI(ABIName); + bool IsRV64 = TT.isArch64Bit(); + bool IsRV32E = FeatureBits[RISCV::FeatureRV32E]; ++ bool IsRV32D = FeatureBits[RISCV::FeatureStdExtD]; ++ bool IsRV32F = FeatureBits[RISCV::FeatureStdExtF]; + + if (!ABIName.empty() && TargetABI == ABI_Unknown) { + errs() +@@ -56,10 +58,10 @@ ABI computeTargetABI(const Triple &TT, FeatureBitset FeatureBits, + if (TargetABI != ABI_Unknown) + return TargetABI; + +- // For now, default to the ilp32/ilp32e/lp64 ABI if no explicit ABI is given +- // or an invalid/unrecognised string is given. In the future, it might be +- // worth changing this to default to ilp32f/lp64f and ilp32d/lp64d when +- // hardware support for floating point is present. ++ if (IsRV32D) ++ return ABI_ILP32D; ++ if (IsRV32F) ++ return ABI_ILP32F; + if (IsRV32E) + return ABI_ILP32E; + if (IsRV64) diff --git a/llvm/outputs.patch b/llvm/outputs.patch new file mode 100644 index 00000000..878460e0 --- /dev/null +++ b/llvm/outputs.patch @@ -0,0 +1,16 @@ +diff --git a/tools/llvm-config/llvm-config.cpp b/tools/llvm-config/llvm-config.cpp +index 94d426b..37f7794 100644 +--- a/tools/llvm-config/llvm-config.cpp ++++ b/tools/llvm-config/llvm-config.cpp +@@ -333,6 +333,11 @@ int main(int argc, char **argv) { + ActiveIncludeOption = "-I" + ActiveIncludeDir; + } + ++ /// Nix-specific multiple-output handling: override ActiveLibDir ++ if (!IsInDevelopmentTree) { ++ ActiveLibDir = std::string("@lib@") + "/lib" + LLVM_LIBDIR_SUFFIX; ++ } ++ + /// We only use `shared library` mode in cases where the static library form + /// of the components provided are not available; note however that this is + /// skipped if we're run from within the build dir. However, once installed, From 3f7389647762417263527b911fd9ce0ec72d93a7 Mon Sep 17 00:00:00 2001 From: Sebastien Bourdeauducq Date: Wed, 8 Dec 2021 17:41:34 +0800 Subject: [PATCH 37/54] remove a small amount of LLVM bloat Also avoids libffi.dll dependency on Windows. --- flake.nix | 6 ++---- llvm/default.nix | 18 +++--------------- nac3artiq/Cargo.toml | 2 +- nac3core/Cargo.toml | 2 +- nac3standalone/Cargo.toml | 2 +- 5 files changed, 8 insertions(+), 22 deletions(-) diff --git a/flake.nix b/flake.nix index 7b7198d9..e61f9f24 100644 --- a/flake.nix +++ b/flake.nix @@ -47,7 +47,7 @@ src = self; inherit cargoSha256; nativeBuildInputs = [ pkgs.python3 llvm-nac3 ]; - buildInputs = [ pkgs.python3 pkgs.libffi pkgs.libxml2 llvm-nac3 ]; + buildInputs = [ pkgs.python3 llvm-nac3 ]; cargoBuildFlags = [ "--package" "nac3artiq" ]; cargoTestFlags = [ "--package" "nac3ast" "--package" "nac3parser" "--package" "nac3core" "--package" "nac3artiq" ]; installPhase = @@ -68,7 +68,7 @@ src = self; inherit cargoSha256; nativeBuildInputs = [ pkgs.zip ]; - buildInputs = [ pkgs-mingw.libffi pkgs-mingw.zlib ]; + buildInputs = [ pkgs-mingw.zlib ]; configurePhase = '' export PYO3_CONFIG_FILE=${pyo3-mingw-config} @@ -109,8 +109,6 @@ cargo cargo-insta rustc - libffi - libxml2 clippy (python3.withPackages(ps: [ ps.numpy ])) ]; diff --git a/llvm/default.nix b/llvm/default.nix index 8f65dfda..ca372677 100644 --- a/llvm/default.nix +++ b/llvm/default.nix @@ -4,21 +4,13 @@ , fetchpatch , cmake , python3 -, libffi , libbfd -, libpfm -, libxml2 , ncurses , zlib , llvmPackages_12 , debugVersion ? false , enableManpages ? false -, enableSharedLibraries ? (!stdenv.hostPlatform.isStatic && !stdenv.targetPlatform.isMinGW) -, enablePFM ? !(stdenv.isDarwin - || stdenv.isAarch64 # broken for Ampere eMAG 8180 (c2.large.arm on Packet) #56245 - || stdenv.isAarch32 # broken for the armv7l builder - || stdenv.targetPlatform.isMinGW -) +, enableSharedLibraries ? false , enablePolly ? false }: @@ -59,8 +51,7 @@ in stdenv.mkDerivation (rec { nativeBuildInputs = [ cmake python3 ] ++ optionals enableManpages [ python3.pkgs.sphinx python3.pkgs.recommonmark ]; - buildInputs = [ libxml2 libffi ] - ++ optional enablePFM libpfm; # exegesis + buildInputs = [ ]; propagatedBuildInputs = optionals (stdenv.buildPlatform == stdenv.hostPlatform) [ ncurses ] ++ [ zlib ]; @@ -128,13 +119,10 @@ in stdenv.mkDerivation (rec { cmakeFlags = with stdenv; [ "-DLLVM_INSTALL_CMAKE_DIR=${placeholder "dev"}/lib/cmake/llvm/" "-DCMAKE_BUILD_TYPE=${if debugVersion then "Debug" else "Release"}" - "-DLLVM_INSTALL_UTILS=ON" # Needed by rustc "-DLLVM_BUILD_TESTS=${if stdenv.targetPlatform.isMinGW then "OFF" else "ON"}" - "-DLLVM_ENABLE_FFI=ON" - "-DLLVM_ENABLE_RTTI=ON" "-DLLVM_HOST_TRIPLE=${stdenv.hostPlatform.config}" "-DLLVM_DEFAULT_TARGET_TRIPLE=${stdenv.hostPlatform.config}" - "-DLLVM_ENABLE_DUMP=ON" + "-DLLVM_ENABLE_UNWIND_TABLES=OFF" "-DLLVM_TARGETS_TO_BUILD=X86;ARM;RISCV" ] ++ optionals enableSharedLibraries [ "-DLLVM_LINK_LLVM_DYLIB=ON" diff --git a/nac3artiq/Cargo.toml b/nac3artiq/Cargo.toml index b527b92e..8b4c8e13 100644 --- a/nac3artiq/Cargo.toml +++ b/nac3artiq/Cargo.toml @@ -19,4 +19,4 @@ nac3core = { path = "../nac3core" } git = "https://github.com/TheDan64/inkwell" branch = "master" default-features = false -features = ["llvm12-0", "target-x86", "target-arm", "target-riscv"] +features = ["llvm12-0", "target-x86", "target-arm", "target-riscv", "no-libffi-linking"] diff --git a/nac3core/Cargo.toml b/nac3core/Cargo.toml index 5aae208f..d69f4c0a 100644 --- a/nac3core/Cargo.toml +++ b/nac3core/Cargo.toml @@ -17,7 +17,7 @@ nac3parser = { path = "../nac3parser" } git = "https://github.com/TheDan64/inkwell" branch = "master" default-features = false -features = ["llvm12-0", "target-x86", "target-arm", "target-riscv"] +features = ["llvm12-0", "target-x86", "target-arm", "target-riscv", "no-libffi-linking"] [dev-dependencies] test-case = "1.2.0" diff --git a/nac3standalone/Cargo.toml b/nac3standalone/Cargo.toml index 41f33069..c538886a 100644 --- a/nac3standalone/Cargo.toml +++ b/nac3standalone/Cargo.toml @@ -13,4 +13,4 @@ nac3core = { path = "../nac3core" } git = "https://github.com/TheDan64/inkwell" branch = "master" default-features = false -features = ["llvm12-0", "target-x86", "target-arm", "target-riscv"] +features = ["llvm12-0", "target-x86", "target-arm", "target-riscv", "no-libffi-linking"] From ae902aac2f7e4cf6fa000e81eb28ee6f39a06357 Mon Sep 17 00:00:00 2001 From: Sebastien Bourdeauducq Date: Wed, 8 Dec 2021 17:43:05 +0800 Subject: [PATCH 38/54] remove devshell inputs from hydraJobs We are not recompiling packages that depend on LLVM anymore, llvm-nac3 is only used for static linking within NAC3. --- flake.nix | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/flake.nix b/flake.nix index e61f9f24..f7be349f 100644 --- a/flake.nix +++ b/flake.nix @@ -115,9 +115,10 @@ }; hydraJobs = { - inherit (packages.x86_64-linux) nac3artiq; + inherit (packages.x86_64-linux) llvm-nac3 nac3artiq; + llvm-nac3-mingw = packages.x86_64-w64-mingw32.llvm-nac3; nac3artiq-mingw = packages.x86_64-w64-mingw32.nac3artiq; - } // (pkgs.lib.foldr (a: b: {"${pkgs.lib.strings.getName a}" = a;} // b) {} devShell.x86_64-linux.buildInputs); + }; }; nixConfig = { From d2ffdeeb4792e28611809fe1aabe103be4dcc51b Mon Sep 17 00:00:00 2001 From: Sebastien Bourdeauducq Date: Wed, 8 Dec 2021 21:21:37 +0800 Subject: [PATCH 39/54] flake: update nixpkgs and work around openssh cross compilation breakage. Closes #123 --- flake.lock | 8 ++++---- flake.nix | 13 +++++++++++-- 2 files changed, 15 insertions(+), 6 deletions(-) diff --git a/flake.lock b/flake.lock index 1930cc3d..b05fe3d9 100644 --- a/flake.lock +++ b/flake.lock @@ -2,17 +2,17 @@ "nodes": { "nixpkgs": { "locked": { - "lastModified": 1637328665, - "narHash": "sha256-z6ufVwquLM0IiNZxd5oT1M33Lv0aB3WICpk8ZKwpxjw=", + "lastModified": 1638887115, + "narHash": "sha256-emjtIeqyJ84Eb3X7APJruTrwcfnHQKs55XGljj62prs=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "0f4b4b85d959200f52c16bbb74036994e7db5f74", + "rev": "1bd4bbd49bef217a3d1adea43498270d6e779d65", "type": "github" }, "original": { "owner": "NixOS", + "ref": "nixos-21.11", "repo": "nixpkgs", - "rev": "0f4b4b85d959200f52c16bbb74036994e7db5f74", "type": "github" } }, diff --git a/flake.nix b/flake.nix index f7be349f..60d18c67 100644 --- a/flake.nix +++ b/flake.nix @@ -1,12 +1,21 @@ { description = "The third-generation ARTIQ compiler"; - inputs.nixpkgs.url = github:NixOS/nixpkgs/0f4b4b85d959200f52c16bbb74036994e7db5f74; + inputs.nixpkgs.url = github:NixOS/nixpkgs/nixos-21.11; outputs = { self, nixpkgs }: let pkgs = import nixpkgs { system = "x86_64-linux"; }; - pkgs-mingw = import nixpkgs { system = "x86_64-linux"; crossSystem = { config = "x86_64-w64-mingw32"; libc = "msvcrt"; }; }; + pkgs-mingw = import nixpkgs { + system = "x86_64-linux"; + crossSystem = { config = "x86_64-w64-mingw32"; libc = "msvcrt"; }; + # work around https://github.com/NixOS/nixpkgs/issues/149593 + overlays = [ + (self: super: { + openssh = super.openssh.overrideAttrs(oa: { doCheck = false; }); + }) + ]; + }; cargoSha256 = "sha256-otKLhr58HYMjVXAof6AdObNpggPnvK6qOl7I+4LWIP8="; msys2-python-tar = pkgs.fetchurl { url = "https://mirror.msys2.org/mingw/mingw64/mingw-w64-x86_64-python-3.9.7-4-any.pkg.tar.zst"; From 01d3249646f91310fb611ec4196ffa0578455023 Mon Sep 17 00:00:00 2001 From: ychenfo Date: Thu, 9 Dec 2021 01:13:34 +0800 Subject: [PATCH 40/54] nac3core: add missing llvm range type --- nac3core/src/codegen/mod.rs | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/nac3core/src/codegen/mod.rs b/nac3core/src/codegen/mod.rs index b01aaf66..3f60baf2 100644 --- a/nac3core/src/codegen/mod.rs +++ b/nac3core/src/codegen/mod.rs @@ -323,6 +323,10 @@ pub fn gen_func<'ctx, G: CodeGenerator + ?Sized>( unifier.get_representative(primitives.str), context.i8_type().ptr_type(AddressSpace::Generic).into(), ), + ( + unifier.get_representative(primitives.range), + context.i32_type().array_type(3).ptr_type(AddressSpace::Generic).into() + ), ] .iter() .cloned() From b7892ce9523ef913a01dfc93fd5cb4f83e2813ae Mon Sep 17 00:00:00 2001 From: ychenfo Date: Thu, 9 Dec 2021 01:14:28 +0800 Subject: [PATCH 41/54] nac3core: add len support for list and range --- nac3core/src/toplevel/builtins.rs | 77 +++++++++++++++++++++++++++++-- 1 file changed, 74 insertions(+), 3 deletions(-) diff --git a/nac3core/src/toplevel/builtins.rs b/nac3core/src/toplevel/builtins.rs index 94b598bc..d9d9045d 100644 --- a/nac3core/src/toplevel/builtins.rs +++ b/nac3core/src/toplevel/builtins.rs @@ -1,6 +1,6 @@ use std::cell::RefCell; -use inkwell::{IntPredicate, FloatPredicate}; -use crate::symbol_resolver::SymbolValue; +use inkwell::{IntPredicate, FloatPredicate, values::BasicValueEnum}; +use crate::{symbol_resolver::SymbolValue, codegen::expr::destructure_range}; use super::*; type BuiltinInfo = ( @@ -557,6 +557,76 @@ pub fn get_builtins(primitives: &mut (PrimitiveStore, Unifier)) -> BuiltinInfo { ) })))), })), + Arc::new(RwLock::new({ + let list_var = primitives.1.get_fresh_var(); + let list = primitives.1.add_ty(TypeEnum::TList { ty: list_var.0 }); + let arg_ty = primitives.1.get_fresh_var_with_range(&[list, primitives.0.range]); + TopLevelDef::Function { + name: "len".into(), + simple_name: "len".into(), + signature: primitives.1.add_ty(TypeEnum::TFunc(RefCell::new(FunSignature { + args: vec![FuncArg { + name: "_".into(), + ty: arg_ty.0, + default_value: None + }], + ret: int32, + vars: vec![(list_var.1, list_var.0), (arg_ty.1, arg_ty.0)].into_iter().collect(), + }))), + var_id: Default::default(), + instance_to_symbol: Default::default(), + instance_to_stmt: Default::default(), + resolver: None, + codegen_callback: Some(Arc::new(GenCall::new(Box::new( + |ctx, _, fun, args| { + let range_ty = ctx.primitives.range; + let arg_ty = fun.0.args[0].ty; + let arg = args[0].1; + let int32 = ctx.ctx.i32_type(); + let zero = int32.const_zero(); + if ctx.unifier.unioned(arg_ty, range_ty) { + let int1 = ctx.ctx.bool_type(); + let one = int32.const_int(1, false); + let falze = int1.const_int(0, false); + let abs_intrinsic = + ctx.module.get_function("llvm.abs.i32").unwrap_or_else(|| { + let fn_type = int32.fn_type(&[int32.into(), int1.into()], false); + ctx.module.add_function("llvm.abs.i32", fn_type, None) + }); + let arg = arg.into_pointer_value(); + let (start, end, step) = destructure_range(ctx, arg); + let diff = ctx.builder.build_int_sub(end, start, "diff"); + let diff = if let BasicValueEnum::IntValue(val) = ctx + .builder + .build_call(abs_intrinsic, &[diff.into(), falze.into()], "absdiff") + .try_as_basic_value() + .left() + .unwrap() { + val + } else { + unreachable!(); + }; + let diff = ctx.builder.build_int_sub(diff, one, "diff"); + let step = if let BasicValueEnum::IntValue(val) = ctx + .builder + .build_call(abs_intrinsic, &[step.into(), falze.into()], "absstep") + .try_as_basic_value() + .left() + .unwrap() { + val + } else { + unreachable!(); + }; + let length = ctx.builder.build_int_signed_div(diff, step, "div"); + let length = ctx.builder.build_int_add(length, int32.const_int(1, false), "add1"); + Some(length.into()) + } else { + Some(ctx.build_gep_and_load(arg.into_pointer_value(), &[zero, zero])) + } + }, + )))), + } + })) ]; let ast_list: Vec>> = (0..top_level_def_list.len()).map(|_| None).collect(); @@ -574,7 +644,8 @@ pub fn get_builtins(primitives: &mut (PrimitiveStore, Unifier)) -> BuiltinInfo { "floor", "floor64", "ceil", - "ceil64" + "ceil64", + "len", ] ) } \ No newline at end of file From 4d2fd9582a61c0d12ebb83373736f2b41b0b4535 Mon Sep 17 00:00:00 2001 From: ychenfo Date: Thu, 9 Dec 2021 01:15:05 +0800 Subject: [PATCH 42/54] nac3core: fix broken tests --- ...oplevel__test__test_analyze__generic_class.snap | 10 +++++----- ...__test__test_analyze__inheritance_override.snap | 14 +++++++------- ...el__test__test_analyze__list_tuple_generic.snap | 10 +++++----- ...3core__toplevel__test__test_analyze__self1.snap | 10 +++++----- ...__test__test_analyze__simple_class_compose.snap | 10 +++++----- 5 files changed, 27 insertions(+), 27 deletions(-) diff --git a/nac3core/src/toplevel/snapshots/nac3core__toplevel__test__test_analyze__generic_class.snap b/nac3core/src/toplevel/snapshots/nac3core__toplevel__test__test_analyze__generic_class.snap index 42e38276..f8d39366 100644 --- a/nac3core/src/toplevel/snapshots/nac3core__toplevel__test__test_analyze__generic_class.snap +++ b/nac3core/src/toplevel/snapshots/nac3core__toplevel__test__test_analyze__generic_class.snap @@ -4,10 +4,10 @@ expression: res_vec --- [ - "Class {\nname: \"Generic_A\",\nancestors: [\"{class: Generic_A, params: [\\\"var4\\\"]}\", \"{class: B, params: []}\"],\nfields: [\"aa\", \"a\"],\nmethods: [(\"__init__\", \"fn[[], none]\"), (\"foo\", \"fn[[b=var3], none]\"), (\"fun\", \"fn[[a=int32], var4]\")],\ntype_vars: [\"var4\"]\n}\n", - "Function {\nname: \"Generic_A.__init__\",\nsig: \"fn[[], none]\",\nvar_id: [4]\n}\n", - "Function {\nname: \"Generic_A.fun\",\nsig: \"fn[[a=int32], var4]\",\nvar_id: [4]\n}\n", - "Class {\nname: \"B\",\nancestors: [\"{class: B, params: []}\"],\nfields: [\"aa\"],\nmethods: [(\"__init__\", \"fn[[], none]\"), (\"foo\", \"fn[[b=var3], none]\")],\ntype_vars: []\n}\n", + "Class {\nname: \"Generic_A\",\nancestors: [\"{class: Generic_A, params: [\\\"var6\\\"]}\", \"{class: B, params: []}\"],\nfields: [\"aa\", \"a\"],\nmethods: [(\"__init__\", \"fn[[], none]\"), (\"foo\", \"fn[[b=var5], none]\"), (\"fun\", \"fn[[a=int32], var6]\")],\ntype_vars: [\"var6\"]\n}\n", + "Function {\nname: \"Generic_A.__init__\",\nsig: \"fn[[], none]\",\nvar_id: [6]\n}\n", + "Function {\nname: \"Generic_A.fun\",\nsig: \"fn[[a=int32], var6]\",\nvar_id: [6]\n}\n", + "Class {\nname: \"B\",\nancestors: [\"{class: B, params: []}\"],\nfields: [\"aa\"],\nmethods: [(\"__init__\", \"fn[[], none]\"), (\"foo\", \"fn[[b=var5], none]\")],\ntype_vars: []\n}\n", "Function {\nname: \"B.__init__\",\nsig: \"fn[[], none]\",\nvar_id: []\n}\n", - "Function {\nname: \"B.foo\",\nsig: \"fn[[b=var3], none]\",\nvar_id: []\n}\n", + "Function {\nname: \"B.foo\",\nsig: \"fn[[b=var5], none]\",\nvar_id: []\n}\n", ] diff --git a/nac3core/src/toplevel/snapshots/nac3core__toplevel__test__test_analyze__inheritance_override.snap b/nac3core/src/toplevel/snapshots/nac3core__toplevel__test__test_analyze__inheritance_override.snap index b08e9352..2067a56e 100644 --- a/nac3core/src/toplevel/snapshots/nac3core__toplevel__test__test_analyze__inheritance_override.snap +++ b/nac3core/src/toplevel/snapshots/nac3core__toplevel__test__test_analyze__inheritance_override.snap @@ -4,13 +4,13 @@ expression: res_vec --- [ - "Class {\nname: \"A\",\nancestors: [\"{class: A, params: [\\\"var3\\\"]}\"],\nfields: [\"a\", \"b\", \"c\"],\nmethods: [(\"__init__\", \"fn[[t=var3], none]\"), (\"fun\", \"fn[[a=int32, b=var3], list[virtual[B[4->bool]]]]\"), (\"foo\", \"fn[[c=C], none]\")],\ntype_vars: [\"var3\"]\n}\n", - "Function {\nname: \"A.__init__\",\nsig: \"fn[[t=var3], none]\",\nvar_id: []\n}\n", - "Function {\nname: \"A.fun\",\nsig: \"fn[[a=int32, b=var3], list[virtual[B[4->bool]]]]\",\nvar_id: []\n}\n", + "Class {\nname: \"A\",\nancestors: [\"{class: A, params: [\\\"var5\\\"]}\"],\nfields: [\"a\", \"b\", \"c\"],\nmethods: [(\"__init__\", \"fn[[t=var5], none]\"), (\"fun\", \"fn[[a=int32, b=var5], list[virtual[B[6->bool]]]]\"), (\"foo\", \"fn[[c=C], none]\")],\ntype_vars: [\"var5\"]\n}\n", + "Function {\nname: \"A.__init__\",\nsig: \"fn[[t=var5], none]\",\nvar_id: []\n}\n", + "Function {\nname: \"A.fun\",\nsig: \"fn[[a=int32, b=var5], list[virtual[B[6->bool]]]]\",\nvar_id: []\n}\n", "Function {\nname: \"A.foo\",\nsig: \"fn[[c=C], none]\",\nvar_id: []\n}\n", - "Class {\nname: \"B\",\nancestors: [\"{class: B, params: [\\\"var4\\\"]}\", \"{class: A, params: [\\\"float\\\"]}\"],\nfields: [\"a\", \"b\", \"c\", \"d\"],\nmethods: [(\"__init__\", \"fn[[], none]\"), (\"fun\", \"fn[[a=int32, b=var3], list[virtual[B[4->bool]]]]\"), (\"foo\", \"fn[[c=C], none]\")],\ntype_vars: [\"var4\"]\n}\n", - "Function {\nname: \"B.__init__\",\nsig: \"fn[[], none]\",\nvar_id: [4]\n}\n", - "Function {\nname: \"B.fun\",\nsig: \"fn[[a=int32, b=var3], list[virtual[B[4->bool]]]]\",\nvar_id: [4]\n}\n", - "Class {\nname: \"C\",\nancestors: [\"{class: C, params: []}\", \"{class: B, params: [\\\"bool\\\"]}\", \"{class: A, params: [\\\"float\\\"]}\"],\nfields: [\"a\", \"b\", \"c\", \"d\", \"e\"],\nmethods: [(\"__init__\", \"fn[[], none]\"), (\"fun\", \"fn[[a=int32, b=var3], list[virtual[B[4->bool]]]]\"), (\"foo\", \"fn[[c=C], none]\")],\ntype_vars: []\n}\n", + "Class {\nname: \"B\",\nancestors: [\"{class: B, params: [\\\"var6\\\"]}\", \"{class: A, params: [\\\"float\\\"]}\"],\nfields: [\"a\", \"b\", \"c\", \"d\"],\nmethods: [(\"__init__\", \"fn[[], none]\"), (\"fun\", \"fn[[a=int32, b=var5], list[virtual[B[6->bool]]]]\"), (\"foo\", \"fn[[c=C], none]\")],\ntype_vars: [\"var6\"]\n}\n", + "Function {\nname: \"B.__init__\",\nsig: \"fn[[], none]\",\nvar_id: [6]\n}\n", + "Function {\nname: \"B.fun\",\nsig: \"fn[[a=int32, b=var5], list[virtual[B[6->bool]]]]\",\nvar_id: [6]\n}\n", + "Class {\nname: \"C\",\nancestors: [\"{class: C, params: []}\", \"{class: B, params: [\\\"bool\\\"]}\", \"{class: A, params: [\\\"float\\\"]}\"],\nfields: [\"a\", \"b\", \"c\", \"d\", \"e\"],\nmethods: [(\"__init__\", \"fn[[], none]\"), (\"fun\", \"fn[[a=int32, b=var5], list[virtual[B[6->bool]]]]\"), (\"foo\", \"fn[[c=C], none]\")],\ntype_vars: []\n}\n", "Function {\nname: \"C.__init__\",\nsig: \"fn[[], none]\",\nvar_id: []\n}\n", ] diff --git a/nac3core/src/toplevel/snapshots/nac3core__toplevel__test__test_analyze__list_tuple_generic.snap b/nac3core/src/toplevel/snapshots/nac3core__toplevel__test__test_analyze__list_tuple_generic.snap index 2083a398..7527143d 100644 --- a/nac3core/src/toplevel/snapshots/nac3core__toplevel__test__test_analyze__list_tuple_generic.snap +++ b/nac3core/src/toplevel/snapshots/nac3core__toplevel__test__test_analyze__list_tuple_generic.snap @@ -4,11 +4,11 @@ expression: res_vec --- [ - "Function {\nname: \"foo\",\nsig: \"fn[[a=list[int32], b=tuple[var3, float]], A[3->B, 4->bool]]\",\nvar_id: []\n}\n", - "Class {\nname: \"A\",\nancestors: [\"{class: A, params: [\\\"var3\\\", \\\"var4\\\"]}\"],\nfields: [\"a\", \"b\"],\nmethods: [(\"__init__\", \"fn[[v=var4], none]\"), (\"fun\", \"fn[[a=var3], var4]\")],\ntype_vars: [\"var3\", \"var4\"]\n}\n", - "Function {\nname: \"A.__init__\",\nsig: \"fn[[v=var4], none]\",\nvar_id: [4]\n}\n", - "Function {\nname: \"A.fun\",\nsig: \"fn[[a=var3], var4]\",\nvar_id: [4]\n}\n", - "Function {\nname: \"gfun\",\nsig: \"fn[[a=A[3->list[float], 4->int32]], none]\",\nvar_id: []\n}\n", + "Function {\nname: \"foo\",\nsig: \"fn[[a=list[int32], b=tuple[var5, float]], A[5->B, 6->bool]]\",\nvar_id: []\n}\n", + "Class {\nname: \"A\",\nancestors: [\"{class: A, params: [\\\"var5\\\", \\\"var6\\\"]}\"],\nfields: [\"a\", \"b\"],\nmethods: [(\"__init__\", \"fn[[v=var6], none]\"), (\"fun\", \"fn[[a=var5], var6]\")],\ntype_vars: [\"var5\", \"var6\"]\n}\n", + "Function {\nname: \"A.__init__\",\nsig: \"fn[[v=var6], none]\",\nvar_id: [6]\n}\n", + "Function {\nname: \"A.fun\",\nsig: \"fn[[a=var5], var6]\",\nvar_id: [6]\n}\n", + "Function {\nname: \"gfun\",\nsig: \"fn[[a=A[5->list[float], 6->int32]], none]\",\nvar_id: []\n}\n", "Class {\nname: \"B\",\nancestors: [\"{class: B, params: []}\"],\nfields: [],\nmethods: [(\"__init__\", \"fn[[], none]\")],\ntype_vars: []\n}\n", "Function {\nname: \"B.__init__\",\nsig: \"fn[[], none]\",\nvar_id: []\n}\n", ] diff --git a/nac3core/src/toplevel/snapshots/nac3core__toplevel__test__test_analyze__self1.snap b/nac3core/src/toplevel/snapshots/nac3core__toplevel__test__test_analyze__self1.snap index 2ded3960..d6886ab2 100644 --- a/nac3core/src/toplevel/snapshots/nac3core__toplevel__test__test_analyze__self1.snap +++ b/nac3core/src/toplevel/snapshots/nac3core__toplevel__test__test_analyze__self1.snap @@ -4,11 +4,11 @@ expression: res_vec --- [ - "Class {\nname: \"A\",\nancestors: [\"{class: A, params: [\\\"var3\\\", \\\"var4\\\"]}\"],\nfields: [\"a\", \"b\"],\nmethods: [(\"__init__\", \"fn[[a=A[3->float, 4->bool], b=B], none]\"), (\"fun\", \"fn[[a=A[3->float, 4->bool]], A[3->bool, 4->int32]]\")],\ntype_vars: [\"var3\", \"var4\"]\n}\n", - "Function {\nname: \"A.__init__\",\nsig: \"fn[[a=A[3->float, 4->bool], b=B], none]\",\nvar_id: [4]\n}\n", - "Function {\nname: \"A.fun\",\nsig: \"fn[[a=A[3->float, 4->bool]], A[3->bool, 4->int32]]\",\nvar_id: [4]\n}\n", - "Class {\nname: \"B\",\nancestors: [\"{class: B, params: []}\", \"{class: A, params: [\\\"int64\\\", \\\"bool\\\"]}\"],\nfields: [\"a\", \"b\"],\nmethods: [(\"__init__\", \"fn[[], none]\"), (\"fun\", \"fn[[a=A[3->float, 4->bool]], A[3->bool, 4->int32]]\"), (\"foo\", \"fn[[b=B], B]\"), (\"bar\", \"fn[[a=A[3->list[B], 4->int32]], tuple[A[3->virtual[A[3->B, 4->int32]], 4->bool], B]]\")],\ntype_vars: []\n}\n", + "Class {\nname: \"A\",\nancestors: [\"{class: A, params: [\\\"var5\\\", \\\"var6\\\"]}\"],\nfields: [\"a\", \"b\"],\nmethods: [(\"__init__\", \"fn[[a=A[5->float, 6->bool], b=B], none]\"), (\"fun\", \"fn[[a=A[5->float, 6->bool]], A[5->bool, 6->int32]]\")],\ntype_vars: [\"var5\", \"var6\"]\n}\n", + "Function {\nname: \"A.__init__\",\nsig: \"fn[[a=A[5->float, 6->bool], b=B], none]\",\nvar_id: [6]\n}\n", + "Function {\nname: \"A.fun\",\nsig: \"fn[[a=A[5->float, 6->bool]], A[5->bool, 6->int32]]\",\nvar_id: [6]\n}\n", + "Class {\nname: \"B\",\nancestors: [\"{class: B, params: []}\", \"{class: A, params: [\\\"int64\\\", \\\"bool\\\"]}\"],\nfields: [\"a\", \"b\"],\nmethods: [(\"__init__\", \"fn[[], none]\"), (\"fun\", \"fn[[a=A[5->float, 6->bool]], A[5->bool, 6->int32]]\"), (\"foo\", \"fn[[b=B], B]\"), (\"bar\", \"fn[[a=A[5->list[B], 6->int32]], tuple[A[5->virtual[A[5->B, 6->int32]], 6->bool], B]]\")],\ntype_vars: []\n}\n", "Function {\nname: \"B.__init__\",\nsig: \"fn[[], none]\",\nvar_id: []\n}\n", "Function {\nname: \"B.foo\",\nsig: \"fn[[b=B], B]\",\nvar_id: []\n}\n", - "Function {\nname: \"B.bar\",\nsig: \"fn[[a=A[3->list[B], 4->int32]], tuple[A[3->virtual[A[3->B, 4->int32]], 4->bool], B]]\",\nvar_id: []\n}\n", + "Function {\nname: \"B.bar\",\nsig: \"fn[[a=A[5->list[B], 6->int32]], tuple[A[5->virtual[A[5->B, 6->int32]], 6->bool], B]]\",\nvar_id: []\n}\n", ] diff --git a/nac3core/src/toplevel/snapshots/nac3core__toplevel__test__test_analyze__simple_class_compose.snap b/nac3core/src/toplevel/snapshots/nac3core__toplevel__test__test_analyze__simple_class_compose.snap index d6f269ac..ab0f0534 100644 --- a/nac3core/src/toplevel/snapshots/nac3core__toplevel__test__test_analyze__simple_class_compose.snap +++ b/nac3core/src/toplevel/snapshots/nac3core__toplevel__test__test_analyze__simple_class_compose.snap @@ -4,15 +4,15 @@ expression: res_vec --- [ - "Class {\nname: \"A\",\nancestors: [\"{class: A, params: []}\"],\nfields: [\"a\"],\nmethods: [(\"__init__\", \"fn[[], none]\"), (\"fun\", \"fn[[b=B], none]\"), (\"foo\", \"fn[[a=var3, b=var4], none]\")],\ntype_vars: []\n}\n", + "Class {\nname: \"A\",\nancestors: [\"{class: A, params: []}\"],\nfields: [\"a\"],\nmethods: [(\"__init__\", \"fn[[], none]\"), (\"fun\", \"fn[[b=B], none]\"), (\"foo\", \"fn[[a=var5, b=var6], none]\")],\ntype_vars: []\n}\n", "Function {\nname: \"A.__init__\",\nsig: \"fn[[], none]\",\nvar_id: []\n}\n", "Function {\nname: \"A.fun\",\nsig: \"fn[[b=B], none]\",\nvar_id: []\n}\n", - "Function {\nname: \"A.foo\",\nsig: \"fn[[a=var3, b=var4], none]\",\nvar_id: [4]\n}\n", - "Class {\nname: \"B\",\nancestors: [\"{class: B, params: []}\", \"{class: C, params: []}\", \"{class: A, params: []}\"],\nfields: [\"a\"],\nmethods: [(\"__init__\", \"fn[[], none]\"), (\"fun\", \"fn[[b=B], none]\"), (\"foo\", \"fn[[a=var3, b=var4], none]\")],\ntype_vars: []\n}\n", + "Function {\nname: \"A.foo\",\nsig: \"fn[[a=var5, b=var6], none]\",\nvar_id: [6]\n}\n", + "Class {\nname: \"B\",\nancestors: [\"{class: B, params: []}\", \"{class: C, params: []}\", \"{class: A, params: []}\"],\nfields: [\"a\"],\nmethods: [(\"__init__\", \"fn[[], none]\"), (\"fun\", \"fn[[b=B], none]\"), (\"foo\", \"fn[[a=var5, b=var6], none]\")],\ntype_vars: []\n}\n", "Function {\nname: \"B.__init__\",\nsig: \"fn[[], none]\",\nvar_id: []\n}\n", - "Class {\nname: \"C\",\nancestors: [\"{class: C, params: []}\", \"{class: A, params: []}\"],\nfields: [\"a\"],\nmethods: [(\"__init__\", \"fn[[], none]\"), (\"fun\", \"fn[[b=B], none]\"), (\"foo\", \"fn[[a=var3, b=var4], none]\")],\ntype_vars: []\n}\n", + "Class {\nname: \"C\",\nancestors: [\"{class: C, params: []}\", \"{class: A, params: []}\"],\nfields: [\"a\"],\nmethods: [(\"__init__\", \"fn[[], none]\"), (\"fun\", \"fn[[b=B], none]\"), (\"foo\", \"fn[[a=var5, b=var6], none]\")],\ntype_vars: []\n}\n", "Function {\nname: \"C.__init__\",\nsig: \"fn[[], none]\",\nvar_id: []\n}\n", "Function {\nname: \"C.fun\",\nsig: \"fn[[b=B], none]\",\nvar_id: []\n}\n", "Function {\nname: \"foo\",\nsig: \"fn[[a=A], none]\",\nvar_id: []\n}\n", - "Function {\nname: \"ff\",\nsig: \"fn[[a=var3], var4]\",\nvar_id: [4]\n}\n", + "Function {\nname: \"ff\",\nsig: \"fn[[a=var5], var6]\",\nvar_id: [6]\n}\n", ] From 82359b81a2c93c7633536486d3769e08788ba9c1 Mon Sep 17 00:00:00 2001 From: ychenfo Date: Mon, 13 Dec 2021 04:11:31 +0800 Subject: [PATCH 43/54] nac3core: fix bool to int conversion --- nac3core/src/toplevel/builtins.rs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/nac3core/src/toplevel/builtins.rs b/nac3core/src/toplevel/builtins.rs index d9d9045d..fc0c0006 100644 --- a/nac3core/src/toplevel/builtins.rs +++ b/nac3core/src/toplevel/builtins.rs @@ -69,10 +69,10 @@ pub fn get_builtins(primitives: &mut (PrimitiveStore, Unifier)) -> BuiltinInfo { if ctx.unifier.unioned(arg_ty, boolean) { Some( ctx.builder - .build_int_s_extend( + .build_int_z_extend( arg.into_int_value(), ctx.ctx.i32_type(), - "sext", + "zext", ) .into(), ) @@ -129,10 +129,10 @@ pub fn get_builtins(primitives: &mut (PrimitiveStore, Unifier)) -> BuiltinInfo { { Some( ctx.builder - .build_int_s_extend( + .build_int_z_extend( arg.into_int_value(), ctx.ctx.i64_type(), - "sext", + "zext", ) .into(), ) From 2c6601d97c111f07278b51272ffd1539847cad9f Mon Sep 17 00:00:00 2001 From: ychenfo Date: Sun, 12 Dec 2021 05:52:22 +0800 Subject: [PATCH 44/54] nac3core: fix len on range with step of different sign --- nac3core/src/toplevel/builtins.rs | 106 ++++++++++++++++++++---------- 1 file changed, 71 insertions(+), 35 deletions(-) diff --git a/nac3core/src/toplevel/builtins.rs b/nac3core/src/toplevel/builtins.rs index fc0c0006..c2e5be4a 100644 --- a/nac3core/src/toplevel/builtins.rs +++ b/nac3core/src/toplevel/builtins.rs @@ -1,5 +1,5 @@ use std::cell::RefCell; -use inkwell::{IntPredicate, FloatPredicate, values::BasicValueEnum}; +use inkwell::{IntPredicate, FloatPredicate, values::{BasicValueEnum, IntValue}}; use crate::{symbol_resolver::SymbolValue, codegen::expr::destructure_range}; use super::*; @@ -573,7 +573,7 @@ pub fn get_builtins(primitives: &mut (PrimitiveStore, Unifier)) -> BuiltinInfo { ret: int32, vars: vec![(list_var.1, list_var.0), (arg_ty.1, arg_ty.0)].into_iter().collect(), }))), - var_id: Default::default(), + var_id: vec![arg_ty.1], instance_to_symbol: Default::default(), instance_to_stmt: Default::default(), resolver: None, @@ -585,41 +585,9 @@ pub fn get_builtins(primitives: &mut (PrimitiveStore, Unifier)) -> BuiltinInfo { let int32 = ctx.ctx.i32_type(); let zero = int32.const_zero(); if ctx.unifier.unioned(arg_ty, range_ty) { - let int1 = ctx.ctx.bool_type(); - let one = int32.const_int(1, false); - let falze = int1.const_int(0, false); - let abs_intrinsic = - ctx.module.get_function("llvm.abs.i32").unwrap_or_else(|| { - let fn_type = int32.fn_type(&[int32.into(), int1.into()], false); - ctx.module.add_function("llvm.abs.i32", fn_type, None) - }); let arg = arg.into_pointer_value(); let (start, end, step) = destructure_range(ctx, arg); - let diff = ctx.builder.build_int_sub(end, start, "diff"); - let diff = if let BasicValueEnum::IntValue(val) = ctx - .builder - .build_call(abs_intrinsic, &[diff.into(), falze.into()], "absdiff") - .try_as_basic_value() - .left() - .unwrap() { - val - } else { - unreachable!(); - }; - let diff = ctx.builder.build_int_sub(diff, one, "diff"); - let step = if let BasicValueEnum::IntValue(val) = ctx - .builder - .build_call(abs_intrinsic, &[step.into(), falze.into()], "absstep") - .try_as_basic_value() - .left() - .unwrap() { - val - } else { - unreachable!(); - }; - let length = ctx.builder.build_int_signed_div(diff, step, "div"); - let length = ctx.builder.build_int_add(length, int32.const_int(1, false), "add1"); - Some(length.into()) + Some(calculate_len_for_slice_range(ctx, start, end, step).into()) } else { Some(ctx.build_gep_and_load(arg.into_pointer_value(), &[zero, zero])) } @@ -648,4 +616,72 @@ pub fn get_builtins(primitives: &mut (PrimitiveStore, Unifier)) -> BuiltinInfo { "len", ] ) +} + +// equivalent code: +// def length(start, end, step != 0): +// diff = end - start +// # if diff == 0 OR `diff` and `step` are of different signs, always zero +// if diff * step <= 0: +// return 0 +// else: +// return ((abs(diff) - 1) // abs(step)) + 1 +pub fn calculate_len_for_slice_range<'ctx, 'a>( + ctx: &mut CodeGenContext<'ctx, 'a>, + start: IntValue<'ctx>, + end: IntValue<'ctx>, + step: IntValue<'ctx>, +) -> IntValue<'ctx> { + let int32 = ctx.ctx.i32_type(); + let int1 = ctx.ctx.bool_type(); + let falze = int1.const_int(0, false); + let abs_intrinsic = + ctx.module.get_function("llvm.abs.i32").unwrap_or_else(|| { + let fn_type = int32.fn_type(&[int32.into(), int1.into()], false); + ctx.module.add_function("llvm.abs.i32", fn_type, None) + }); + let diff = ctx.builder.build_int_sub(end, start, "diff"); + let test_mult = ctx.builder.build_int_mul(diff, step, "test_mult"); + let test = + ctx.builder.build_int_compare(inkwell::IntPredicate::SLE, test_mult, int32.const_zero(), "cmp"); + let current = ctx.builder.get_insert_block().unwrap().get_parent().unwrap(); + let then_bb = ctx.ctx.append_basic_block(current, "then"); + let else_bb = ctx.ctx.append_basic_block(current, "else"); + let cont_bb = ctx.ctx.append_basic_block(current, "cont"); + ctx.builder.build_conditional_branch(test, then_bb, else_bb); + + ctx.builder.position_at_end(then_bb); + let length_zero = int32.const_zero(); + ctx.builder.build_unconditional_branch(cont_bb); + + ctx.builder.position_at_end(else_bb); + let diff = if let BasicValueEnum::IntValue(val) = ctx + .builder + .build_call(abs_intrinsic, &[diff.into(), falze.into()], "absdiff") + .try_as_basic_value() + .left() + .unwrap() { + val + } else { + unreachable!(); + }; + let diff = ctx.builder.build_int_sub(diff, int32.const_int(1, false), "diff"); + let step = if let BasicValueEnum::IntValue(val) = ctx + .builder + .build_call(abs_intrinsic, &[step.into(), falze.into()], "absstep") + .try_as_basic_value() + .left() + .unwrap() { + val + } else { + unreachable!(); + }; + let length = ctx.builder.build_int_signed_div(diff, step, "div"); + let length = ctx.builder.build_int_add(length, int32.const_int(1, false), "add1"); + ctx.builder.build_unconditional_branch(cont_bb); + + ctx.builder.position_at_end(cont_bb); + let phi = ctx.builder.build_phi(length_zero.get_type(), "lenphi"); + phi.add_incoming(&[(&length_zero, then_bb), (&length, else_bb)]); + phi.as_basic_value().into_int_value() } \ No newline at end of file From b5637a04e95dbb7ef2b37a0c1f1d37bfd44d1e1d Mon Sep 17 00:00:00 2001 From: ychenfo Date: Mon, 13 Dec 2021 04:02:30 +0800 Subject: [PATCH 45/54] nac3core: use official implementation for len --- nac3core/src/toplevel/builtins.rs | 97 +++++++++++++++++-------------- 1 file changed, 52 insertions(+), 45 deletions(-) diff --git a/nac3core/src/toplevel/builtins.rs b/nac3core/src/toplevel/builtins.rs index c2e5be4a..1bf24619 100644 --- a/nac3core/src/toplevel/builtins.rs +++ b/nac3core/src/toplevel/builtins.rs @@ -1,5 +1,5 @@ use std::cell::RefCell; -use inkwell::{IntPredicate, FloatPredicate, values::{BasicValueEnum, IntValue}}; +use inkwell::{IntPredicate::{self, *}, FloatPredicate, values::IntValue}; use crate::{symbol_resolver::SymbolValue, codegen::expr::destructure_range}; use super::*; @@ -570,7 +570,7 @@ pub fn get_builtins(primitives: &mut (PrimitiveStore, Unifier)) -> BuiltinInfo { ty: arg_ty.0, default_value: None }], - ret: int32, + ret: int64, vars: vec![(list_var.1, list_var.0), (arg_ty.1, arg_ty.0)].into_iter().collect(), }))), var_id: vec![arg_ty.1], @@ -582,13 +582,13 @@ pub fn get_builtins(primitives: &mut (PrimitiveStore, Unifier)) -> BuiltinInfo { let range_ty = ctx.primitives.range; let arg_ty = fun.0.args[0].ty; let arg = args[0].1; - let int32 = ctx.ctx.i32_type(); - let zero = int32.const_zero(); if ctx.unifier.unioned(arg_ty, range_ty) { let arg = arg.into_pointer_value(); let (start, end, step) = destructure_range(ctx, arg); Some(calculate_len_for_slice_range(ctx, start, end, step).into()) } else { + let int32 = ctx.ctx.i32_type(); + let zero = int32.const_zero(); Some(ctx.build_gep_and_load(arg.into_pointer_value(), &[zero, zero])) } }, @@ -621,67 +621,74 @@ pub fn get_builtins(primitives: &mut (PrimitiveStore, Unifier)) -> BuiltinInfo { // equivalent code: // def length(start, end, step != 0): // diff = end - start -// # if diff == 0 OR `diff` and `step` are of different signs, always zero -// if diff * step <= 0: -// return 0 +// if diff > 0 and step > 0: +// return ((diff - 1) // step) + 1 +// elif diff < 0 and step < 0: +// return ((diff + 1) // step) + 1 // else: -// return ((abs(diff) - 1) // abs(step)) + 1 +// return 0 pub fn calculate_len_for_slice_range<'ctx, 'a>( ctx: &mut CodeGenContext<'ctx, 'a>, start: IntValue<'ctx>, end: IntValue<'ctx>, step: IntValue<'ctx>, ) -> IntValue<'ctx> { - let int32 = ctx.ctx.i32_type(); - let int1 = ctx.ctx.bool_type(); - let falze = int1.const_int(0, false); - let abs_intrinsic = - ctx.module.get_function("llvm.abs.i32").unwrap_or_else(|| { - let fn_type = int32.fn_type(&[int32.into(), int1.into()], false); - ctx.module.add_function("llvm.abs.i32", fn_type, None) - }); + let int64 = ctx.ctx.i64_type(); + let start = ctx.builder.build_int_s_extend(start, int64, "start"); + let end = ctx.builder.build_int_s_extend(end, int64, "end"); + let step = ctx.builder.build_int_s_extend(step, int64, "step"); let diff = ctx.builder.build_int_sub(end, start, "diff"); - let test_mult = ctx.builder.build_int_mul(diff, step, "test_mult"); - let test = - ctx.builder.build_int_compare(inkwell::IntPredicate::SLE, test_mult, int32.const_zero(), "cmp"); + + let diff_pos = ctx.builder.build_int_compare(SGT, diff, int64.const_zero(), "diffpos"); + let step_pos = ctx.builder.build_int_compare(SGT, step, int64.const_zero(), "steppos"); + let test_1 = ctx.builder.build_and(diff_pos, step_pos, "bothpos"); + let current = ctx.builder.get_insert_block().unwrap().get_parent().unwrap(); let then_bb = ctx.ctx.append_basic_block(current, "then"); let else_bb = ctx.ctx.append_basic_block(current, "else"); + let then_bb_2 = ctx.ctx.append_basic_block(current, "then_2"); + let else_bb_2 = ctx.ctx.append_basic_block(current, "else_2"); + let cont_bb_2 = ctx.ctx.append_basic_block(current, "cont_2"); let cont_bb = ctx.ctx.append_basic_block(current, "cont"); - ctx.builder.build_conditional_branch(test, then_bb, else_bb); + ctx.builder.build_conditional_branch(test_1, then_bb, else_bb); ctx.builder.position_at_end(then_bb); - let length_zero = int32.const_zero(); + let length_pos = { + let diff_pos_min_1 = ctx.builder.build_int_sub(diff, int64.const_int(1, false), "diffminone"); + let length_pos = ctx.builder.build_int_signed_div(diff_pos_min_1, step, "div"); + ctx.builder.build_int_add(length_pos, int64.const_int(1, false), "add1") + }; ctx.builder.build_unconditional_branch(cont_bb); ctx.builder.position_at_end(else_bb); - let diff = if let BasicValueEnum::IntValue(val) = ctx - .builder - .build_call(abs_intrinsic, &[diff.into(), falze.into()], "absdiff") - .try_as_basic_value() - .left() - .unwrap() { - val - } else { - unreachable!(); + let phi_1 = { + let diff_neg = ctx.builder.build_int_compare(SLT, diff, int64.const_zero(), "diffneg"); + let step_neg = ctx.builder.build_int_compare(SLT, step, int64.const_zero(), "stepneg"); + let test_2 = ctx.builder.build_and(diff_neg, step_neg, "bothneg"); + + ctx.builder.build_conditional_branch(test_2, then_bb_2, else_bb_2); + + ctx.builder.position_at_end(then_bb_2); + let length_neg = { + let diff_neg_add_1 = ctx.builder.build_int_add(diff, int64.const_int(1, false), "diffminone"); + let length_neg = ctx.builder.build_int_signed_div(diff_neg_add_1, step, "div"); + ctx.builder.build_int_add(length_neg, int64.const_int(1, false), "add1") }; - let diff = ctx.builder.build_int_sub(diff, int32.const_int(1, false), "diff"); - let step = if let BasicValueEnum::IntValue(val) = ctx - .builder - .build_call(abs_intrinsic, &[step.into(), falze.into()], "absstep") - .try_as_basic_value() - .left() - .unwrap() { - val - } else { - unreachable!(); - }; - let length = ctx.builder.build_int_signed_div(diff, step, "div"); - let length = ctx.builder.build_int_add(length, int32.const_int(1, false), "add1"); + ctx.builder.build_unconditional_branch(cont_bb_2); + + ctx.builder.position_at_end(else_bb_2); + let length_zero = int64.const_zero(); + ctx.builder.build_unconditional_branch(cont_bb_2); + + ctx.builder.position_at_end(cont_bb_2); + let phi_1 = ctx.builder.build_phi(int64, "lenphi1"); + phi_1.add_incoming(&[(&length_neg, then_bb_2), (&length_zero, else_bb_2)]); + phi_1.as_basic_value().into_int_value() + }; ctx.builder.build_unconditional_branch(cont_bb); ctx.builder.position_at_end(cont_bb); - let phi = ctx.builder.build_phi(length_zero.get_type(), "lenphi"); - phi.add_incoming(&[(&length_zero, then_bb), (&length, else_bb)]); + let phi = ctx.builder.build_phi(int64, "lenphi"); + phi.add_incoming(&[(&length_pos, then_bb), (&phi_1, cont_bb_2)]); phi.as_basic_value().into_int_value() } \ No newline at end of file From ccfcba4066be83d37aa1bb68cd83e2fe875f8051 Mon Sep 17 00:00:00 2001 From: ychenfo Date: Mon, 13 Dec 2021 04:06:38 +0800 Subject: [PATCH 46/54] nac3standalone: add output_long --- nac3standalone/demo/classes.py | 4 ++++ nac3standalone/demo/demo.c | 4 ++++ 2 files changed, 8 insertions(+) diff --git a/nac3standalone/demo/classes.py b/nac3standalone/demo/classes.py index ba489b48..14df8f2f 100644 --- a/nac3standalone/demo/classes.py +++ b/nac3standalone/demo/classes.py @@ -2,6 +2,10 @@ def output_int(x: int32): ... +@extern +def output_long(x: int64): + ... + class A: a: int32 diff --git a/nac3standalone/demo/demo.c b/nac3standalone/demo/demo.c index fb9dbd1f..90e4c05b 100644 --- a/nac3standalone/demo/demo.c +++ b/nac3standalone/demo/demo.c @@ -5,6 +5,10 @@ void output_int(int x) { printf("%d\n", x); } +void output_long(long x) { + printf("%ld\n", x); +} + void output_asciiart(int x) { static char chars[] = " .,-:;i+hHM$*#@ "; if(x < 0) { From 69b9ac5152279a544e6cd3db2555db5acb669c3b Mon Sep 17 00:00:00 2001 From: Sebastien Bourdeauducq Date: Mon, 13 Dec 2021 11:18:44 +0800 Subject: [PATCH 47/54] nac3standalone: consistent naming --- nac3standalone/demo/classes.py | 12 ++++++------ nac3standalone/demo/demo.c | 4 ++-- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/nac3standalone/demo/classes.py b/nac3standalone/demo/classes.py index 14df8f2f..f3cf9517 100644 --- a/nac3standalone/demo/classes.py +++ b/nac3standalone/demo/classes.py @@ -1,9 +1,9 @@ @extern -def output_int(x: int32): +def output_int32(x: int32): ... @extern -def output_long(x: int64): +def output_int64(x: int64): ... @@ -31,10 +31,10 @@ class B: def run() -> int32: a = A(10) - output_int(a.a) + output_int32(a.a) a = A(20) - output_int(a.a) - output_int(a.get_a()) - output_int(a.get_b().b) + output_int32(a.a) + output_int32(a.get_a()) + output_int32(a.get_b().b) return 0 diff --git a/nac3standalone/demo/demo.c b/nac3standalone/demo/demo.c index 90e4c05b..ad19ee1e 100644 --- a/nac3standalone/demo/demo.c +++ b/nac3standalone/demo/demo.c @@ -1,11 +1,11 @@ #include #include -void output_int(int x) { +void output_int32(int x) { printf("%d\n", x); } -void output_long(long x) { +void output_int64(long x) { printf("%ld\n", x); } From 7420ce185b83fa8c390ad0309947b324c6f0f284 Mon Sep 17 00:00:00 2001 From: Sebastien Bourdeauducq Date: Mon, 13 Dec 2021 19:02:13 +0800 Subject: [PATCH 48/54] README: update --- README.md | 47 ++++++++++++++++++++++++++--------------------- 1 file changed, 26 insertions(+), 21 deletions(-) diff --git a/README.md b/README.md index 6480476c..412bedd2 100644 --- a/README.md +++ b/README.md @@ -1,25 +1,30 @@ -# NAC3 compiler +# NAC3 + +NAC3 is a major, backward-incompatible rewrite of the compiler for the [ARTIQ](https://m-labs.hk/artiq) physics experiment control and data acquisition system. It features greatly improved compilation speeds, a much better type system, and more predictable and transparent operation. + +NAC3 has a modular design and its applicability reaches beyond ARTIQ. The ``nac3core`` module does not contain anything specific to ARTIQ, and can be used in any project that requires compiling Python to machine code. + +**WARNING: NAC3 is currently experimental software and several important features are not implemented yet.** + +## Packaging + +NAC3 is packaged using the [Nix](https://nixos.org) Flakes system. Install Nix 2.4+ and enable flakes by adding ``experimental-features = nix-command flakes`` to ``nix.conf`` (e.g. ``~/.config/nix/nix.conf``). + +## Try NAC3 + +After setting up Nix as above, use ``nix shell github:m-labs/artiq/nac3`` to get a shell with the NAC3 version of ARTIQ. See the ``examples`` directory in ARTIQ (``nac3`` Git branch) for some samples of NAC3 kernel code. + +## For developers This repository contains: -- nac3ast: Python abstract syntax tree definition (based on RustPython). -- nac3parser: Python parser (based on RustPython). -- nac3core: Core compiler library, containing type-checking and code - generation. -- nac3standalone: Standalone compiler tool (core language only). -- nac3artiq: Integration with ARTIQ and implementation of ARTIQ-specific - extensions to the core language. -- runkernel: Simple program that runs compiled ARTIQ kernels on the host - and displays RTIO operations. Useful for testing without hardware. +- ``nac3ast``: Python abstract syntax tree definition (based on RustPython). +- ``nac3parser``: Python parser (based on RustPython). +- ``nac3core``: Core compiler library, containing type-checking and code generation. +- ``nac3standalone``: Standalone compiler tool (core language only). +- ``nac3artiq``: Integration with ARTIQ and implementation of ARTIQ-specific extensions to the core language. +- ``runkernel``: Simple program that runs compiled ARTIQ kernels on the host and displays RTIO operations. Useful for testing without hardware. +Use ``nix develop`` in this repository to enter a development shell. +If you are using a different shell than bash you can use e.g. ``nix develop --command fish``. -The core compiler knows nothing about symbol resolution, host variables -etc. nac3artiq and nac3standalone provide (implement) the -symbol resolver to the core compiler for resolving the type and value for -unknown symbols. The core compiler only type checks classes and functions -requested by nac3artiq/nac3standalone (the API should allow the -caller to specify which methods should be compiled). After type checking, the -compiler analyses the set of functions/classes that are used and performs -code generation. - -value could be integer values, boolean values, bytes (for memcpy), function ID -(full name + concrete type) +Build NAC3 with ``cargo build --release``. See the demonstrations in ``nac3artiq`` and ``nac3standalone``. From 91625dd327d9feb7a6744451a039811bff3f74cb Mon Sep 17 00:00:00 2001 From: ychenfo Date: Sun, 19 Dec 2021 11:04:53 +0800 Subject: [PATCH 49/54] update kernel-only attribute annotation Reviewed-on: https://git.m-labs.hk/M-Labs/nac3/pulls/127 Co-authored-by: ychenfo Co-committed-by: ychenfo --- nac3artiq/demo/min_artiq.py | 5 +++- nac3artiq/src/lib.rs | 12 ++++++-- nac3core/src/toplevel/composer.rs | 49 ++++++++++++++++++++----------- nac3standalone/src/main.rs | 16 +++++----- 4 files changed, 53 insertions(+), 29 deletions(-) diff --git a/nac3artiq/demo/min_artiq.py b/nac3artiq/demo/min_artiq.py index 9adf1667..64f41c73 100644 --- a/nac3artiq/demo/min_artiq.py +++ b/nac3artiq/demo/min_artiq.py @@ -9,7 +9,7 @@ import nac3artiq __all__ = [ - "KernelInvariant", "virtual", + "Kernel", "KernelInvariant", "virtual", "round64", "floor64", "ceil64", "extern", "kernel", "portable", "nac3", "ms", "us", "ns", @@ -21,6 +21,9 @@ __all__ = [ T = TypeVar('T') +class Kernel(Generic[T]): + pass + class KernelInvariant(Generic[T]): pass diff --git a/nac3artiq/src/lib.rs b/nac3artiq/src/lib.rs index 405645d8..de34180e 100644 --- a/nac3artiq/src/lib.rs +++ b/nac3artiq/src/lib.rs @@ -22,7 +22,7 @@ use parking_lot::{Mutex, RwLock}; use nac3core::{ codegen::{concrete_type::ConcreteTypeStore, CodeGenTask, WithCall, WorkerRegistry}, symbol_resolver::SymbolResolver, - toplevel::{composer::TopLevelComposer, DefinitionId, GenCall, TopLevelDef}, + toplevel::{composer::{TopLevelComposer, ComposerConfig}, DefinitionId, GenCall, TopLevelDef}, typecheck::typedef::{FunSignature, FuncArg}, typecheck::{type_inferencer::PrimitiveStore, typedef::Type}, }; @@ -239,7 +239,10 @@ impl Nac3 { }))), ), ]; - let (_, builtins_def, builtins_ty) = TopLevelComposer::new(builtins.clone()); + let (_, builtins_def, builtins_ty) = TopLevelComposer::new(builtins.clone(), ComposerConfig { + kernel_ann: Some("Kernel"), + kernel_invariant_ann: "KernelInvariant" + }); let builtins_mod = PyModule::import(py, "builtins").unwrap(); let id_fn = builtins_mod.getattr("id").unwrap(); @@ -375,7 +378,10 @@ impl Nac3 { filename: &str, py: Python, ) -> PyResult<()> { - let (mut composer, _, _) = TopLevelComposer::new(self.builtins.clone()); + let (mut composer, _, _) = TopLevelComposer::new(self.builtins.clone(), ComposerConfig { + kernel_ann: Some("Kernel"), + kernel_invariant_ann: "KernelInvariant" + }); let mut id_to_def = HashMap::new(); let mut id_to_type = HashMap::new(); diff --git a/nac3core/src/toplevel/composer.rs b/nac3core/src/toplevel/composer.rs index 02b800da..4c633691 100644 --- a/nac3core/src/toplevel/composer.rs +++ b/nac3core/src/toplevel/composer.rs @@ -9,6 +9,20 @@ use crate::{ use super::*; +pub struct ComposerConfig { + pub kernel_ann: Option<&'static str>, + pub kernel_invariant_ann: &'static str, +} + +impl Default for ComposerConfig { + fn default() -> Self { + ComposerConfig { + kernel_ann: None, + kernel_invariant_ann: "Invariant" + } + } +} + type DefAst = (Arc>, Option>); pub struct TopLevelComposer { // list of top level definitions, same as top level context @@ -25,11 +39,12 @@ pub struct TopLevelComposer { pub method_class: HashMap, // number of built-in function and classes in the definition list, later skip pub builtin_num: usize, + pub core_config: ComposerConfig, } impl Default for TopLevelComposer { fn default() -> Self { - Self::new(vec![]).0 + Self::new(vec![], Default::default()).0 } } @@ -38,6 +53,7 @@ impl TopLevelComposer { /// resolver can later figure out primitive type definitions when passed a primitive type name pub fn new( builtins: Vec<(StrRef, FunSignature, Arc)>, + core_config: ComposerConfig ) -> (Self, HashMap, HashMap) { let mut primitives = Self::make_primitives(); let (mut definition_ast_list, builtin_name_list) = builtins::get_builtins(&mut primitives); @@ -108,6 +124,7 @@ impl TopLevelComposer { keyword_list, defined_names, method_class, + core_config, }, builtin_id, builtin_ty, @@ -554,7 +571,7 @@ impl TopLevelComposer { unifier, primitives, &mut type_var_to_concrete_def, - &self.keyword_list, + (&self.keyword_list, &self.core_config) )? } } @@ -827,8 +844,9 @@ impl TopLevelComposer { unifier: &mut Unifier, primitives: &PrimitiveStore, type_var_to_concrete_def: &mut HashMap, - keyword_list: &HashSet, + core_info: (&HashSet, &ComposerConfig), ) -> Result<(), String> { + let (keyword_list, core_config) = core_info; let mut class_def = class_def.write(); let ( class_id, @@ -1058,20 +1076,17 @@ impl TopLevelComposer { let dummy_field_type = unifier.get_fresh_var().0; // handle Kernel[T], KernelInvariant[T] - let (annotation, mutable) = { - let mut result = None; - if let ast::ExprKind::Subscript { value, slice, .. } = &annotation.as_ref().node { - if let ast::ExprKind::Name { id, .. } = &value.node { - result = if id == &"Kernel".into() { - Some((slice, true)) - } else if id == &"KernelInvariant".into() { - Some((slice, false)) - } else { - None - } - } - } - result.unwrap_or((annotation, true)) + let (annotation, mutable) = match &annotation.node { + ast::ExprKind::Subscript { value, slice, .. } if matches!( + &value.node, + ast::ExprKind::Name { id, .. } if id == &core_config.kernel_invariant_ann.into() + ) => (slice, false), + ast::ExprKind::Subscript { value, slice, .. } if matches!( + &value.node, + ast::ExprKind::Name { id, .. } if core_config.kernel_ann.map_or(false, |c| id == &c.into()) + ) => (slice, true), + _ if core_config.kernel_ann.is_none() => (annotation, true), + _ => continue // ignore fields annotated otherwise }; class_fields_def.push((*attr, dummy_field_type, mutable)); diff --git a/nac3standalone/src/main.rs b/nac3standalone/src/main.rs index 238511c2..8212b923 100644 --- a/nac3standalone/src/main.rs +++ b/nac3standalone/src/main.rs @@ -3,13 +3,10 @@ use inkwell::{ targets::*, OptimizationLevel, }; -use nac3core::typecheck::{type_inferencer::PrimitiveStore, typedef::{Type, Unifier}}; -use nac3parser::{ast::{Expr, ExprKind, StmtKind}, parser}; +use std::{borrow::Borrow, collections::HashMap, env, fs, path::Path, sync::Arc, time::SystemTime}; use parking_lot::RwLock; -use std::{borrow::Borrow, env}; -use std::fs; -use std::{collections::HashMap, path::Path, sync::Arc, time::SystemTime}; +use nac3parser::{ast::{Expr, ExprKind, StmtKind}, parser}; use nac3core::{ codegen::{ concrete_type::ConcreteTypeStore, CodeGenTask, DefaultCodeGenerator, WithCall, @@ -17,11 +14,11 @@ use nac3core::{ }, symbol_resolver::SymbolResolver, toplevel::{ - composer::TopLevelComposer, + composer::{TopLevelComposer, ComposerConfig}, TopLevelDef, helper::parse_parameter_default_value, type_annotation::*, }, - typecheck::typedef::FunSignature, + typecheck::{type_inferencer::PrimitiveStore, typedef::{Type, Unifier, FunSignature}} }; mod basic_symbol_resolver; @@ -47,7 +44,10 @@ fn main() { }; let primitive: PrimitiveStore = TopLevelComposer::make_primitives().0; - let (mut composer, builtins_def, builtins_ty) = TopLevelComposer::new(vec![]); + let (mut composer, builtins_def, builtins_ty) = TopLevelComposer::new( + vec![], + Default::default() + ); let internal_resolver: Arc = ResolverInternal { id_to_type: builtins_ty.into(), From ff27a1697e26ec84f0d657b81ca09c3e081a8090 Mon Sep 17 00:00:00 2001 From: ychenfo Date: Sun, 12 Dec 2021 05:39:48 +0800 Subject: [PATCH 50/54] nac3core: fix for loop type inference --- nac3core/src/typecheck/type_inferencer/mod.rs | 42 ++++++++++++++----- 1 file changed, 31 insertions(+), 11 deletions(-) diff --git a/nac3core/src/typecheck/type_inferencer/mod.rs b/nac3core/src/typecheck/type_inferencer/mod.rs index 4ab40e61..c18bed89 100644 --- a/nac3core/src/typecheck/type_inferencer/mod.rs +++ b/nac3core/src/typecheck/type_inferencer/mod.rs @@ -122,9 +122,36 @@ impl<'a> fold::Fold<()> for Inferencer<'a> { }, } } - ast::StmtKind::For { ref target, .. } => { - self.infer_pattern(target)?; - fold::fold_stmt(self, node)? + ast::StmtKind::For { target, iter, body, orelse, config_comment, type_comment } => { + self.infer_pattern(&target)?; + let target = self.fold_expr(*target)?; + let iter = self.fold_expr(*iter)?; + if self.unifier.unioned(iter.custom.unwrap(), self.primitives.range) { + self.unify(self.primitives.int32, target.custom.unwrap(), &target.location)?; + } else { + let list = self.unifier.add_ty(TypeEnum::TList { ty: target.custom.unwrap() }); + self.unify(list, iter.custom.unwrap(), &iter.location)?; + } + let body = body + .into_iter() + .map(|b| self.fold_stmt(b)) + .collect::, _>>()?; + let orelse = orelse + .into_iter() + .map(|o| self.fold_stmt(o)) + .collect::, _>>()?; + Located { + location: node.location, + node: ast::StmtKind::For { + target: Box::new(target), + iter: Box::new(iter), + body, + orelse, + config_comment, + type_comment, + }, + custom: None + } } ast::StmtKind::Assign { ref mut targets, ref config_comment, .. } => { for target in targets.iter_mut() { @@ -201,14 +228,7 @@ impl<'a> fold::Fold<()> for Inferencer<'a> { _ => fold::fold_stmt(self, node)?, }; match &stmt.node { - ast::StmtKind::For { target, iter, .. } => { - if self.unifier.unioned(iter.custom.unwrap(), self.primitives.range) { - self.unify(self.primitives.int32, target.custom.unwrap(), &target.location)?; - } else { - let list = self.unifier.add_ty(TypeEnum::TList { ty: target.custom.unwrap() }); - self.unify(list, iter.custom.unwrap(), &iter.location)?; - } - } + ast::StmtKind::For { .. } => {} ast::StmtKind::If { test, .. } | ast::StmtKind::While { test, .. } => { self.unify(test.custom.unwrap(), self.primitives.bool, &test.location)?; } From cb450372d657d5d859f633fbeaf88fa80404bbb3 Mon Sep 17 00:00:00 2001 From: ychenfo Date: Sun, 12 Dec 2021 05:40:22 +0800 Subject: [PATCH 51/54] nac3core: add missing expr concrete type check --- nac3core/src/typecheck/function_check.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/nac3core/src/typecheck/function_check.rs b/nac3core/src/typecheck/function_check.rs index 65c810e0..b65f4995 100644 --- a/nac3core/src/typecheck/function_check.rs +++ b/nac3core/src/typecheck/function_check.rs @@ -24,11 +24,13 @@ impl<'a> Inferencer<'a> { if !defined_identifiers.contains(id) { defined_identifiers.insert(*id); } + self.check_expr(pattern, defined_identifiers)?; self.should_have_value(pattern)?; Ok(()) } ExprKind::Tuple { elts, .. } => { for elt in elts.iter() { + self.check_expr(pattern, defined_identifiers)?; self.check_pattern(elt, defined_identifiers)?; self.should_have_value(elt)?; } From 2008db8097ff20bf0bb772aab46aea4827e640be Mon Sep 17 00:00:00 2001 From: Sebastien Bourdeauducq Date: Mon, 20 Dec 2021 17:39:16 +0800 Subject: [PATCH 52/54] nac3standalone: remove unused import --- nac3standalone/src/main.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nac3standalone/src/main.rs b/nac3standalone/src/main.rs index 8212b923..d5007129 100644 --- a/nac3standalone/src/main.rs +++ b/nac3standalone/src/main.rs @@ -14,7 +14,7 @@ use nac3core::{ }, symbol_resolver::SymbolResolver, toplevel::{ - composer::{TopLevelComposer, ComposerConfig}, + composer::TopLevelComposer, TopLevelDef, helper::parse_parameter_default_value, type_annotation::*, }, From e2b44a066be5244ca044f643b57697aa962710dd Mon Sep 17 00:00:00 2001 From: Sebastien Bourdeauducq Date: Mon, 20 Dec 2021 17:44:42 +0800 Subject: [PATCH 53/54] return int32 in len(). Closes #141 --- nac3core/src/toplevel/builtins.rs | 34 +++++++++++++++---------------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/nac3core/src/toplevel/builtins.rs b/nac3core/src/toplevel/builtins.rs index 1bf24619..e8a523a7 100644 --- a/nac3core/src/toplevel/builtins.rs +++ b/nac3core/src/toplevel/builtins.rs @@ -570,7 +570,7 @@ pub fn get_builtins(primitives: &mut (PrimitiveStore, Unifier)) -> BuiltinInfo { ty: arg_ty.0, default_value: None }], - ret: int64, + ret: int32, vars: vec![(list_var.1, list_var.0), (arg_ty.1, arg_ty.0)].into_iter().collect(), }))), var_id: vec![arg_ty.1], @@ -633,14 +633,14 @@ pub fn calculate_len_for_slice_range<'ctx, 'a>( end: IntValue<'ctx>, step: IntValue<'ctx>, ) -> IntValue<'ctx> { - let int64 = ctx.ctx.i64_type(); - let start = ctx.builder.build_int_s_extend(start, int64, "start"); - let end = ctx.builder.build_int_s_extend(end, int64, "end"); - let step = ctx.builder.build_int_s_extend(step, int64, "step"); + let int32 = ctx.ctx.i32_type(); + let start = ctx.builder.build_int_s_extend(start, int32, "start"); + let end = ctx.builder.build_int_s_extend(end, int32, "end"); + let step = ctx.builder.build_int_s_extend(step, int32, "step"); let diff = ctx.builder.build_int_sub(end, start, "diff"); - let diff_pos = ctx.builder.build_int_compare(SGT, diff, int64.const_zero(), "diffpos"); - let step_pos = ctx.builder.build_int_compare(SGT, step, int64.const_zero(), "steppos"); + let diff_pos = ctx.builder.build_int_compare(SGT, diff, int32.const_zero(), "diffpos"); + let step_pos = ctx.builder.build_int_compare(SGT, step, int32.const_zero(), "steppos"); let test_1 = ctx.builder.build_and(diff_pos, step_pos, "bothpos"); let current = ctx.builder.get_insert_block().unwrap().get_parent().unwrap(); @@ -654,41 +654,41 @@ pub fn calculate_len_for_slice_range<'ctx, 'a>( ctx.builder.position_at_end(then_bb); let length_pos = { - let diff_pos_min_1 = ctx.builder.build_int_sub(diff, int64.const_int(1, false), "diffminone"); + let diff_pos_min_1 = ctx.builder.build_int_sub(diff, int32.const_int(1, false), "diffminone"); let length_pos = ctx.builder.build_int_signed_div(diff_pos_min_1, step, "div"); - ctx.builder.build_int_add(length_pos, int64.const_int(1, false), "add1") + ctx.builder.build_int_add(length_pos, int32.const_int(1, false), "add1") }; ctx.builder.build_unconditional_branch(cont_bb); ctx.builder.position_at_end(else_bb); let phi_1 = { - let diff_neg = ctx.builder.build_int_compare(SLT, diff, int64.const_zero(), "diffneg"); - let step_neg = ctx.builder.build_int_compare(SLT, step, int64.const_zero(), "stepneg"); + let diff_neg = ctx.builder.build_int_compare(SLT, diff, int32.const_zero(), "diffneg"); + let step_neg = ctx.builder.build_int_compare(SLT, step, int32.const_zero(), "stepneg"); let test_2 = ctx.builder.build_and(diff_neg, step_neg, "bothneg"); ctx.builder.build_conditional_branch(test_2, then_bb_2, else_bb_2); ctx.builder.position_at_end(then_bb_2); let length_neg = { - let diff_neg_add_1 = ctx.builder.build_int_add(diff, int64.const_int(1, false), "diffminone"); + let diff_neg_add_1 = ctx.builder.build_int_add(diff, int32.const_int(1, false), "diffminone"); let length_neg = ctx.builder.build_int_signed_div(diff_neg_add_1, step, "div"); - ctx.builder.build_int_add(length_neg, int64.const_int(1, false), "add1") + ctx.builder.build_int_add(length_neg, int32.const_int(1, false), "add1") }; ctx.builder.build_unconditional_branch(cont_bb_2); ctx.builder.position_at_end(else_bb_2); - let length_zero = int64.const_zero(); + let length_zero = int32.const_zero(); ctx.builder.build_unconditional_branch(cont_bb_2); ctx.builder.position_at_end(cont_bb_2); - let phi_1 = ctx.builder.build_phi(int64, "lenphi1"); + let phi_1 = ctx.builder.build_phi(int32, "lenphi1"); phi_1.add_incoming(&[(&length_neg, then_bb_2), (&length_zero, else_bb_2)]); phi_1.as_basic_value().into_int_value() }; ctx.builder.build_unconditional_branch(cont_bb); ctx.builder.position_at_end(cont_bb); - let phi = ctx.builder.build_phi(int64, "lenphi"); + let phi = ctx.builder.build_phi(int32, "lenphi"); phi.add_incoming(&[(&length_pos, then_bb), (&phi_1, cont_bb_2)]); phi.as_basic_value().into_int_value() -} \ No newline at end of file +} From 0ff995722c99c9b4260eff9d94c581be0088f7f6 Mon Sep 17 00:00:00 2001 From: Sebastien Bourdeauducq Date: Mon, 20 Dec 2021 18:13:45 +0800 Subject: [PATCH 54/54] Revert "nac3core: add missing expr concrete type check" This reverts commit cb450372d657d5d859f633fbeaf88fa80404bbb3. --- nac3core/src/typecheck/function_check.rs | 2 -- 1 file changed, 2 deletions(-) diff --git a/nac3core/src/typecheck/function_check.rs b/nac3core/src/typecheck/function_check.rs index b65f4995..65c810e0 100644 --- a/nac3core/src/typecheck/function_check.rs +++ b/nac3core/src/typecheck/function_check.rs @@ -24,13 +24,11 @@ impl<'a> Inferencer<'a> { if !defined_identifiers.contains(id) { defined_identifiers.insert(*id); } - self.check_expr(pattern, defined_identifiers)?; self.should_have_value(pattern)?; Ok(()) } ExprKind::Tuple { elts, .. } => { for elt in elts.iter() { - self.check_expr(pattern, defined_identifiers)?; self.check_pattern(elt, defined_identifiers)?; self.should_have_value(elt)?; }