forked from M-Labs/nac3
1
0
Fork 0

core: WIP - Add tracer runtime

This commit is contained in:
David Mak 2024-06-01 15:10:43 +08:00
parent 25d2de67f7
commit a9ca91c387
11 changed files with 300 additions and 27 deletions

View File

@ -47,6 +47,7 @@ use parking_lot::{Mutex, RwLock};
use nac3core::{ use nac3core::{
codegen::irrt::load_irrt, codegen::irrt::load_irrt,
codegen::tracert::TraceRuntimeConfig,
codegen::{concrete_type::ConcreteTypeStore, CodeGenTask, WithCall, WorkerRegistry}, codegen::{concrete_type::ConcreteTypeStore, CodeGenTask, WithCall, WorkerRegistry},
symbol_resolver::SymbolResolver, symbol_resolver::SymbolResolver,
toplevel::{ toplevel::{
@ -635,8 +636,13 @@ impl Nac3 {
let membuffer = membuffers.clone(); let membuffer = membuffers.clone();
py.allow_threads(|| { py.allow_threads(|| {
let (registry, handles) = let (registry, handles) = WorkerRegistry::create_workers(
WorkerRegistry::create_workers(threads, top_level.clone(), &self.llvm_options, &f); threads,
top_level.clone(),
&self.llvm_options,
&TraceRuntimeConfig::default(),
&f,
);
registry.add_task(task); registry.add_task(task);
registry.wait_tasks_complete(handles); registry.wait_tasks_complete(handles);

View File

@ -4,6 +4,9 @@ version = "0.1.0"
authors = ["M-Labs"] authors = ["M-Labs"]
edition = "2021" edition = "2021"
[features]
tracing = []
[dependencies] [dependencies]
itertools = "0.13" itertools = "0.13"
crossbeam = "0.8" crossbeam = "0.8"
@ -25,4 +28,5 @@ indoc = "2.0"
insta = "=1.11.0" insta = "=1.11.0"
[build-dependencies] [build-dependencies]
itertools = "0.13"
regex = "1.10" regex = "1.10"

View File

@ -1,4 +1,4 @@
use regex::Regex; use std::ffi::OsStr;
use std::{ use std::{
env, env,
fs::File, fs::File,
@ -7,16 +7,23 @@ use std::{
process::{Command, Stdio}, process::{Command, Stdio},
}; };
fn main() { use itertools::Itertools;
const FILE: &str = "src/codegen/irrt/irrt.cpp"; use regex::Regex;
/// Extracts the extension-less filename from a [`Path`].
fn path_to_extless_filename(path: &Path) -> &str {
path.file_name().map(Path::new).and_then(Path::file_stem).and_then(OsStr::to_str).unwrap()
}
/// Compiles a source C file into LLVM bitcode.
fn compile_file_to_ir(path: &Path, filename_without_ext: &str) {
/* /*
* HACK: Sadly, clang doesn't let us emit generic LLVM bitcode. * HACK: Sadly, clang doesn't let us emit generic LLVM bitcode.
* Compiling for WASM32 and filtering the output with regex is the closest we can get. * Compiling for WASM32 and filtering the output with regex is the closest we can get.
*/ */
let flags: &[&str] = &[ let flags: &[&str] = &[
"--target=wasm32", "--target=wasm32",
FILE, path.to_str().unwrap(),
"-x", "c++", "-x", "c++",
"-fno-discard-value-names", "-fno-discard-value-names",
"-fno-exceptions", "-fno-exceptions",
@ -34,7 +41,7 @@ fn main() {
"-", "-",
]; ];
println!("cargo:rerun-if-changed={FILE}"); println!("cargo:rerun-if-changed={}", path.to_str().unwrap());
let out_dir = env::var("OUT_DIR").unwrap(); let out_dir = env::var("OUT_DIR").unwrap();
let out_path = Path::new(&out_dir); let out_path = Path::new(&out_dir);
@ -47,12 +54,11 @@ fn main() {
}) })
.unwrap(); .unwrap();
// https://github.com/rust-lang/regex/issues/244 let output = std::str::from_utf8(&output.stdout).unwrap();
let output = std::str::from_utf8(&output.stdout).unwrap().replace("\r\n", "\n");
let mut filtered_output = String::with_capacity(output.len()); let mut filtered_output = String::with_capacity(output.len());
let regex_filter = Regex::new(r"(?ms:^define.*?\}$)|(?m:^declare.*?$)").unwrap(); let regex_filter = Regex::new(r"(?ms:^define.*?\}$)|(?m:^declare.*?$)").unwrap();
for f in regex_filter.captures_iter(&output) { for f in regex_filter.captures_iter(output) {
assert_eq!(f.len(), 1); assert_eq!(f.len(), 1);
filtered_output.push_str(&f[0]); filtered_output.push_str(&f[0]);
filtered_output.push('\n'); filtered_output.push('\n');
@ -64,18 +70,32 @@ fn main() {
println!("cargo:rerun-if-env-changed=DEBUG_DUMP_IRRT"); println!("cargo:rerun-if-env-changed=DEBUG_DUMP_IRRT");
if env::var("DEBUG_DUMP_IRRT").is_ok() { if env::var("DEBUG_DUMP_IRRT").is_ok() {
let mut file = File::create(out_path.join("irrt.ll")).unwrap(); let mut file = File::create(out_path.join(format!("{filename_without_ext}.ll"))).unwrap();
file.write_all(output.as_bytes()).unwrap(); file.write_all(output.as_bytes()).unwrap();
let mut file = File::create(out_path.join("irrt-filtered.ll")).unwrap(); let mut file =
File::create(out_path.join(format!("{filename_without_ext}-filtered.ll"))).unwrap();
file.write_all(filtered_output.as_bytes()).unwrap(); file.write_all(filtered_output.as_bytes()).unwrap();
} }
let mut llvm_as = Command::new("llvm-as-irrt") let mut llvm_as = Command::new("llvm-as-irrt")
.stdin(Stdio::piped()) .stdin(Stdio::piped())
.arg("-o") .arg("-o")
.arg(out_path.join("irrt.bc")) .arg(out_path.join(format!("{filename_without_ext}.bc")))
.spawn() .spawn()
.unwrap(); .unwrap();
llvm_as.stdin.as_mut().unwrap().write_all(filtered_output.as_bytes()).unwrap(); llvm_as.stdin.as_mut().unwrap().write_all(filtered_output.as_bytes()).unwrap();
assert!(llvm_as.wait().unwrap().success()); assert!(llvm_as.wait().unwrap().success());
} }
fn main() {
const IRRT_SOURCE_PATHS: &[&str] =
&["src/codegen/irrt/irrt.cpp", "src/codegen/tracert/tracert.cpp"];
assert!(IRRT_SOURCE_PATHS.iter().map(Path::new).map(path_to_extless_filename).all_unique());
for path in IRRT_SOURCE_PATHS {
let path = Path::new(path);
compile_file_to_ir(path, path_to_extless_filename(path))
}
}

View File

@ -1,6 +1,8 @@
use inkwell::attributes::{Attribute, AttributeLoc}; use inkwell::attributes::{Attribute, AttributeLoc};
use inkwell::values::{BasicValueEnum, CallSiteValue, FloatValue, IntValue}; use inkwell::values::{BasicValue, BasicValueEnum, CallSiteValue, FloatValue, IntValue};
use itertools::Either; use inkwell::AddressSpace;
use itertools::{Either, Itertools};
use std::iter::once;
use crate::codegen::CodeGenContext; use crate::codegen::CodeGenContext;
@ -611,3 +613,49 @@ pub fn call_nextafter<'ctx>(
.map(Either::unwrap_left) .map(Either::unwrap_left)
.unwrap() .unwrap()
} }
/// Invokes the `printf` function.
pub fn call_printf<'ctx>(
ctx: &CodeGenContext<'ctx, '_>,
format: &str,
args: &[BasicValueEnum<'ctx>],
) -> IntValue<'ctx> {
const FN_NAME: &str = "printf";
let llvm_i8 = ctx.ctx.i8_type();
let llvm_i32 = ctx.ctx.i32_type();
let llvm_pi8 = llvm_i8.ptr_type(AddressSpace::default());
let extern_fn = ctx.module.get_function(FN_NAME).unwrap_or_else(|| {
let fn_type = llvm_i32.fn_type(&[llvm_pi8.into()], true);
let func = ctx.module.add_function(FN_NAME, fn_type, None);
for attr in ["nofree", "nounwind"] {
func.add_attribute(
AttributeLoc::Function,
ctx.ctx.create_enum_attribute(Attribute::get_named_enum_kind_id(attr), 0),
);
}
func
});
let pformat = ctx
.builder
.build_global_string_ptr(&format!("{format}\0"), "")
.map(|v| v.as_basic_value_enum())
.map(BasicValueEnum::into_pointer_value)
.unwrap();
let fn_args = once(&pformat.as_basic_value_enum())
.chain(args)
.copied()
.map(BasicValueEnum::into)
.collect_vec();
ctx.builder
.build_call(extern_fn, fn_args.as_slice(), "")
.map(CallSiteValue::try_as_basic_value)
.map(|v| v.map_left(BasicValueEnum::into_int_value))
.map(Either::unwrap_left)
.unwrap()
}

View File

@ -1,5 +1,6 @@
use crate::{ use crate::{
codegen::classes::{ListType, NDArrayType, ProxyType, RangeType}, codegen::classes::{ListType, NDArrayType, ProxyType, RangeType},
codegen::tracert::{TraceRuntimeConfig, TraceRuntimeState},
symbol_resolver::{StaticValue, SymbolResolver}, symbol_resolver::{StaticValue, SymbolResolver},
toplevel::{helper::PrimDef, numpy::unpack_ndarray_var_tys, TopLevelContext, TopLevelDef}, toplevel::{helper::PrimDef, numpy::unpack_ndarray_var_tys, TopLevelContext, TopLevelDef},
typecheck::{ typecheck::{
@ -43,6 +44,7 @@ pub mod irrt;
pub mod llvm_intrinsics; pub mod llvm_intrinsics;
pub mod numpy; pub mod numpy;
pub mod stmt; pub mod stmt;
pub mod tracert;
#[cfg(test)] #[cfg(test)]
mod test; mod test;
@ -189,6 +191,8 @@ pub struct CodeGenContext<'ctx, 'a> {
/// See [`need_sret`]. /// See [`need_sret`].
pub need_sret: bool, pub need_sret: bool,
pub tracert_state: Option<TraceRuntimeState>,
/// The current source location. /// The current source location.
pub current_loc: Location, pub current_loc: Location,
} }
@ -236,6 +240,8 @@ pub struct WorkerRegistry {
/// LLVM-related options for code generation. /// LLVM-related options for code generation.
pub llvm_options: CodeGenLLVMOptions, pub llvm_options: CodeGenLLVMOptions,
tracert_config: TraceRuntimeConfig,
} }
impl WorkerRegistry { impl WorkerRegistry {
@ -245,6 +251,7 @@ impl WorkerRegistry {
generators: Vec<Box<G>>, generators: Vec<Box<G>>,
top_level_ctx: Arc<TopLevelContext>, top_level_ctx: Arc<TopLevelContext>,
llvm_options: &CodeGenLLVMOptions, llvm_options: &CodeGenLLVMOptions,
tracert_config: &TraceRuntimeConfig,
f: &Arc<WithCall>, f: &Arc<WithCall>,
) -> (Arc<WorkerRegistry>, Vec<thread::JoinHandle<()>>) { ) -> (Arc<WorkerRegistry>, Vec<thread::JoinHandle<()>>) {
let (sender, receiver) = unbounded(); let (sender, receiver) = unbounded();
@ -266,6 +273,7 @@ impl WorkerRegistry {
wait_condvar, wait_condvar,
top_level_ctx, top_level_ctx,
llvm_options: llvm_options.clone(), llvm_options: llvm_options.clone(),
tracert_config: tracert_config.clone(),
}); });
let mut handles = Vec::new(); let mut handles = Vec::new();
@ -895,6 +903,13 @@ pub fn gen_func_impl<
unifier, unifier,
static_value_store, static_value_store,
need_sret: has_sret, need_sret: has_sret,
tracert_state: if cfg!(feature = "tracing")
|| registry.tracert_config.enabled_tags.is_empty()
{
None
} else {
Some(TraceRuntimeState::create(registry.tracert_config.clone()))
},
current_loc: Location::default(), current_loc: Location::default(),
debug_info: (dibuilder, compile_unit, func_scope.as_debug_info_scope()), debug_info: (dibuilder, compile_unit, func_scope.as_debug_info_scope()),
}; };

View File

@ -2,6 +2,7 @@ use crate::{
codegen::{ codegen::{
classes::{ListType, NDArrayType, ProxyType, RangeType}, classes::{ListType, NDArrayType, ProxyType, RangeType},
concrete_type::ConcreteTypeStore, concrete_type::ConcreteTypeStore,
tracert::TraceRuntimeConfig,
CodeGenContext, CodeGenLLVMOptions, CodeGenTargetMachineOptions, CodeGenTask, CodeGenContext, CodeGenLLVMOptions, CodeGenTargetMachineOptions, CodeGenTask,
CodeGenerator, DefaultCodeGenerator, WithCall, WorkerRegistry, CodeGenerator, DefaultCodeGenerator, WithCall, WorkerRegistry,
}, },
@ -228,7 +229,13 @@ fn test_primitives() {
opt_level: OptimizationLevel::Default, opt_level: OptimizationLevel::Default,
target: CodeGenTargetMachineOptions::from_host_triple(), target: CodeGenTargetMachineOptions::from_host_triple(),
}; };
let (registry, handles) = WorkerRegistry::create_workers(threads, top_level, &llvm_options, &f); let (registry, handles) = WorkerRegistry::create_workers(
threads,
top_level,
&llvm_options,
&TraceRuntimeConfig::default(),
&f,
);
registry.add_task(task); registry.add_task(task);
registry.wait_tasks_complete(handles); registry.wait_tasks_complete(handles);
} }
@ -413,7 +420,13 @@ fn test_simple_call() {
opt_level: OptimizationLevel::Default, opt_level: OptimizationLevel::Default,
target: CodeGenTargetMachineOptions::from_host_triple(), target: CodeGenTargetMachineOptions::from_host_triple(),
}; };
let (registry, handles) = WorkerRegistry::create_workers(threads, top_level, &llvm_options, &f); let (registry, handles) = WorkerRegistry::create_workers(
threads,
top_level,
&llvm_options,
&TraceRuntimeConfig::default(),
&f,
);
registry.add_task(task); registry.add_task(task);
registry.wait_tasks_complete(handles); registry.wait_tasks_complete(handles);
} }

View File

@ -0,0 +1,110 @@
use std::panic::Location;
use inkwell::context::Context;
use inkwell::memory_buffer::MemoryBuffer;
use inkwell::module::Module;
use inkwell::values::BasicValueEnum;
use inkwell::AtomicOrdering;
use crate::codegen::{extern_fns, CodeGenContext};
#[derive(Clone, Default, Eq, PartialEq)]
pub struct TraceRuntimeConfig {
pub enabled_tags: Vec<String>,
}
#[derive(Eq, Clone, PartialEq)]
pub struct TraceRuntimeState {
config: TraceRuntimeConfig,
indent: usize,
}
impl TraceRuntimeState {
#[must_use]
pub fn create(config: TraceRuntimeConfig) -> TraceRuntimeState {
TraceRuntimeState { config, indent: 0 }
}
}
#[must_use]
pub fn load_tracert<'ctx>(ctx: &'ctx Context, config: &TraceRuntimeConfig) -> Option<Module<'ctx>> {
if cfg!(feature = "tracing") && !config.enabled_tags.is_empty() {
let bitcode_buf = MemoryBuffer::create_from_memory_range(
include_bytes!(concat!(env!("OUT_DIR"), "/tracert.bc")),
"tracert_bitcode_buffer",
);
let module = Module::parse_bitcode_from_buffer(&bitcode_buf, ctx).unwrap();
return Some(module);
}
None
}
// TODO: Might need to redesign how trace logging should be done
pub fn trace_log<'ctx>(
ctx: &mut CodeGenContext<'ctx, '_>,
tag: &'static str,
format: &'static str,
args: &[BasicValueEnum<'ctx>],
) {
if ctx.tracert_state.is_none() {
return;
}
// TODO: Add indentation
let str = format!("[TRACING] {tag} - {format}\n\0");
extern_fns::call_printf(ctx, &str, args);
}
#[track_caller]
pub fn trace_log_with_location<'ctx>(
ctx: &CodeGenContext<'ctx, '_>,
tag: &'static str,
format: &str,
args: &[BasicValueEnum<'ctx>],
) {
if ctx.tracert_state.is_none() {
return;
}
// TODO: Add indentation
let caller_loc = Location::caller();
let str = format!(
"[TRACING] {}:{}:{}: {tag} - {format}\n\0",
caller_loc.file(),
caller_loc.line(),
caller_loc.column()
);
extern_fns::call_printf(ctx, &str, args);
}
pub fn trace_push_level(ctx: &mut CodeGenContext<'_, '_>) {
let Some(tracert_state) = &mut ctx.tracert_state else {
return;
};
debug_assert!(tracert_state.indent < usize::MAX);
if tracert_state.indent < usize::MAX {
tracert_state.indent = tracert_state.indent.saturating_add(1);
}
}
pub fn trace_pop_level(ctx: &mut CodeGenContext<'_, '_>) {
let Some(tracert_state) = &mut ctx.tracert_state else {
return;
};
debug_assert!(tracert_state.indent > 0);
if tracert_state.indent > 0 {
tracert_state.indent = tracert_state.indent.saturating_sub(1);
}
}
#[inline]
pub fn mfence(ctx: &mut CodeGenContext<'_, '_>) {
if ctx.tracert_state.is_some() {
ctx.builder.build_fence(AtomicOrdering::SequentiallyConsistent, 0, "").unwrap();
}
}

View File

@ -0,0 +1,4 @@
extern "C" {
// stdio.h
int printf(const char *format, ...);
} // extern "C"

View File

@ -4,6 +4,9 @@ version = "0.1.0"
authors = ["M-Labs"] authors = ["M-Labs"]
edition = "2021" edition = "2021"
[features]
tracing = ["nac3core/tracing"]
[dependencies] [dependencies]
parking_lot = "0.12" parking_lot = "0.12"
nac3parser = { path = "../nac3parser" } nac3parser = { path = "../nac3parser" }

View File

@ -70,7 +70,7 @@ struct cslice {
usize len; usize len;
}; };
void output_int32_list(struct cslice *slice) { void output_int32_list(const struct cslice *slice) {
const int32_t *data = (int32_t *) slice->data; const int32_t *data = (int32_t *) slice->data;
putchar('['); putchar('[');
@ -85,7 +85,7 @@ void output_int32_list(struct cslice *slice) {
putchar('\n'); putchar('\n');
} }
void output_str(struct cslice *slice) { void output_str(const struct cslice *slice) {
const char *data = (const char *) slice->data; const char *data = (const char *) slice->data;
for (usize i = 0; i < slice->len; ++i) { for (usize i = 0; i < slice->len; ++i) {
@ -93,12 +93,12 @@ void output_str(struct cslice *slice) {
} }
} }
void output_strln(struct cslice *slice) { void output_strln(const struct cslice *slice) {
output_str(slice); output_str(slice);
putchar('\n'); putchar('\n');
} }
uint64_t dbg_stack_address(__attribute__((unused)) struct cslice *slice) { uint64_t dbg_stack_address(__attribute__((unused)) const struct cslice *slice) {
int i; int i;
void *ptr = (void *) &i; void *ptr = (void *) &i;
return (uintptr_t) ptr; return (uintptr_t) ptr;
@ -110,8 +110,34 @@ uint32_t __nac3_personality(uint32_t state, uint32_t exception_object, uint32_t
__builtin_unreachable(); __builtin_unreachable();
} }
struct exception {
uint32_t name;
struct cslice file;
uint32_t line;
uint32_t col;
struct cslice func;
struct cslice message;
uint64_t param0;
uint64_t param1;
uint64_t param2;
};
void output_exception(const struct exception *ex) {
fputs("exception { location: ", stdout);
output_str(&ex->file);
printf(":%u:%u, func: ", ex->line, ex->col);
output_str(&ex->func);
fputs(", message: ", stdout);
output_str(&ex->message);
printf(", params: [%lu, %lu, %lu] }\n", ex->param0, ex->param1, ex->param2);
}
uint32_t __nac3_raise(uint32_t state, uint32_t exception_object, uint32_t context) { uint32_t __nac3_raise(uint32_t state, uint32_t exception_object, uint32_t context) {
printf("__nac3_raise(state: %u, exception_object: %u, context: %u)\n", state, exception_object, context); printf("__nac3_raise(state: %#10x, exception_object: %#10x, context: %#10x)\n", state, exception_object, context);
struct exception *ex = (struct exception *)((0x7ffffffful << 16) | state);
output_exception(ex);
exit(101); exit(101);
__builtin_unreachable(); __builtin_unreachable();
} }

View File

@ -20,8 +20,11 @@ use std::{collections::HashMap, fs, path::Path, sync::Arc};
use nac3core::{ use nac3core::{
codegen::{ codegen::{
concrete_type::ConcreteTypeStore, irrt::load_irrt, CodeGenLLVMOptions, concrete_type::ConcreteTypeStore,
CodeGenTargetMachineOptions, CodeGenTask, DefaultCodeGenerator, WithCall, WorkerRegistry, irrt::load_irrt,
tracert::{load_tracert, TraceRuntimeConfig},
CodeGenLLVMOptions, CodeGenTargetMachineOptions, CodeGenTask, DefaultCodeGenerator,
WithCall, WorkerRegistry,
}, },
symbol_resolver::SymbolResolver, symbol_resolver::SymbolResolver,
toplevel::{ toplevel::{
@ -76,6 +79,9 @@ struct CommandLineArgs {
/// Additional target features to enable/disable, specified using the `+`/`-` prefixes. /// Additional target features to enable/disable, specified using the `+`/`-` prefixes.
#[arg(long)] #[arg(long)]
target_features: Option<String>, target_features: Option<String>,
#[arg(long)]
trace: Vec<String>,
} }
fn handle_typevar_definition( fn handle_typevar_definition(
@ -242,8 +248,16 @@ fn main() {
const SIZE_T: u32 = usize::BITS; const SIZE_T: u32 = usize::BITS;
let cli = CommandLineArgs::parse(); let cli = CommandLineArgs::parse();
let CommandLineArgs { file_name, threads, opt_level, emit_llvm, triple, mcpu, target_features } = let CommandLineArgs {
cli; file_name,
threads,
opt_level,
emit_llvm,
triple,
mcpu,
target_features,
trace,
} = cli;
Target::initialize_all(&InitializationConfig::default()); Target::initialize_all(&InitializationConfig::default());
@ -272,6 +286,7 @@ fn main() {
// The default behavior for -O<n> where n>3 defaults to O3 for both Clang and GCC // The default behavior for -O<n> where n>3 defaults to O3 for both Clang and GCC
_ => OptimizationLevel::Aggressive, _ => OptimizationLevel::Aggressive,
}; };
let tracert_config = TraceRuntimeConfig { enabled_tags: trace };
let program = match fs::read_to_string(file_name.clone()) { let program = match fs::read_to_string(file_name.clone()) {
Ok(program) => program, Ok(program) => program,
@ -405,7 +420,8 @@ fn main() {
let threads = (0..threads) let threads = (0..threads)
.map(|i| Box::new(DefaultCodeGenerator::new(format!("module{i}"), SIZE_T))) .map(|i| Box::new(DefaultCodeGenerator::new(format!("module{i}"), SIZE_T)))
.collect(); .collect();
let (registry, handles) = WorkerRegistry::create_workers(threads, top_level, &llvm_options, &f); let (registry, handles) =
WorkerRegistry::create_workers(threads, top_level, &llvm_options, &tracert_config, &f);
registry.add_task(task); registry.add_task(task);
registry.wait_tasks_complete(handles); registry.wait_tasks_complete(handles);
@ -436,6 +452,14 @@ fn main() {
} }
main.link_in_module(irrt).unwrap(); main.link_in_module(irrt).unwrap();
if let Some(tracert) = load_tracert(&context, &tracert_config) {
if emit_llvm {
tracert.write_bitcode_to_path(Path::new("tracert.bc"));
}
main.link_in_module(tracert).unwrap();
}
let mut function_iter = main.get_first_function(); let mut function_iter = main.get_first_function();
while let Some(func) = function_iter { while let Some(func) = function_iter {
if func.count_basic_blocks() > 0 && func.get_name().to_str().unwrap() != "run" { if func.count_basic_blocks() > 0 && func.get_name().to_str().unwrap() != "run" {