threadpool for parallel code generation

This commit is contained in:
pca006132 2021-08-13 14:48:46 +08:00
parent cb01c79603
commit e2adf82229
4 changed files with 184 additions and 80 deletions

View File

@ -6,6 +6,7 @@ use crate::{
typedef::{FunSignature, Type, TypeEnum, Unifier},
},
};
use crossbeam::channel::{unbounded, Receiver, Sender};
use inkwell::{
basic_block::BasicBlock,
builder::Builder,
@ -16,9 +17,11 @@ use inkwell::{
AddressSpace,
};
use itertools::Itertools;
use parking_lot::{Condvar, Mutex};
use rustpython_parser::ast::Stmt;
use std::collections::HashMap;
use std::sync::Arc;
use std::thread;
mod expr;
mod stmt;
@ -43,6 +46,112 @@ pub struct CodeGenContext<'ctx, 'a> {
pub loop_bb: Option<(BasicBlock<'ctx>, BasicBlock<'ctx>)>,
}
type Fp = Box<dyn Fn(&Module) + Send + Sync>;
pub struct WithCall {
fp: Fp,
}
impl WithCall {
pub fn new(fp: Fp) -> WithCall {
WithCall { fp }
}
pub fn run<'ctx>(&self, m: &Module<'ctx>) {
(self.fp)(m)
}
}
pub struct WorkerRegistry {
sender: Arc<Sender<Option<CodeGenTask>>>,
receiver: Arc<Receiver<Option<CodeGenTask>>>,
task_count: Mutex<usize>,
thread_count: usize,
wait_condvar: Condvar,
}
impl WorkerRegistry {
pub fn create_workers(
names: &[&str],
top_level_ctx: Arc<TopLevelContext>,
f: Arc<WithCall>,
) -> Arc<WorkerRegistry> {
let (sender, receiver) = unbounded();
let task_count = Mutex::new(0);
let wait_condvar = Condvar::new();
let registry = Arc::new(WorkerRegistry {
sender: Arc::new(sender),
receiver: Arc::new(receiver),
thread_count: names.len(),
task_count,
wait_condvar,
});
for name in names.iter() {
let top_level_ctx = top_level_ctx.clone();
let registry = registry.clone();
let name = name.to_string();
let f = f.clone();
thread::spawn(move || {
registry.worker_thread(name, top_level_ctx, f);
});
}
registry
}
pub fn wait_tasks_complete(&self) {
{
let mut count = self.task_count.lock();
while *count != 0 {
self.wait_condvar.wait(&mut count);
}
}
for _ in 0..self.thread_count {
self.sender.send(None).unwrap();
}
{
let mut count = self.task_count.lock();
while *count != self.thread_count {
self.wait_condvar.wait(&mut count);
}
}
}
pub fn add_task(&self, task: CodeGenTask) {
*self.task_count.lock() += 1;
self.sender.send(Some(task)).unwrap();
}
fn worker_thread(
&self,
module_name: String,
top_level_ctx: Arc<TopLevelContext>,
f: Arc<WithCall>,
) {
let context = Context::create();
let mut builder = context.create_builder();
let mut module = context.create_module(&module_name);
while let Some(task) = self.receiver.recv().unwrap() {
let result = gen_func(&context, builder, module, task, top_level_ctx.clone());
builder = result.0;
module = result.1;
println!("{}", *self.task_count.lock());
*self.task_count.lock() -= 1;
self.wait_condvar.notify_all();
}
// do whatever...
let mut lock = self.task_count.lock();
module.verify().unwrap();
f.run(&module);
*lock += 1;
self.wait_condvar.notify_all();
}
}
pub struct CodeGenTask {
pub subst: Vec<(Type, Type)>,
pub symbol_name: String,

View File

@ -1,5 +1,6 @@
use super::{gen_func, CodeGenTask};
use super::{CodeGenTask, WorkerRegistry};
use crate::{
codegen::WithCall,
location::Location,
symbol_resolver::{SymbolResolver, SymbolValue},
top_level::{DefinitionId, TopLevelContext},
@ -10,7 +11,6 @@ use crate::{
},
};
use indoc::indoc;
use inkwell::context::Context;
use parking_lot::RwLock;
use rustpython_parser::{ast::fold::Fold, parser::parse_program};
use std::collections::HashMap;
@ -109,7 +109,7 @@ impl TestEnvironment {
top_level: TopLevelContext {
definitions: Default::default(),
unifiers: Default::default(),
conetexts: Default::default(),
// conetexts: Default::default(),
},
function_data: FunctionData {
resolver,
@ -140,10 +140,7 @@ impl TestEnvironment {
#[test]
fn test_primitives() {
let mut env = TestEnvironment::basic_test_env();
let context = Context::create();
let module = context.create_module("test");
let builder = context.create_builder();
let threads = ["test"];
let signature = FunSignature {
args: vec![
FuncArg { name: "a".to_string(), ty: env.primitives.int32, default_value: None },
@ -170,9 +167,8 @@ fn test_primitives() {
let top_level = Arc::new(TopLevelContext {
definitions: Default::default(),
unifiers: Arc::new(RwLock::new(vec![(env.unifier.get_shared_unifier(), env.primitives)])),
conetexts: Default::default(),
// conetexts: Default::default(),
});
let task = CodeGenTask {
subst: Default::default(),
symbol_name: "testing".to_string(),
@ -182,7 +178,7 @@ fn test_primitives() {
signature,
};
let module = gen_func(&context, builder, module, task, top_level);
let f = Arc::new(WithCall::new(Box::new(|module| {
// the following IR is equivalent to
// ```
// ; ModuleID = 'test.ll'
@ -239,8 +235,9 @@ fn test_primitives() {
}
"}
.trim();
let ir = module.1.print_to_string().to_string();
println!("src:\n{}", source);
println!("IR:\n{}", ir);
assert_eq!(expected, ir.trim());
assert_eq!(expected, module.print_to_string().to_str().unwrap().trim());
})));
let registry = WorkerRegistry::create_workers(&threads, top_level, f);
registry.add_task(task);
registry.wait_tasks_complete();
}

View File

@ -4,7 +4,6 @@ use std::{collections::HashMap, sync::Arc};
use super::typecheck::type_inferencer::PrimitiveStore;
use super::typecheck::typedef::{SharedUnifier, Type, TypeEnum, Unifier};
use crate::symbol_resolver::SymbolResolver;
use inkwell::context::Context;
use parking_lot::{Mutex, RwLock};
use rustpython_parser::ast::{self, Stmt};
@ -54,7 +53,6 @@ pub enum TopLevelDef {
pub struct TopLevelContext {
pub definitions: Arc<RwLock<Vec<RwLock<TopLevelDef>>>>,
pub unifiers: Arc<RwLock<Vec<(SharedUnifier, PrimitiveStore)>>>,
pub conetexts: Arc<RwLock<Vec<Mutex<Context>>>>,
}
// like adding some info on top of the TopLevelDef for

View File

@ -100,7 +100,7 @@ impl TestEnvironment {
top_level: TopLevelContext {
definitions: Default::default(),
unifiers: Default::default(),
conetexts: Default::default(),
// conetexts: Default::default(),
},
unifier,
function_data: FunctionData {
@ -259,7 +259,7 @@ impl TestEnvironment {
let top_level = TopLevelContext {
definitions: Arc::new(RwLock::new(top_level_defs)),
unifiers: Default::default(),
conetexts: Default::default(),
// conetexts: Default::default(),
};
let resolver = Arc::new(Resolver {