Compare commits

..

9 Commits

9 changed files with 175 additions and 242 deletions

View File

@ -1,25 +1,30 @@
# NAC3 compiler
# NAC3
NAC3 is a major, backward-incompatible rewrite of the compiler for the [ARTIQ](https://m-labs.hk/artiq) physics experiment control and data acquisition system. It features greatly improved compilation speeds, a much better type system, and more predictable and transparent operation.
NAC3 has a modular design and its applicability reaches beyond ARTIQ. The ``nac3core`` module does not contain anything specific to ARTIQ, and can be used in any project that requires compiling Python to machine code.
**WARNING: NAC3 is currently experimental software and several important features are not implemented yet.**
## Packaging
NAC3 is packaged using the [Nix](https://nixos.org) Flakes system. Install Nix 2.4+ and enable flakes by adding ``experimental-features = nix-command flakes`` to ``nix.conf`` (e.g. ``~/.config/nix/nix.conf``).
## Try NAC3
After setting up Nix as above, use ``nix shell github:m-labs/artiq/nac3`` to get a shell with the NAC3 version of ARTIQ. See the ``examples`` directory in ARTIQ (``nac3`` Git branch) for some samples of NAC3 kernel code.
## For developers
This repository contains:
- nac3ast: Python abstract syntax tree definition (based on RustPython).
- nac3parser: Python parser (based on RustPython).
- nac3core: Core compiler library, containing type-checking and code
generation.
- nac3standalone: Standalone compiler tool (core language only).
- nac3artiq: Integration with ARTIQ and implementation of ARTIQ-specific
extensions to the core language.
- runkernel: Simple program that runs compiled ARTIQ kernels on the host
and displays RTIO operations. Useful for testing without hardware.
- ``nac3ast``: Python abstract syntax tree definition (based on RustPython).
- ``nac3parser``: Python parser (based on RustPython).
- ``nac3core``: Core compiler library, containing type-checking and code generation.
- ``nac3standalone``: Standalone compiler tool (core language only).
- ``nac3artiq``: Integration with ARTIQ and implementation of ARTIQ-specific extensions to the core language.
- ``runkernel``: Simple program that runs compiled ARTIQ kernels on the host and displays RTIO operations. Useful for testing without hardware.
Use ``nix develop`` in this repository to enter a development shell.
If you are using a different shell than bash you can use e.g. ``nix develop --command fish``.
The core compiler knows nothing about symbol resolution, host variables
etc. nac3artiq and nac3standalone provide (implement) the
symbol resolver to the core compiler for resolving the type and value for
unknown symbols. The core compiler only type checks classes and functions
requested by nac3artiq/nac3standalone (the API should allow the
caller to specify which methods should be compiled). After type checking, the
compiler analyses the set of functions/classes that are used and performs
code generation.
value could be integer values, boolean values, bytes (for memcpy), function ID
(full name + concrete type)
Build NAC3 with ``cargo build --release``. See the demonstrations in ``nac3artiq`` and ``nac3standalone``.

View File

@ -1,73 +0,0 @@
from min_artiq import *
from numpy import int32
@nac3
class Base:
c: Kernel[int32]
def __init__(self):
self.c = 4
self.a = 3
self.a = 5
@nac3
class D:
da: KernelInvariant[int32]
def __init__(self):
self.da = 321
@nac3
class C:
ca: KernelInvariant[int32]
cb: KernelInvariant[D]
def __init__(self, d: D):
self.ca = 123
self.cb = d
@nac3
class A(Base):
core: KernelInvariant[Core]
led0: KernelInvariant[TTLOut]
led1: KernelInvariant[TTLOut]
d: Kernel[bool]
cc: Kernel[C]
def __init__(self, c):
super().__init__()
self.core = Core()
self.led0 = TTLOut(self.core, 18)
self.led1 = TTLOut(self.core, 19)
self.b = 3
self.d = False
self.cc = c
@kernel
def run(self):
print_int32(self.cc.cb.da)
if __name__ == '__main__':
d = D()
print(d.da)
c = C(d)
print(d.da)
print(c.cb.da)
a = A(c)
print(a.a)
print(a.b)
# print(c.ca) # fail
# print(c.cb.da) # fail
a.run()
# d = D() # redefine, ok
# c = C(d) # redefine, ok
# print(a.c) # fail
# a.c = 2 # fail
# a.d = 1 # fail
# a.cc = 1 # fail
# c.ca = 1 # fail
# c.cb = 1 # fail
# c.cb.da = 1 # fail
# d.da = 1 # fail

View File

@ -1,15 +1,15 @@
from inspect import getfullargspec, isclass, getmro
from inspect import getfullargspec
from functools import wraps
from types import SimpleNamespace
from numpy import int32, int64
from typing import Generic, TypeVar, get_origin
from typing import Generic, TypeVar
from math import floor, ceil
import nac3artiq
__all__ = [
"KernelInvariant", "Kernel", "virtual",
"Kernel", "KernelInvariant", "virtual",
"round64", "floor64", "ceil64",
"extern", "kernel", "portable", "nac3",
"ms", "us", "ns",
@ -21,16 +21,17 @@ __all__ = [
T = TypeVar('T')
class KernelInvariant(Generic[T]):
class Kernel(Generic[T]):
pass
class Kernel(Generic[T]):
class KernelInvariant(Generic[T]):
pass
# The virtual class must exist before nac3artiq.NAC3 is created.
class virtual(Generic[T]):
pass
def round64(x):
return round(x)
@ -46,7 +47,6 @@ core_arguments = device_db.device_db["core"]["arguments"]
compiler = nac3artiq.NAC3(core_arguments["target"])
allow_registration = True
allow_kernel_read = False
# Delay NAC3 analysis until all referenced variables are supposed to exist on the CPython side.
registered_functions = set()
registered_classes = set()
@ -93,65 +93,7 @@ def nac3(cls):
Decorates a class to be analyzed by NAC3.
All classes containing kernels or portable methods must use this decorator.
"""
# python does not allow setting magic method on specific instances
# (https://docs.python.org/3/reference/datamodel.html#special-method-lookup).
# use this set to keep track of those custom class instances that are
# assigned to the `Kernel` fields of a class
cls.__nac3_kernel_only_instances__ = set()
def apply_kernel_only_constraints(val):
kernel_only_set = getattr(type(val), '__nac3_kernel_only_instances__', None)
if kernel_only_set is None:
return
else:
for (_, attr_val) in val.__dict__.items():
if not (attr_val == val):
apply_kernel_only_constraints(attr_val)
kernel_only_set.add(val)
if not isclass(cls):
raise ValueError("nac3 annotation decorator should only be applied to classes")
if not cls.__setattr__ in {base.__setattr__ for base in cls.__bases__}:
raise ValueError("custom __setattr__ is not supported in kernel classes")
register_class(cls)
immutable_fields = {
n for b in getmro(cls)
for (n, ty) in b.__dict__.get('__annotations__', {}).items() if get_origin(ty) == Kernel
}
def __setattr__(obj, key, value):
if obj in type(obj).__nac3_kernel_only_instances__:
raise TypeError("attempting to write to kernel only variable")
# should allow init to set value, if no attribute then allow to set attr, then
# recursively apply constraint to all the fields of that specific object,
# regardless of whether they are marked with `Kernel` or not
if key in immutable_fields:
if hasattr(obj, key):
raise TypeError("attempting to write to kernel only variable")
else:
apply_kernel_only_constraints(value)
object.__setattr__(obj, key, value)
def __getattribute__(obj, key):
# need to use `object.__getattribute__` to get attr before checking
# the key in immutable_fields for `__init__`.
# since that in `__init__` when setting a instance variable like `self.a = 3`
# the sequence of internal magic call is still calling cls.__getattribute__(self, 'a')
# first, and if only "AttributeError" is raised, it will then call `__setattr__`
# if we raise `TypeError` too early, python will just halt at this `TypeError`.
attr = object.__getattribute__(obj, key)
if not allow_kernel_read:
if obj in type(obj).__nac3_kernel_only_instances__:
raise TypeError("attempting to read kernel only variable")
if key in immutable_fields:
raise TypeError("attempting to read kernel only variable")
return attr
cls.__setattr__ = __setattr__
cls.__getattribute__ = __getattribute__
return cls
@ -203,7 +145,6 @@ class Core:
self.ref_period = core_arguments["ref_period"]
def run(self, method, *args, **kwargs):
global allow_kernel_read
global allow_registration
if allow_registration:
compiler.analyze(registered_functions, registered_classes)
@ -215,9 +156,8 @@ class Core:
else:
obj = method
name = ""
allow_kernel_read = True
compiler.compile_method_to_file(obj, name, args, "module.elf")
allow_kernel_read = False
@kernel
def reset(self):

View File

@ -22,7 +22,7 @@ use parking_lot::{Mutex, RwLock};
use nac3core::{
codegen::{concrete_type::ConcreteTypeStore, CodeGenTask, WithCall, WorkerRegistry},
symbol_resolver::SymbolResolver,
toplevel::{composer::TopLevelComposer, DefinitionId, GenCall, TopLevelDef},
toplevel::{composer::{TopLevelComposer, CoreMode}, DefinitionId, GenCall, TopLevelDef},
typecheck::typedef::{FunSignature, FuncArg},
typecheck::{type_inferencer::PrimitiveStore, typedef::Type},
};
@ -239,7 +239,7 @@ impl Nac3 {
}))),
),
];
let (_, builtins_def, builtins_ty) = TopLevelComposer::new(builtins.clone());
let (_, builtins_def, builtins_ty) = TopLevelComposer::new(builtins.clone(), CoreMode::Artiq);
let builtins_mod = PyModule::import(py, "builtins").unwrap();
let id_fn = builtins_mod.getattr("id").unwrap();
@ -375,7 +375,7 @@ impl Nac3 {
filename: &str,
py: Python,
) -> PyResult<()> {
let (mut composer, _, _) = TopLevelComposer::new(self.builtins.clone());
let (mut composer, _, _) = TopLevelComposer::new(self.builtins.clone(), CoreMode::Artiq);
let mut id_to_def = HashMap::new();
let mut id_to_type = HashMap::new();

View File

@ -1,5 +1,5 @@
use std::cell::RefCell;
use inkwell::{IntPredicate, FloatPredicate, values::BasicValueEnum};
use inkwell::{IntPredicate::{self, *}, FloatPredicate, values::IntValue};
use crate::{symbol_resolver::SymbolValue, codegen::expr::destructure_range};
use super::*;
@ -69,10 +69,10 @@ pub fn get_builtins(primitives: &mut (PrimitiveStore, Unifier)) -> BuiltinInfo {
if ctx.unifier.unioned(arg_ty, boolean) {
Some(
ctx.builder
.build_int_s_extend(
.build_int_z_extend(
arg.into_int_value(),
ctx.ctx.i32_type(),
"sext",
"zext",
)
.into(),
)
@ -129,10 +129,10 @@ pub fn get_builtins(primitives: &mut (PrimitiveStore, Unifier)) -> BuiltinInfo {
{
Some(
ctx.builder
.build_int_s_extend(
.build_int_z_extend(
arg.into_int_value(),
ctx.ctx.i64_type(),
"sext",
"zext",
)
.into(),
)
@ -570,10 +570,10 @@ pub fn get_builtins(primitives: &mut (PrimitiveStore, Unifier)) -> BuiltinInfo {
ty: arg_ty.0,
default_value: None
}],
ret: int32,
ret: int64,
vars: vec![(list_var.1, list_var.0), (arg_ty.1, arg_ty.0)].into_iter().collect(),
}))),
var_id: Default::default(),
var_id: vec![arg_ty.1],
instance_to_symbol: Default::default(),
instance_to_stmt: Default::default(),
resolver: None,
@ -582,45 +582,13 @@ pub fn get_builtins(primitives: &mut (PrimitiveStore, Unifier)) -> BuiltinInfo {
let range_ty = ctx.primitives.range;
let arg_ty = fun.0.args[0].ty;
let arg = args[0].1;
let int32 = ctx.ctx.i32_type();
let zero = int32.const_zero();
if ctx.unifier.unioned(arg_ty, range_ty) {
let int1 = ctx.ctx.bool_type();
let one = int32.const_int(1, false);
let falze = int1.const_int(0, false);
let abs_intrinsic =
ctx.module.get_function("llvm.abs.i32").unwrap_or_else(|| {
let fn_type = int32.fn_type(&[int32.into(), int1.into()], false);
ctx.module.add_function("llvm.abs.i32", fn_type, None)
});
let arg = arg.into_pointer_value();
let (start, end, step) = destructure_range(ctx, arg);
let diff = ctx.builder.build_int_sub(end, start, "diff");
let diff = if let BasicValueEnum::IntValue(val) = ctx
.builder
.build_call(abs_intrinsic, &[diff.into(), falze.into()], "absdiff")
.try_as_basic_value()
.left()
.unwrap() {
val
} else {
unreachable!();
};
let diff = ctx.builder.build_int_sub(diff, one, "diff");
let step = if let BasicValueEnum::IntValue(val) = ctx
.builder
.build_call(abs_intrinsic, &[step.into(), falze.into()], "absstep")
.try_as_basic_value()
.left()
.unwrap() {
val
} else {
unreachable!();
};
let length = ctx.builder.build_int_signed_div(diff, step, "div");
let length = ctx.builder.build_int_add(length, int32.const_int(1, false), "add1");
Some(length.into())
Some(calculate_len_for_slice_range(ctx, start, end, step).into())
} else {
let int32 = ctx.ctx.i32_type();
let zero = int32.const_zero();
Some(ctx.build_gep_and_load(arg.into_pointer_value(), &[zero, zero]))
}
},
@ -648,4 +616,79 @@ pub fn get_builtins(primitives: &mut (PrimitiveStore, Unifier)) -> BuiltinInfo {
"len",
]
)
}
// equivalent code:
// def length(start, end, step != 0):
// diff = end - start
// if diff > 0 and step > 0:
// return ((diff - 1) // step) + 1
// elif diff < 0 and step < 0:
// return ((diff + 1) // step) + 1
// else:
// return 0
pub fn calculate_len_for_slice_range<'ctx, 'a>(
ctx: &mut CodeGenContext<'ctx, 'a>,
start: IntValue<'ctx>,
end: IntValue<'ctx>,
step: IntValue<'ctx>,
) -> IntValue<'ctx> {
let int64 = ctx.ctx.i64_type();
let start = ctx.builder.build_int_s_extend(start, int64, "start");
let end = ctx.builder.build_int_s_extend(end, int64, "end");
let step = ctx.builder.build_int_s_extend(step, int64, "step");
let diff = ctx.builder.build_int_sub(end, start, "diff");
let diff_pos = ctx.builder.build_int_compare(SGT, diff, int64.const_zero(), "diffpos");
let step_pos = ctx.builder.build_int_compare(SGT, step, int64.const_zero(), "steppos");
let test_1 = ctx.builder.build_and(diff_pos, step_pos, "bothpos");
let current = ctx.builder.get_insert_block().unwrap().get_parent().unwrap();
let then_bb = ctx.ctx.append_basic_block(current, "then");
let else_bb = ctx.ctx.append_basic_block(current, "else");
let then_bb_2 = ctx.ctx.append_basic_block(current, "then_2");
let else_bb_2 = ctx.ctx.append_basic_block(current, "else_2");
let cont_bb_2 = ctx.ctx.append_basic_block(current, "cont_2");
let cont_bb = ctx.ctx.append_basic_block(current, "cont");
ctx.builder.build_conditional_branch(test_1, then_bb, else_bb);
ctx.builder.position_at_end(then_bb);
let length_pos = {
let diff_pos_min_1 = ctx.builder.build_int_sub(diff, int64.const_int(1, false), "diffminone");
let length_pos = ctx.builder.build_int_signed_div(diff_pos_min_1, step, "div");
ctx.builder.build_int_add(length_pos, int64.const_int(1, false), "add1")
};
ctx.builder.build_unconditional_branch(cont_bb);
ctx.builder.position_at_end(else_bb);
let phi_1 = {
let diff_neg = ctx.builder.build_int_compare(SLT, diff, int64.const_zero(), "diffneg");
let step_neg = ctx.builder.build_int_compare(SLT, step, int64.const_zero(), "stepneg");
let test_2 = ctx.builder.build_and(diff_neg, step_neg, "bothneg");
ctx.builder.build_conditional_branch(test_2, then_bb_2, else_bb_2);
ctx.builder.position_at_end(then_bb_2);
let length_neg = {
let diff_neg_add_1 = ctx.builder.build_int_add(diff, int64.const_int(1, false), "diffminone");
let length_neg = ctx.builder.build_int_signed_div(diff_neg_add_1, step, "div");
ctx.builder.build_int_add(length_neg, int64.const_int(1, false), "add1")
};
ctx.builder.build_unconditional_branch(cont_bb_2);
ctx.builder.position_at_end(else_bb_2);
let length_zero = int64.const_zero();
ctx.builder.build_unconditional_branch(cont_bb_2);
ctx.builder.position_at_end(cont_bb_2);
let phi_1 = ctx.builder.build_phi(int64, "lenphi1");
phi_1.add_incoming(&[(&length_neg, then_bb_2), (&length_zero, else_bb_2)]);
phi_1.as_basic_value().into_int_value()
};
ctx.builder.build_unconditional_branch(cont_bb);
ctx.builder.position_at_end(cont_bb);
let phi = ctx.builder.build_phi(int64, "lenphi");
phi.add_incoming(&[(&length_pos, then_bb), (&phi_1, cont_bb_2)]);
phi.as_basic_value().into_int_value()
}

View File

@ -9,6 +9,11 @@ use crate::{
use super::*;
pub enum CoreMode {
Artiq,
Standalone
}
type DefAst = (Arc<RwLock<TopLevelDef>>, Option<ast::Stmt<()>>);
pub struct TopLevelComposer {
// list of top level definitions, same as top level context
@ -25,11 +30,13 @@ pub struct TopLevelComposer {
pub method_class: HashMap<DefinitionId, DefinitionId>,
// number of built-in function and classes in the definition list, later skip
pub builtin_num: usize,
// indicate the mode that we are using the core
pub mode: CoreMode,
}
impl Default for TopLevelComposer {
fn default() -> Self {
Self::new(vec![]).0
Self::new(vec![], CoreMode::Standalone).0
}
}
@ -38,6 +45,7 @@ impl TopLevelComposer {
/// resolver can later figure out primitive type definitions when passed a primitive type name
pub fn new(
builtins: Vec<(StrRef, FunSignature, Arc<GenCall>)>,
mode: CoreMode
) -> (Self, HashMap<StrRef, DefinitionId>, HashMap<StrRef, Type>) {
let mut primitives = Self::make_primitives();
let (mut definition_ast_list, builtin_name_list) = builtins::get_builtins(&mut primitives);
@ -108,6 +116,7 @@ impl TopLevelComposer {
keyword_list,
defined_names,
method_class,
mode
},
builtin_id,
builtin_ty,
@ -554,7 +563,7 @@ impl TopLevelComposer {
unifier,
primitives,
&mut type_var_to_concrete_def,
&self.keyword_list,
(&self.keyword_list, &self.mode)
)?
}
}
@ -827,8 +836,9 @@ impl TopLevelComposer {
unifier: &mut Unifier,
primitives: &PrimitiveStore,
type_var_to_concrete_def: &mut HashMap<Type, TypeAnnotation>,
keyword_list: &HashSet<StrRef>,
core_info: (&HashSet<StrRef>, &CoreMode),
) -> Result<(), String> {
let (keyword_list, core_mode) = core_info;
let mut class_def = class_def.write();
let (
class_id,
@ -1058,27 +1068,27 @@ impl TopLevelComposer {
let dummy_field_type = unifier.get_fresh_var().0;
// handle Kernel[T], KernelInvariant[T]
let (annotation, mutable) = match &annotation.node {
ast::ExprKind::Subscript { value, slice, .. }
if matches!(&value.node, ast::ExprKind::Name { id, .. } if id == &"Kernel".into()) =>
{
match &slice.node {
ast::ExprKind::Subscript { value, .. }
if matches!(&value.node, ast::ExprKind::Name { id, .. } if id == &"list".into()) =>
{
return Err(format!("list is not allowed to be `Kernel` at {}", value.location))
}
_ => (slice, true)
let (annotation, mutable) = {
match core_mode {
CoreMode::Artiq => match &annotation.as_ref().node {
ast::ExprKind::Subscript { value, slice, .. } if matches!(
&value.node,
ast::ExprKind::Name { id, .. } if id == &"Kernel".into()
) => (slice, true),
ast::ExprKind::Subscript { value, slice, .. } if matches!(
&value.node,
ast::ExprKind::Name { id, .. } if id == &"KernelInvariant".into()
) => (slice, false),
_ => continue // ignore fields annotated otherwise
},
CoreMode::Standalone => match &annotation.as_ref().node {
ast::ExprKind::Subscript { value, slice, .. } if matches!(
&value.node,
ast::ExprKind::Name { id, .. } if id == &"Invariant".into()
) => (slice, false),
_ => (annotation, true)
}
}
ast::ExprKind::Subscript { value, slice, .. }
if matches!(&value.node, ast::ExprKind::Name { id, .. } if id == &"KernelInvariant".into()) => {
(slice, false)
}
_ => {
eprintln!("attributes not annotated with `Kernel` or `KernelInvariants` at {}", &annotation.location);
(annotation, true)
}
};
class_fields_def.push((*attr, dummy_field_type, mutable));

View File

@ -1,5 +1,9 @@
@extern
def output_int(x: int32):
def output_int32(x: int32):
...
@extern
def output_int64(x: int64):
...
@ -27,10 +31,10 @@ class B:
def run() -> int32:
a = A(10)
output_int(a.a)
output_int32(a.a)
a = A(20)
output_int(a.a)
output_int(a.get_a())
output_int(a.get_b().b)
output_int32(a.a)
output_int32(a.get_a())
output_int32(a.get_b().b)
return 0

View File

@ -1,10 +1,14 @@
#include <stdio.h>
#include <string.h>
void output_int(int x) {
void output_int32(int x) {
printf("%d\n", x);
}
void output_int64(long x) {
printf("%ld\n", x);
}
void output_asciiart(int x) {
static char chars[] = " .,-:;i+hHM$*#@ ";
if(x < 0) {

View File

@ -3,13 +3,10 @@ use inkwell::{
targets::*,
OptimizationLevel,
};
use nac3core::typecheck::{type_inferencer::PrimitiveStore, typedef::{Type, Unifier}};
use nac3parser::{ast::{Expr, ExprKind, StmtKind}, parser};
use std::{borrow::Borrow, collections::HashMap, env, fs, path::Path, sync::Arc, time::SystemTime};
use parking_lot::RwLock;
use std::{borrow::Borrow, env};
use std::fs;
use std::{collections::HashMap, path::Path, sync::Arc, time::SystemTime};
use nac3parser::{ast::{Expr, ExprKind, StmtKind}, parser};
use nac3core::{
codegen::{
concrete_type::ConcreteTypeStore, CodeGenTask, DefaultCodeGenerator, WithCall,
@ -17,11 +14,11 @@ use nac3core::{
},
symbol_resolver::SymbolResolver,
toplevel::{
composer::TopLevelComposer,
composer::{TopLevelComposer, CoreMode},
TopLevelDef, helper::parse_parameter_default_value,
type_annotation::*,
},
typecheck::typedef::FunSignature,
typecheck::{type_inferencer::PrimitiveStore, typedef::{Type, Unifier, FunSignature}}
};
mod basic_symbol_resolver;
@ -47,7 +44,10 @@ fn main() {
};
let primitive: PrimitiveStore = TopLevelComposer::make_primitives().0;
let (mut composer, builtins_def, builtins_ty) = TopLevelComposer::new(vec![]);
let (mut composer, builtins_def, builtins_ty) = TopLevelComposer::new(
vec![],
CoreMode::Standalone
);
let internal_resolver: Arc<ResolverInternal> = ResolverInternal {
id_to_type: builtins_ty.into(),