[WIP] Move builtins to modules

nac3artiq/demo/demo.py

@@ -1,5 +1,6 @@
 from min_artiq import *
-
+import string_attribute_issue337
+import support_class_attr_issue102
 
 @nac3
 class Demo:
@@ -15,6 +16,8 @@ class Demo:
     @kernel
     def run(self):
         self.core.reset()
+        string_attribute_issue337.Demo2().run()
+        support_class_attr_issue102.Demo3().run()
         while True:
             with parallel:
                 self.led0.pulse(100.*ms)

nac3artiq/demo/nac3artiq.so

@@ -1 +0,0 @@
-../../target/release/libnac3artiq.so

nac3artiq/demo/sample

@@ -0,0 +1,229 @@
+Registration Completed: Demo3, DefinitionId(129)
+Registration Completed: NAC3Devices, DefinitionId(131)
+Registration Completed: rtio_init, DefinitionId(133)
+Registration Completed: rtio_get_counter, DefinitionId(134)
+Registration Completed: rtio_output, DefinitionId(135)
+Registration Completed: rtio_input_timestamp, DefinitionId(136)
+Registration Completed: rtio_input_data, DefinitionId(137)
+Registration Completed: print_int32, DefinitionId(138)
+Registration Completed: print_int64, DefinitionId(139)
+Registration Completed: Core, DefinitionId(140)
+Registration Completed: TTLOut, DefinitionId(146)
+Registration Completed: KernelContextManager, DefinitionId(153)
+Registration Completed: UnwrapNoneError, DefinitionId(156)
+Registration Completed: Demo, DefinitionId(157)
+Registration Completed: Demo2, DefinitionId(159)
+1
+2
+3
+4
+Run Analyze 2
+Searching for: self
+"self"
+Searching for: Demo3
+"self"
+Searching for: self
+"self"
+"Demo3"
+Searching for: Demo3
+"self"
+"Demo3"
+Searching for: obj
+"self"
+"obj"
+"Demo3"
+Searching for: NAC3Devices
+"self"
+"obj"
+"Demo3"
+TypeID: 140737352879872, TypeTypeID: 140737352879872, Module: 140737352893760
+TypeID: 140737352879872, TypeTypeID: 140737352879872, Module: 140737352893760
+TypeID: 140737352879872, TypeTypeID: 140737352879872, Module: 140737352893760
+Searching for: rtio_init
+"self"
+Searching for: at_mu
+"self"
+"rtio_init"
+Searching for: rtio_get_counter
+"self"
+"rtio_init"
+"at_mu"
+Searching for: rtio_get_counter
+"self"
+Searching for: now_mu
+"self"
+"rtio_get_counter"
+"min_now"
+Searching for: min_now
+"now_mu"
+"self"
+"rtio_get_counter"
+"min_now"
+Searching for: at_mu
+"now_mu"
+"self"
+"rtio_get_counter"
+"min_now"
+Searching for: min_now
+"at_mu"
+"now_mu"
+"self"
+"rtio_get_counter"
+"min_now"
+Searching for: seconds
+"self"
+"seconds"
+Searching for: self
+"self"
+"seconds"
+Searching for: mu
+"self"
+"mu"
+Searching for: self
+"self"
+"mu"
+Searching for: delay_mu
+"self"
+"dt"
+Searching for: self
+"self"
+"dt"
+"delay_mu"
+Searching for: dt
+"self"
+"dt"
+"delay_mu"
+Searching for: rtio_output
+"self"
+"o"
+Searching for: self
+"self"
+"rtio_output"
+"o"
+Searching for: o
+"self"
+"rtio_output"
+"o"
+Searching for: self
+"self"
+Searching for: self
+"self"
+Searching for: self
+"self"
+"duration"
+Searching for: delay_mu
+"self"
+"duration"
+Searching for: duration
+"delay_mu"
+"self"
+"duration"
+Searching for: self
+"delay_mu"
+"self"
+"duration"
+Searching for: self
+"self"
+"duration"
+Searching for: self
+"self"
+"duration"
+Searching for: duration
+"self"
+"duration"
+Searching for: self
+"self"
+"duration"
+Searching for: self
+"self"
+Searching for: string_attribute_issue337
+"self"
+nac3artiq => 140737337865888
+extern => 140735144254240
+nac3 => 140735144254880
+__cached__ => 140737352893264
+us => 140737337478960
+__file__ => 140737337508448
+__spec__ => 140737352893264
+__package__ => 140737352893264
+__builtins__ => 140737339048560
+none => 140735143778288
+ns => 140737337478608
+__loader__ => 140737339453808
+virtual => 10355312
+print_int32 => 140735144255840
+Demo => 10404320
+__doc__ => 140737352893264
+kernel => 140735144254560
+support_class_attr_issue102 => 140735144205168
+string_attribute_issue337 => 140737337854688
+__annotations__ => 140737337365760
+KernelInvariant => 10353536
+floor64 => 140735144253600
+device_db => 140735144204288
+ConstGeneric => 140735146392640
+__name__ => 140737337365616
+rpc => 140735144254400
+TTLOut => 10519408
+Core => 10532336
+sequential => 140735144264768
+ms => 140737337478992
+parallel => 140735144265152
+portable => 140735144254720
+ceil64 => 140735144253760
+print_int64 => 140735144256000
+round64 => 140735144253440
+Kernel => 10350368
+Some => 140737337529248
+Option => 10357088
+UnwrapNoneError => 10408640
+Found A module!!!! 140737352879872
+nac3artiq => 140737337865888
+extern => 140735144254240
+nac3 => 140735144254880
+__cached__ => 140737352893264
+us => 140737337478960
+__file__ => 140737337508448
+__spec__ => 140737352893264
+__package__ => 140737352893264
+__builtins__ => 140737339048560
+none => 140735143778288
+ns => 140737337478608
+__loader__ => 140737339453808
+virtual => 10355312
+print_int32 => 140735144255840
+Demo => 10404320
+__doc__ => 140737352893264
+kernel => 140735144254560
+support_class_attr_issue102 => 140735144205168
+string_attribute_issue337 => 140737337854688
+__annotations__ => 140737337365760
+KernelInvariant => 10353536
+floor64 => 140735144253600
+device_db => 140735144204288
+ConstGeneric => 140735146392640
+__name__ => 140737337365616
+rpc => 140735144254400
+TTLOut => 10519408
+Core => 10532336
+sequential => 140735144264768
+ms => 140737337478992
+parallel => 140735144265152
+portable => 140735144254720
+ceil64 => 140735144253760
+print_int64 => 140735144256000
+round64 => 140735144253440
+Kernel => 10350368
+Some => 140737337529248
+Option => 10357088
+UnwrapNoneError => 10408640
+Found A module!!!! 140737352879872
+Searching for: print_int32
+"self"
+Searching for: self
+"print_int32"
+"self"
+Searching for: self
+"print_int32"
+"self"
+Searching for: base

nac3artiq/demo/string_attribute_issue337.py

@@ -3,7 +3,7 @@ from numpy import int32
 
 
 @nac3
-class Demo:
+class Demo2:
     core: KernelInvariant[Core]
     attr1: KernelInvariant[str]
     attr2: KernelInvariant[int32]
@@ -21,4 +21,4 @@ class Demo:
 
 
 if __name__ == "__main__":
-    Demo().run()
+    Demo2().run()

nac3artiq/demo/support_class_attr_issue102.py

@@ -3,7 +3,7 @@ from numpy import int32
 
 
 @nac3
-class Demo:
+class Demo3:
     attr1: KernelInvariant[int32] = 2
     attr2: int32 = 4
     attr3: Kernel[int32]
@@ -23,14 +23,14 @@ class NAC3Devices:
 
     @kernel
     def run(self):
-        Demo.attr1  # Supported
+        Demo3.attr1  # Supported
         # Demo.attr2  # Field not accessible on Kernel
         # Demo.attr3  # Only attributes can be accessed in this way
         # Demo.attr1 = 2  # Attributes are immutable
 
         self.attr4  # Attributes can be accessed within class
 
-        obj = Demo()
+        obj = Demo3()
         obj.attr1  # Attributes can be accessed by class objects
 
         NAC3Devices.attr4  # Attributes accessible for classes without __init__

nac3artiq/src/lib.rs

@@ -43,7 +43,7 @@ use nac3core::{
         OptimizationLevel,
     },
     nac3parser::{
-        ast::{Constant, ExprKind, Located, Stmt, StmtKind, StrRef},
+        ast::{Constant, ExprKind, Located, Location, Stmt, StmtKind, StrRef},
         parser::parse_program,
     },
     symbol_resolver::SymbolResolver,
@@ -111,6 +111,7 @@ pub struct PrimitivePythonId {
     generic_alias: (u64, u64),
     virtual_id: u64,
     option: u64,
+    module: u64,
 }
 
 type TopLevelComponent = (Stmt, String, PyObject);
@@ -132,6 +133,7 @@ struct Nac3 {
     deferred_eval_store: DeferredEvaluationStore,
     /// LLVM-related options for code generation.
     llvm_options: CodeGenLLVMOptions,
+    modules: HashMap<StrRef, u64>,
 }
 
 create_exception!(nac3artiq, CompileError, exceptions::PyException);
@@ -168,11 +170,27 @@ impl Nac3 {
                 Ok((module.getattr("__name__")?.extract()?, source_file.to_string(), source))
             })?;
 
-        let parser_result = parse_program(&source, source_file.into())
+        let parser_result = parse_program(&source, source_file.clone().into())
             .map_err(|e| exceptions::PySyntaxError::new_err(format!("parse error: {e}")))?;
 
+        let mut module_content: Vec<Stmt> = Vec::default();
         for mut stmt in parser_result {
             let include = match stmt.node {
+                StmtKind::Import { ref names, .. } => {
+                    for name in names {
+                        let import_name = &name.name;
+                        let alias = name.asname.clone().unwrap_or_else(|| import_name.clone());
+                        let module_id = Python::with_gil(|py| -> PyResult<u64> {
+                            let imported_module =
+                                PyModule::import(py, import_name.to_string().as_str())?;
+                            let id_fn = PyModule::import(py, "builtins")?.getattr("id")?;
+                            Ok(id_fn.call1((imported_module,)).unwrap().extract().unwrap())
+                        })
+                        .unwrap();
+                        self.modules.insert(alias.clone(), module_id);
+                    }
+                    false
+                }
                 StmtKind::ClassDef { ref decorator_list, ref mut body, ref mut bases, .. } => {
                     let nac3_class = decorator_list.iter().any(|decorator| {
                         if let ExprKind::Name { id, .. } = decorator.node {
@@ -232,9 +250,28 @@ impl Nac3 {
             };
 
             if include {
-                self.top_levels.push((stmt, module_name.clone(), module.clone()));
+                module_content.push(stmt.clone());
+                // self.top_levels.push((stmt, module_name.clone(), module.clone()));
             }
         }
+        let mut module_body: Vec<Stmt> = Vec::default();
+        for stmt in module_content.iter() {
+            match &stmt.node {
+                StmtKind::FunctionDef { .. } => module_body.push(stmt.clone()),
+                _ => {}
+            }
+        }
+        // let module_def = StmtKind::ModuleDef {
+        //     name: module_name.clone().into(),
+        //     body: module_body
+        // };
+        // let module_stmt = Located::new(Location::new(1, 1, source_file.into()), module_def);
+        // self.top_levels.push((module_stmt, module_name, module.clone()));
+
+        for stmt in module_content {
+            self.top_levels.push((stmt, module_name.clone().into(), module.clone()));
+        }
+
         Ok(())
     }
 
@@ -444,6 +481,7 @@ impl Nac3 {
             let (name_to_pyid, resolver) =
                 module_to_resolver_cache.get(&module_id).cloned().unwrap_or_else(|| {
                     let mut name_to_pyid: HashMap<StrRef, u64> = HashMap::new();
+                    name_to_pyid.extend(self.modules.clone());
                     let members: &PyDict =
                         py_module.getattr("__dict__").unwrap().downcast().unwrap();
                     for (key, val) in members {
@@ -480,6 +518,7 @@ impl Nac3 {
                 .map_err(|e| {
                     CompileError::new_err(format!("compilation failed\n----------\n{e}"))
                 })?;
+            println!("Registration Completed: {}, {:?}", name, def_id);
             if let Some(class_obj) = class_obj {
                 self.exception_ids
                     .write()
@@ -707,7 +746,7 @@ impl Nac3 {
         let size_t = context
             .ptr_sized_int_type(&self.get_llvm_target_machine().get_target_data(), None)
             .get_bit_width();
-        let num_threads = if is_multithreaded() { 4 } else { 1 };
+        let num_threads = if is_multithreaded() { 1 } else { 1 };
         let thread_names: Vec<String> = (0..num_threads).map(|_| "main".to_string()).collect();
         let threads: Vec<_> = thread_names
             .iter()
@@ -1044,6 +1083,34 @@ impl Nac3 {
             ),
         ];
 
+        // let mut def_idx: usize = 1;
+        // let mut builtin_methods = Vec::new();
+        // let mut top_levels: Vec<TopLevelComponent> = Vec::new();
+        // for (name, signature, callback) in builtins {
+        //     let function_def = TopLevelDef::Function {
+        //         name: format!("__builtins__.{name}").into(),
+        //         simple_name: name,
+        //         signature: signature.ret,
+        //         var_id: Vec::new(),
+        //         instance_to_symbol: HashMap::new(),
+        //         instance_to_stmt: HashMap::new(),
+        //         resolver: None,
+        //         codegen_callback: Some(callback),
+        //         loc: None
+        //     };
+        //     top_levels.push((
+        //         Arc::new(RwLock::new(function_def)),
+        //         "__builtins__",
+        //         None
+        //     ));
+        //     builtin_methods.push((
+        //         name,
+        //         signature.ret,
+        //         DefinitionId(def_idx)
+        //     ));
+        //     def_idx += 1;
+        // }
+
         let builtins_mod = PyModule::import(py, "builtins").unwrap();
         let id_fn = builtins_mod.getattr("id").unwrap();
         let numpy_mod = PyModule::import(py, "numpy").unwrap();
@@ -1081,6 +1148,7 @@ impl Nac3 {
             tuple: get_attr_id(builtins_mod, "tuple"),
             exception: get_attr_id(builtins_mod, "Exception"),
             option: get_id(artiq_builtins.get_item("Option").ok().flatten().unwrap()),
+            module: get_attr_id(types_mod, "ModuleType"),
         };
 
         let working_directory = tempfile::Builder::new().prefix("nac3-").tempdir().unwrap();
@@ -1144,6 +1212,7 @@ impl Nac3 {
                 opt_level: OptimizationLevel::Default,
                 target: Nac3::get_llvm_target_options(isa),
             },
+            modules: HashMap::default(),
         })
     }
 

nac3artiq/src/symbol_resolver.rs

@@ -626,6 +626,37 @@ impl InnerResolver {
                 _ => unimplemented!(),
             }
         } else if ty_id == self.primitive_ids.virtual_id {
+            Ok(Ok((
+                {
+                    let ty = TypeEnum::TVirtual { ty: unifier.get_dummy_var().ty };
+                    unifier.add_ty(ty)
+                },
+                false,
+            )))
+        } else if ty_id == self.primitive_ids.module {
+            /*
+            Current TODOS
+            ty_ty_id does not module id stored in lib.rs
+            Use id to definition function here to map to corresponding module object
+            Once the module object is matched, return a TObj type representing module functions
+            Alternatively, create a new TModule Type for better errors and separate handling
+            The type definition for module will contain constructors for classes and function definition for functions
+            Initially work with just the functions and ignore class definitions
+            Add class attributes later to allow global access            
+            
+            Separate PR for this part
+            The builtin functions will be moved to __main__ module which can be accessed by all functions
+            The definitions of builtins will be added to the class and removed from main
+            Move Exceptions to its own module exns and use it instead of matching ID
+
+            Separate PR here
+            Special module registration #78
+             */
+            for entry in self.name_to_pyid.iter() {
+                println!("{} => {}", entry.0, entry.1);
+            }
+            println!("Found A module!!!! {}", ty_ty_id);
+
             Ok(Ok((
                 {
                     let ty = TypeEnum::TVirtual { ty: unifier.get_dummy_var().ty };
@@ -634,6 +665,10 @@ impl InnerResolver {
                 false,
             )))
         } else {
+            println!(
+                "TypeID: {:?}, TypeTypeID: {}, Module: {}",
+                ty_id, ty_ty_id, self.primitive_ids.module
+            );
             let str_fn =
                 pyo3::types::PyModule::import(py, "builtins").unwrap().getattr("repr").unwrap();
             let str_repr: String = str_fn.call1((pyty,)).unwrap().extract().unwrap();

nac3ast/src/ast_gen.rs

@@ -115,6 +115,10 @@ pub enum StmtKind<U = ()> {
         type_comment: Option<String>,
         config_comment: Vec<Ident>,
     },
+    // ModuleDef {
+    //     name: Ident,
+    //     body: Vec<Stmt<U>>,
+    // },
     ClassDef {
         name: Ident,
         bases: Vec<Expr<U>>,
@@ -626,6 +630,12 @@ pub mod fold {
                 type_comment: Foldable::fold(type_comment, folder)?,
                 config_comment: Foldable::fold(config_comment, folder)?,
             }),
+            // StmtKind::ModuleDef { name, body } => {
+            //     Ok(StmtKind::ModuleDef {
+            //         name: Foldable::fold(name, folder)?,
+            //         body: Foldable::fold(body, folder)?,
+            //     })
+            // }
             StmtKind::ClassDef { name, bases, keywords, body, decorator_list, config_comment } => {
                 Ok(StmtKind::ClassDef {
                     name: Foldable::fold(name, folder)?,

nac3core/src/toplevel/composer.rs

@@ -223,7 +223,16 @@ impl TopLevelComposer {
         );
 
         let defined_names = &mut self.defined_names;
+        if !mod_path.is_empty() {
+            defined_names.insert(mod_path.to_string());
+        }
         match &ast.node {
+            // ast::StmtKind::ModuleDef { name: module_name, body } => {
+            //     // Add any exclusions to module naming here
+            //     // Module classes and functions are handled separately
+            //     // Register module name and fields here if any
+            //     Ok(())
+            // }
             ast::StmtKind::ClassDef { name: class_name, bases, body, .. } => {
                 if self.keyword_list.contains(class_name) {
                     return Err(format!(
@@ -468,12 +477,17 @@ impl TopLevelComposer {
 
     /// Analyze the AST and modify the corresponding `TopLevelDef`
     pub fn start_analysis(&mut self, inference: bool) -> Result<(), HashSet<String>> {
+        println!("1");
         self.analyze_top_level_class_definition()?;
+        println!("2");
         self.analyze_top_level_class_fields_methods()?;
+        println!("3");
         self.analyze_top_level_function()?;
+        println!("4");
         if inference {
             self.analyze_function_instance()?;
         }
+        println!("5");
         self.analyze_top_level_variables()?;
         Ok(())
     }
@@ -1650,7 +1664,7 @@ impl TopLevelComposer {
 
                 (name.clone(), *simple_name, *signature, resolver.clone())
             };
-
+            // println!("Name: {name}, Simple Name: {simple_name}");
             let signature_ty_enum = unifier.get_ty(signature);
             let TypeEnum::TFunc(FunSignature { args, ret, vars, .. }) = signature_ty_enum.as_ref()
             else {
@@ -1675,6 +1689,7 @@ impl TopLevelComposer {
                         &ty_ann,
                         &mut None,
                     )?;
+                    // println!("Self Type annotation: {:?}", self_ty);
                     vars.extend(type_vars.iter().map(|ty| {
                         let TypeEnum::TVar { id, .. } = &*unifier.get_ty(*ty) else {
                             unreachable!()
@@ -1929,6 +1944,7 @@ impl TopLevelComposer {
             Ok(())
         };
 
+        println!("Run Analyze 2");
         for (id, (def, ast)) in self.definition_ast_list.iter().enumerate().skip(self.builtin_num) {
             if ast.is_none() {
                 continue;
@@ -1940,6 +1956,7 @@ impl TopLevelComposer {
         if !errors.is_empty() {
             return Err(errors);
         }
+        println!("Clear");
         Ok(())
     }
 

nac3core/src/toplevel/helper.rs

@@ -359,8 +359,8 @@ pub fn make_exception_fields(int32: Type, int64: Type, str: Type) -> Vec<(StrRef
 impl TopLevelDef {
     pub fn to_string(&self, unifier: &mut Unifier) -> String {
         match self {
-            TopLevelDef::Module { name, fields, methods, .. } => {
+            TopLevelDef::Module { name, attributes, methods, .. } => {
-                let fields_str = fields
+                let fields_str = attributes
                     .iter()
                     .map(|(n, ty, _)| (n.to_string(), unifier.stringify(*ty)))
                     .collect_vec();
@@ -370,7 +370,7 @@ impl TopLevelDef {
                     .map(|(n, ty, id)| (n.to_string(), unifier.stringify(*ty), *id))
                     .collect_vec();
                 format!(
-                    "Module {{\nname: {:?},\nfields: {:?},\nmethods: {:?}\n}}",
+                    "Module {{\nname: {:?},\nattributes: {:?},\nmethods: {:?}\n}}",
                     name,
                     fields_str.iter().map(|(a, _)| a).collect_vec(),
                     methods_str.iter().map(|(a, b, _)| (a, b)).collect_vec(),

nac3core/src/toplevel/mod.rs

@@ -95,10 +95,6 @@ pub enum TopLevelDef {
     Module {
         /// Module name
         name: StrRef,
-        /// Module fields.
-        ///
-        /// Name and type is mutable.
-        fields: Vec<(StrRef, Type, bool)>,
         /// Module Attributes.
         ///
         /// Name, type, value.

nac3core/src/typecheck/type_inferencer/mod.rs

@@ -585,6 +585,10 @@ impl<'a> Fold<()> for Inferencer<'a> {
                         unreachable!("must be tobj")
                     }
                 } else {
+                    println!("Searching for: {id}");
+                    for ident in self.defined_identifiers.iter() {
+                        println!("{:?}", ident.0);
+                    }
                     if !self.defined_identifiers.contains_key(id) {
                         match self.function_data.resolver.get_symbol_type(
                             self.unifier,

nac3core/src/typecheck/typedef/mod.rs

@@ -270,6 +270,11 @@ pub enum TypeEnum {
 
     /// A function type.
     TFunc(FunSignature),
+
+    TModule {
+        module_id: DefinitionId,
+        attributes: Mapping<StrRef, (Type, bool)>,
+    },
 }
 
 impl TypeEnum {
@@ -284,6 +289,7 @@ impl TypeEnum {
             TypeEnum::TVirtual { .. } => "TVirtual",
             TypeEnum::TCall { .. } => "TCall",
             TypeEnum::TFunc { .. } => "TFunc",
+            TypeEnum::TModule { .. } => "TModule",
         }
     }
 }
@@ -601,7 +607,8 @@ impl Unifier {
             TTuple { ty, .. } => ty.iter().all(|ty| self.is_concrete(*ty, allowed_typevars)),
             TObj { params: vars, .. } => {
                 vars.values().all(|ty| self.is_concrete(*ty, allowed_typevars))
-            }
+            },
+            TModule { .. } => true,
         }
     }
 
@@ -1417,6 +1424,9 @@ impl Unifier {
                     format!("{}[{}]", name, params.join(", "))
                 }
             }
+            TypeEnum::TModule { module_id, .. } => {
+                obj_to_name(module_id.0) // Separate object and module
+            }
             TypeEnum::TCall { .. } => "call".to_owned(),
             TypeEnum::TFunc(signature) => {
                 let params = signature
@@ -1592,7 +1602,7 @@ impl Unifier {
                     None
                 }
             }
-            TypeEnum::TCall(_) => {
+            TypeEnum::TCall(_) | TypeEnum::TModule { .. } => {
                 unreachable!("{} not expected", ty.get_type_name())
             }
         }

pyo3/nac3artiq.so

@@ -0,0 +1 @@
+../target/debug/libnac3artiq.so