From bf7e2c295af0c1f0774a3636a6e1545b064eae50 Mon Sep 17 00:00:00 2001 From: Sebastien Bourdeauducq Date: Wed, 3 Nov 2021 17:11:00 +0800 Subject: [PATCH] integrate nac3parser --- Cargo.lock | 65 +++++++++---------- Cargo.toml | 2 + flake.nix | 2 +- nac3artiq/Cargo.toml | 2 +- nac3artiq/src/codegen.rs | 2 +- nac3artiq/src/lib.rs | 2 +- nac3artiq/src/symbol_resolver.rs | 2 +- nac3ast/Cargo.toml | 4 +- nac3core/Cargo.toml | 2 +- nac3core/src/codegen/concrete_type.rs | 2 +- nac3core/src/codegen/expr.rs | 2 +- nac3core/src/codegen/generator.rs | 2 +- nac3core/src/codegen/mod.rs | 2 +- nac3core/src/codegen/stmt.rs | 2 +- nac3core/src/codegen/test.rs | 2 +- nac3core/src/location.rs | 2 +- nac3core/src/symbol_resolver.rs | 4 +- nac3core/src/toplevel/composer.rs | 2 +- nac3core/src/toplevel/mod.rs | 2 +- nac3core/src/toplevel/test.rs | 2 +- nac3core/src/typecheck/function_check.rs | 2 +- nac3core/src/typecheck/magic_methods.rs | 4 +- nac3core/src/typecheck/type_inferencer/mod.rs | 2 +- .../src/typecheck/type_inferencer/test.rs | 2 +- nac3core/src/typecheck/typedef/mod.rs | 2 +- nac3parser/Cargo.toml | 7 +- nac3parser/README.md | 18 ++--- nac3parser/src/lib.rs | 7 +- nac3parser/src/parser.rs | 2 +- nac3standalone/Cargo.toml | 2 +- nac3standalone/src/basic_symbol_resolver.rs | 2 +- nac3standalone/src/main.rs | 2 +- 32 files changed, 73 insertions(+), 86 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 3e6406d06..0490281c5 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -489,9 +489,20 @@ version = "0.1.0" dependencies = [ "inkwell", "nac3core", + "nac3parser", "parking_lot", "pyo3", - "rustpython-parser", +] + +[[package]] +name = "nac3ast" +version = "0.1.0" +dependencies = [ + "fxhash", + "lazy_static", + "num-bigint 0.4.2", + "parking_lot", + "string-interner", ] [[package]] @@ -503,22 +514,40 @@ dependencies = [ "inkwell", "insta", "itertools", + "nac3parser", "num-bigint 0.3.3", "num-traits", "parking_lot", "rayon", - "rustpython-parser", "test-case", ] +[[package]] +name = "nac3parser" +version = "0.1.2" +dependencies = [ + "ahash", + "insta", + "lalrpop", + "lalrpop-util", + "log", + "nac3ast", + "num-bigint 0.4.2", + "num-traits", + "phf", + "unic-emoji-char", + "unic-ucd-ident", + "unicode_names2", +] + [[package]] name = "nac3standalone" version = "0.1.0" dependencies = [ "inkwell", "nac3core", + "nac3parser", "parking_lot", - "rustpython-parser", ] [[package]] @@ -898,36 +927,6 @@ dependencies = [ "libloading", ] -[[package]] -name = "rustpython-ast" -version = "0.1.0" -source = "git+https://github.com/m-labs/RustPython?branch=parser-mod#efdf7829ba1a5f87d30df8eaff12a330544f3cbd" -dependencies = [ - "fxhash", - "lazy_static", - "num-bigint 0.4.2", - "parking_lot", - "string-interner", -] - -[[package]] -name = "rustpython-parser" -version = "0.1.2" -source = "git+https://github.com/m-labs/RustPython?branch=parser-mod#efdf7829ba1a5f87d30df8eaff12a330544f3cbd" -dependencies = [ - "ahash", - "lalrpop", - "lalrpop-util", - "log", - "num-bigint 0.4.2", - "num-traits", - "phf", - "rustpython-ast", - "unic-emoji-char", - "unic-ucd-ident", - "unicode_names2", -] - [[package]] name = "rustversion" version = "1.0.5" diff --git a/Cargo.toml b/Cargo.toml index b301a6a38..caaef0515 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,5 +1,7 @@ [workspace] members = [ + "nac3ast", + "nac3parser", "nac3core", "nac3standalone", "nac3artiq", diff --git a/flake.nix b/flake.nix index 9398c1a96..cb5f0b576 100644 --- a/flake.nix +++ b/flake.nix @@ -12,7 +12,7 @@ pkgs.rustPlatform.buildRustPackage { name = "nac3artiq"; src = self; - cargoSha256 = "sha256-si/yuVrRnXXLp8fMo5ZORHXa5JUqI2YqQ9U4R3UEUL0="; + cargoSha256 = "sha256-o/3QRbe1WEOA7NPyO1bXCLxUxUWBex8bPcApl/aT040="; nativeBuildInputs = [ pkgs.python3 pkgs.llvm_12 ]; buildInputs = [ pkgs.python3 pkgs.libffi pkgs.libxml2 pkgs.llvm_12 ]; cargoBuildFlags = [ "--package" "nac3artiq" ]; diff --git a/nac3artiq/Cargo.toml b/nac3artiq/Cargo.toml index e73bbd316..5f6f8fd04 100644 --- a/nac3artiq/Cargo.toml +++ b/nac3artiq/Cargo.toml @@ -10,7 +10,7 @@ crate-type = ["cdylib"] [dependencies] pyo3 = { version = "0.14", features = ["extension-module"] } -rustpython-parser = { git = "https://github.com/m-labs/RustPython", branch = "parser-mod" } inkwell = { git = "https://github.com/TheDan64/inkwell", branch = "master", features = ["llvm12-0"] } parking_lot = "0.11" +nac3parser = { path = "../nac3parser" } nac3core = { path = "../nac3core" } diff --git a/nac3artiq/src/codegen.rs b/nac3artiq/src/codegen.rs index cb79c5499..a462cf931 100644 --- a/nac3artiq/src/codegen.rs +++ b/nac3artiq/src/codegen.rs @@ -4,7 +4,7 @@ use nac3core::{ typecheck::typedef::{FunSignature, Type}, }; -use rustpython_parser::ast::{Expr, ExprKind, Located, Stmt, StmtKind, StrRef}; +use nac3parser::ast::{Expr, ExprKind, Located, Stmt, StmtKind, StrRef}; use inkwell::values::BasicValueEnum; diff --git a/nac3artiq/src/lib.rs b/nac3artiq/src/lib.rs index fe72cd76b..45a01648b 100644 --- a/nac3artiq/src/lib.rs +++ b/nac3artiq/src/lib.rs @@ -11,7 +11,7 @@ use inkwell::{ }; use pyo3::prelude::*; use pyo3::{exceptions, types::PyList}; -use rustpython_parser::{ +use nac3parser::{ ast::{self, StrRef}, parser::{self, parse_program}, }; diff --git a/nac3artiq/src/symbol_resolver.rs b/nac3artiq/src/symbol_resolver.rs index 05dd6936e..6c255800a 100644 --- a/nac3artiq/src/symbol_resolver.rs +++ b/nac3artiq/src/symbol_resolver.rs @@ -14,7 +14,7 @@ use pyo3::{ types::{PyList, PyModule, PyTuple}, PyAny, PyObject, PyResult, Python, }; -use rustpython_parser::ast::StrRef; +use nac3parser::ast::StrRef; use std::{ cell::RefCell, collections::{HashMap, HashSet}, diff --git a/nac3ast/Cargo.toml b/nac3ast/Cargo.toml index ab49bf021..5e0da134f 100644 --- a/nac3ast/Cargo.toml +++ b/nac3ast/Cargo.toml @@ -1,7 +1,7 @@ [package] -name = "rustpython-ast" +name = "nac3ast" version = "0.1.0" -authors = ["RustPython Team"] +authors = ["RustPython Team", "M-Labs"] edition = "2018" [features] diff --git a/nac3core/Cargo.toml b/nac3core/Cargo.toml index 6f2f730f9..24d26b915 100644 --- a/nac3core/Cargo.toml +++ b/nac3core/Cargo.toml @@ -8,11 +8,11 @@ edition = "2018" num-bigint = "0.3" num-traits = "0.2" inkwell = { git = "https://github.com/TheDan64/inkwell", branch = "master", features = ["llvm12-0"] } -rustpython-parser = { git = "https://github.com/m-labs/RustPython", branch = "parser-mod" } itertools = "0.10.1" crossbeam = "0.8.1" parking_lot = "0.11.1" rayon = "1.5.1" +nac3parser = { path = "../nac3parser" } [dev-dependencies] test-case = "1.2.0" diff --git a/nac3core/src/codegen/concrete_type.rs b/nac3core/src/codegen/concrete_type.rs index 347956aff..3d734a8f8 100644 --- a/nac3core/src/codegen/concrete_type.rs +++ b/nac3core/src/codegen/concrete_type.rs @@ -7,7 +7,7 @@ use crate::{ }, }; -use rustpython_parser::ast::StrRef; +use nac3parser::ast::StrRef; use std::collections::HashMap; pub struct ConcreteTypeStore { diff --git a/nac3core/src/codegen/expr.rs b/nac3core/src/codegen/expr.rs index 7d3656ccf..5b31a15dc 100644 --- a/nac3core/src/codegen/expr.rs +++ b/nac3core/src/codegen/expr.rs @@ -15,7 +15,7 @@ use inkwell::{ AddressSpace, }; use itertools::{chain, izip, zip, Itertools}; -use rustpython_parser::ast::{ +use nac3parser::ast::{ self, Boolop, Comprehension, Constant, Expr, ExprKind, Operator, StrRef, }; diff --git a/nac3core/src/codegen/generator.rs b/nac3core/src/codegen/generator.rs index 22ee541f8..0f5bd3a01 100644 --- a/nac3core/src/codegen/generator.rs +++ b/nac3core/src/codegen/generator.rs @@ -4,7 +4,7 @@ use crate::{ typecheck::typedef::{FunSignature, Type}, }; use inkwell::values::{BasicValueEnum, PointerValue}; -use rustpython_parser::ast::{Expr, Stmt, StrRef}; +use nac3parser::ast::{Expr, Stmt, StrRef}; pub trait CodeGenerator { /// Return the module name for the code generator. diff --git a/nac3core/src/codegen/mod.rs b/nac3core/src/codegen/mod.rs index 8a982337e..1ec0cf4e3 100644 --- a/nac3core/src/codegen/mod.rs +++ b/nac3core/src/codegen/mod.rs @@ -19,7 +19,7 @@ use inkwell::{ }; use itertools::Itertools; use parking_lot::{Condvar, Mutex}; -use rustpython_parser::ast::{Stmt, StrRef}; +use nac3parser::ast::{Stmt, StrRef}; use std::collections::HashMap; use std::sync::{ atomic::{AtomicBool, Ordering}, diff --git a/nac3core/src/codegen/stmt.rs b/nac3core/src/codegen/stmt.rs index 41779259c..0ad9cdd1a 100644 --- a/nac3core/src/codegen/stmt.rs +++ b/nac3core/src/codegen/stmt.rs @@ -1,7 +1,7 @@ use super::{expr::destructure_range, CodeGenContext, CodeGenerator}; use crate::typecheck::typedef::Type; use inkwell::values::{BasicValue, BasicValueEnum, PointerValue}; -use rustpython_parser::ast::{Expr, ExprKind, Stmt, StmtKind}; +use nac3parser::ast::{Expr, ExprKind, Stmt, StmtKind}; pub fn gen_var<'ctx, 'a>(ctx: &mut CodeGenContext<'ctx, 'a>, ty: Type) -> PointerValue<'ctx> { // put the alloca in init block diff --git a/nac3core/src/codegen/test.rs b/nac3core/src/codegen/test.rs index 87cd79c22..ce958ff32 100644 --- a/nac3core/src/codegen/test.rs +++ b/nac3core/src/codegen/test.rs @@ -16,7 +16,7 @@ use crate::{ use indoc::indoc; use inkwell::values::BasicValueEnum; use parking_lot::RwLock; -use rustpython_parser::{ +use nac3parser::{ ast::{fold::Fold, StrRef}, parser::parse_program, }; diff --git a/nac3core/src/location.rs b/nac3core/src/location.rs index 6e9021ca4..f0968523f 100644 --- a/nac3core/src/location.rs +++ b/nac3core/src/location.rs @@ -1,4 +1,4 @@ -use rustpython_parser::ast; +use nac3parser::ast; use std::vec::Vec; #[derive(Clone, Copy, PartialEq)] diff --git a/nac3core/src/symbol_resolver.rs b/nac3core/src/symbol_resolver.rs index ac0bbaa77..5395a4dd8 100644 --- a/nac3core/src/symbol_resolver.rs +++ b/nac3core/src/symbol_resolver.rs @@ -10,7 +10,7 @@ use crate::typecheck::{ use crate::{location::Location, typecheck::typedef::TypeEnum}; use itertools::{chain, izip}; use parking_lot::RwLock; -use rustpython_parser::ast::{Expr, StrRef}; +use nac3parser::ast::{Expr, StrRef}; use inkwell::values::BasicValueEnum; #[derive(Clone, PartialEq, Debug)] @@ -61,7 +61,7 @@ pub fn parse_type_annotation( primitives: &PrimitiveStore, expr: &Expr, ) -> Result { - use rustpython_parser::ast::ExprKind::*; + use nac3parser::ast::ExprKind::*; let ids = IDENTIFIER_ID.with(|ids| { *ids }); diff --git a/nac3core/src/toplevel/composer.rs b/nac3core/src/toplevel/composer.rs index b34d776aa..604262ddd 100644 --- a/nac3core/src/toplevel/composer.rs +++ b/nac3core/src/toplevel/composer.rs @@ -1,6 +1,6 @@ use std::cell::RefCell; -use rustpython_parser::ast::fold::Fold; +use nac3parser::ast::fold::Fold; use crate::{ symbol_resolver::SymbolValue, diff --git a/nac3core/src/toplevel/mod.rs b/nac3core/src/toplevel/mod.rs index 9d7c4d719..f80571120 100644 --- a/nac3core/src/toplevel/mod.rs +++ b/nac3core/src/toplevel/mod.rs @@ -16,7 +16,7 @@ use crate::{ }; use itertools::{izip, Itertools}; use parking_lot::RwLock; -use rustpython_parser::ast::{self, Stmt, StrRef}; +use nac3parser::ast::{self, Stmt, StrRef}; use inkwell::values::BasicValueEnum; #[derive(PartialEq, Eq, PartialOrd, Ord, Clone, Copy, Hash, Debug)] diff --git a/nac3core/src/toplevel/test.rs b/nac3core/src/toplevel/test.rs index 343baded0..25a39c42b 100644 --- a/nac3core/src/toplevel/test.rs +++ b/nac3core/src/toplevel/test.rs @@ -10,7 +10,7 @@ use crate::{ }; use indoc::indoc; use parking_lot::Mutex; -use rustpython_parser::{ast::fold::Fold, parser::parse_program}; +use nac3parser::{ast::fold::Fold, parser::parse_program}; use std::{collections::HashMap, sync::Arc}; use test_case::test_case; diff --git a/nac3core/src/typecheck/function_check.rs b/nac3core/src/typecheck/function_check.rs index 096ce244a..5b6812973 100644 --- a/nac3core/src/typecheck/function_check.rs +++ b/nac3core/src/typecheck/function_check.rs @@ -2,7 +2,7 @@ use crate::typecheck::typedef::TypeEnum; use super::type_inferencer::Inferencer; use super::typedef::Type; -use rustpython_parser::ast::{self, Expr, ExprKind, Stmt, StmtKind, StrRef}; +use nac3parser::ast::{self, Expr, ExprKind, Stmt, StmtKind, StrRef}; use std::{collections::HashSet, iter::once}; impl<'a> Inferencer<'a> { diff --git a/nac3core/src/typecheck/magic_methods.rs b/nac3core/src/typecheck/magic_methods.rs index 30e3c7531..0dcc0ea97 100644 --- a/nac3core/src/typecheck/magic_methods.rs +++ b/nac3core/src/typecheck/magic_methods.rs @@ -2,8 +2,8 @@ use crate::typecheck::{ type_inferencer::*, typedef::{FunSignature, FuncArg, Type, TypeEnum, Unifier}, }; -use rustpython_parser::ast; -use rustpython_parser::ast::{Cmpop, Operator, Unaryop}; +use nac3parser::ast; +use nac3parser::ast::{Cmpop, Operator, Unaryop}; use std::borrow::Borrow; use std::collections::HashMap; diff --git a/nac3core/src/typecheck/type_inferencer/mod.rs b/nac3core/src/typecheck/type_inferencer/mod.rs index 371909346..1a8fa6385 100644 --- a/nac3core/src/typecheck/type_inferencer/mod.rs +++ b/nac3core/src/typecheck/type_inferencer/mod.rs @@ -7,7 +7,7 @@ use super::typedef::{Call, FunSignature, FuncArg, Type, TypeEnum, Unifier}; use super::{magic_methods::*, typedef::CallId}; use crate::{symbol_resolver::SymbolResolver, toplevel::TopLevelContext}; use itertools::izip; -use rustpython_parser::ast::{ +use nac3parser::ast::{ self, fold::{self, Fold}, Arguments, Comprehension, ExprKind, Located, Location, StrRef, diff --git a/nac3core/src/typecheck/type_inferencer/test.rs b/nac3core/src/typecheck/type_inferencer/test.rs index f4e3f16d0..182116d41 100644 --- a/nac3core/src/typecheck/type_inferencer/test.rs +++ b/nac3core/src/typecheck/type_inferencer/test.rs @@ -9,7 +9,7 @@ use indoc::indoc; use inkwell::values::BasicValueEnum; use itertools::zip; use parking_lot::RwLock; -use rustpython_parser::parser::parse_program; +use nac3parser::parser::parse_program; use test_case::test_case; struct Resolver { diff --git a/nac3core/src/typecheck/typedef/mod.rs b/nac3core/src/typecheck/typedef/mod.rs index f12980d26..aa2c4517b 100644 --- a/nac3core/src/typecheck/typedef/mod.rs +++ b/nac3core/src/typecheck/typedef/mod.rs @@ -5,7 +5,7 @@ use std::rc::Rc; use std::sync::{Arc, Mutex}; use std::{borrow::Cow, collections::HashSet}; -use rustpython_parser::ast::StrRef; +use nac3parser::ast::StrRef; use super::unification_table::{UnificationKey, UnificationTable}; use crate::symbol_resolver::SymbolValue; diff --git a/nac3parser/Cargo.toml b/nac3parser/Cargo.toml index b938b3a6b..1db1b1bb6 100644 --- a/nac3parser/Cargo.toml +++ b/nac3parser/Cargo.toml @@ -1,10 +1,9 @@ [package] -name = "rustpython-parser" +name = "nac3parser" version = "0.1.2" description = "Parser for python code." -authors = [ "RustPython Team" ] +authors = [ "RustPython Team", "M-Labs" ] build = "build.rs" -repository = "https://github.com/RustPython/RustPython" license = "MIT" edition = "2018" @@ -12,7 +11,7 @@ edition = "2018" lalrpop = "0.19.6" [dependencies] -rustpython-ast = { path = "../ast" } +nac3ast = { path = "../nac3ast" } lalrpop-util = "0.19.6" log = "0.4.1" num-bigint = "0.4.0" diff --git a/nac3parser/README.md b/nac3parser/README.md index ebe9eb685..9dcbbaaf2 100644 --- a/nac3parser/README.md +++ b/nac3parser/README.md @@ -1,14 +1,14 @@ -# RustPython/parser +# nac3parser This directory has the code for python lexing, parsing and generating Abstract Syntax Trees (AST). +This is the RustPython parser with modifications for NAC3. + The steps are: - Lexical analysis: splits the source code into tokens. - Parsing and generating the AST: transforms those tokens into an AST. Uses `LALRPOP`, a Rust parser generator framework. -This crate is published on [https://docs.rs/rustpython-parser](https://docs.rs/rustpython-parser). - -We wrote [a blog post](https://rustpython.github.io/2020/04/02/thing-explainer-parser.html) with screenshots and an explanation to help you understand the steps by seeing them in action. +The RustPython team wrote [a blog post](https://rustpython.github.io/2020/04/02/thing-explainer-parser.html) with screenshots and an explanation to help you understand the steps by seeing them in action. For more information on LALRPOP, here is a link to the [LALRPOP book](https://github.com/lalrpop/lalrpop). @@ -54,13 +54,3 @@ Datatypes to support source location information. **mode.rs** Execution mode check. Allowed modes are `exec`, `eval` or `single`. - - -## How to use - -For example, one could do this: -``` - use rustpython_parser::{parser, ast}; - let python_source = "print('Hello world')"; - let python_ast = parser::parse_expression(python_source).unwrap(); -``` diff --git a/nac3parser/src/lib.rs b/nac3parser/src/lib.rs index 80030e61b..cab63529a 100644 --- a/nac3parser/src/lib.rs +++ b/nac3parser/src/lib.rs @@ -8,20 +8,17 @@ //! For example, one could do this: //! //! ``` -//! use rustpython_parser::{parser, ast}; +//! use nac3parser::{parser, ast}; //! //! let python_source = "print('Hello world')"; //! let python_ast = parser::parse_expression(python_source).unwrap(); //! //! ``` -#![doc(html_logo_url = "https://raw.githubusercontent.com/RustPython/RustPython/master/logo.png")] -#![doc(html_root_url = "https://docs.rs/rustpython-parser/")] - #[macro_use] extern crate log; use lalrpop_util::lalrpop_mod; -pub use rustpython_ast as ast; +pub use nac3ast as ast; pub mod error; mod fstring; diff --git a/nac3parser/src/parser.rs b/nac3parser/src/parser.rs index 02845f775..3df481b2e 100644 --- a/nac3parser/src/parser.rs +++ b/nac3parser/src/parser.rs @@ -32,7 +32,7 @@ pub fn parse_program(source: &str) -> Result { /// # Example /// ``` /// extern crate num_bigint; -/// use rustpython_parser::{parser, ast}; +/// use nac3parser::{parser, ast}; /// let expr = parser::parse_expression("1 + 2").unwrap(); /// /// assert_eq!( diff --git a/nac3standalone/Cargo.toml b/nac3standalone/Cargo.toml index ed3ac1f3c..df776f1a9 100644 --- a/nac3standalone/Cargo.toml +++ b/nac3standalone/Cargo.toml @@ -6,6 +6,6 @@ edition = "2018" [dependencies] inkwell = { git = "https://github.com/TheDan64/inkwell", branch = "master", features = ["llvm12-0"] } -rustpython-parser = { git = "https://github.com/m-labs/RustPython", branch = "parser-mod" } parking_lot = "0.11.1" +nac3parser = { path = "../nac3parser" } nac3core = { path = "../nac3core" } diff --git a/nac3standalone/src/basic_symbol_resolver.rs b/nac3standalone/src/basic_symbol_resolver.rs index 28061c102..82c1ac312 100644 --- a/nac3standalone/src/basic_symbol_resolver.rs +++ b/nac3standalone/src/basic_symbol_resolver.rs @@ -10,7 +10,7 @@ use nac3core::{ }, }; use parking_lot::{Mutex, RwLock}; -use rustpython_parser::ast::StrRef; +use nac3parser::ast::StrRef; use std::{collections::HashMap, sync::Arc}; pub struct ResolverInternal { diff --git a/nac3standalone/src/main.rs b/nac3standalone/src/main.rs index 4fc9b8d51..5b4e31dd0 100644 --- a/nac3standalone/src/main.rs +++ b/nac3standalone/src/main.rs @@ -4,7 +4,7 @@ use inkwell::{ OptimizationLevel, }; use nac3core::typecheck::type_inferencer::PrimitiveStore; -use rustpython_parser::parser; +use nac3parser::parser; use std::env; use std::fs; use std::{collections::HashMap, path::Path, sync::Arc, time::SystemTime};