integrate nac3parser

This commit is contained in:
Sebastien Bourdeauducq 2021-11-03 17:11:00 +08:00
parent 48ce6bb6c5
commit bf7e2c295a
32 changed files with 73 additions and 86 deletions

65
Cargo.lock generated
View File

@ -489,9 +489,20 @@ version = "0.1.0"
dependencies = [ dependencies = [
"inkwell", "inkwell",
"nac3core", "nac3core",
"nac3parser",
"parking_lot", "parking_lot",
"pyo3", "pyo3",
"rustpython-parser", ]
[[package]]
name = "nac3ast"
version = "0.1.0"
dependencies = [
"fxhash",
"lazy_static",
"num-bigint 0.4.2",
"parking_lot",
"string-interner",
] ]
[[package]] [[package]]
@ -503,22 +514,40 @@ dependencies = [
"inkwell", "inkwell",
"insta", "insta",
"itertools", "itertools",
"nac3parser",
"num-bigint 0.3.3", "num-bigint 0.3.3",
"num-traits", "num-traits",
"parking_lot", "parking_lot",
"rayon", "rayon",
"rustpython-parser",
"test-case", "test-case",
] ]
[[package]]
name = "nac3parser"
version = "0.1.2"
dependencies = [
"ahash",
"insta",
"lalrpop",
"lalrpop-util",
"log",
"nac3ast",
"num-bigint 0.4.2",
"num-traits",
"phf",
"unic-emoji-char",
"unic-ucd-ident",
"unicode_names2",
]
[[package]] [[package]]
name = "nac3standalone" name = "nac3standalone"
version = "0.1.0" version = "0.1.0"
dependencies = [ dependencies = [
"inkwell", "inkwell",
"nac3core", "nac3core",
"nac3parser",
"parking_lot", "parking_lot",
"rustpython-parser",
] ]
[[package]] [[package]]
@ -898,36 +927,6 @@ dependencies = [
"libloading", "libloading",
] ]
[[package]]
name = "rustpython-ast"
version = "0.1.0"
source = "git+https://github.com/m-labs/RustPython?branch=parser-mod#efdf7829ba1a5f87d30df8eaff12a330544f3cbd"
dependencies = [
"fxhash",
"lazy_static",
"num-bigint 0.4.2",
"parking_lot",
"string-interner",
]
[[package]]
name = "rustpython-parser"
version = "0.1.2"
source = "git+https://github.com/m-labs/RustPython?branch=parser-mod#efdf7829ba1a5f87d30df8eaff12a330544f3cbd"
dependencies = [
"ahash",
"lalrpop",
"lalrpop-util",
"log",
"num-bigint 0.4.2",
"num-traits",
"phf",
"rustpython-ast",
"unic-emoji-char",
"unic-ucd-ident",
"unicode_names2",
]
[[package]] [[package]]
name = "rustversion" name = "rustversion"
version = "1.0.5" version = "1.0.5"

View File

@ -1,5 +1,7 @@
[workspace] [workspace]
members = [ members = [
"nac3ast",
"nac3parser",
"nac3core", "nac3core",
"nac3standalone", "nac3standalone",
"nac3artiq", "nac3artiq",

View File

@ -12,7 +12,7 @@
pkgs.rustPlatform.buildRustPackage { pkgs.rustPlatform.buildRustPackage {
name = "nac3artiq"; name = "nac3artiq";
src = self; src = self;
cargoSha256 = "sha256-si/yuVrRnXXLp8fMo5ZORHXa5JUqI2YqQ9U4R3UEUL0="; cargoSha256 = "sha256-o/3QRbe1WEOA7NPyO1bXCLxUxUWBex8bPcApl/aT040=";
nativeBuildInputs = [ pkgs.python3 pkgs.llvm_12 ]; nativeBuildInputs = [ pkgs.python3 pkgs.llvm_12 ];
buildInputs = [ pkgs.python3 pkgs.libffi pkgs.libxml2 pkgs.llvm_12 ]; buildInputs = [ pkgs.python3 pkgs.libffi pkgs.libxml2 pkgs.llvm_12 ];
cargoBuildFlags = [ "--package" "nac3artiq" ]; cargoBuildFlags = [ "--package" "nac3artiq" ];

View File

@ -10,7 +10,7 @@ crate-type = ["cdylib"]
[dependencies] [dependencies]
pyo3 = { version = "0.14", features = ["extension-module"] } pyo3 = { version = "0.14", features = ["extension-module"] }
rustpython-parser = { git = "https://github.com/m-labs/RustPython", branch = "parser-mod" }
inkwell = { git = "https://github.com/TheDan64/inkwell", branch = "master", features = ["llvm12-0"] } inkwell = { git = "https://github.com/TheDan64/inkwell", branch = "master", features = ["llvm12-0"] }
parking_lot = "0.11" parking_lot = "0.11"
nac3parser = { path = "../nac3parser" }
nac3core = { path = "../nac3core" } nac3core = { path = "../nac3core" }

View File

@ -4,7 +4,7 @@ use nac3core::{
typecheck::typedef::{FunSignature, Type}, typecheck::typedef::{FunSignature, Type},
}; };
use rustpython_parser::ast::{Expr, ExprKind, Located, Stmt, StmtKind, StrRef}; use nac3parser::ast::{Expr, ExprKind, Located, Stmt, StmtKind, StrRef};
use inkwell::values::BasicValueEnum; use inkwell::values::BasicValueEnum;

View File

@ -11,7 +11,7 @@ use inkwell::{
}; };
use pyo3::prelude::*; use pyo3::prelude::*;
use pyo3::{exceptions, types::PyList}; use pyo3::{exceptions, types::PyList};
use rustpython_parser::{ use nac3parser::{
ast::{self, StrRef}, ast::{self, StrRef},
parser::{self, parse_program}, parser::{self, parse_program},
}; };

View File

@ -14,7 +14,7 @@ use pyo3::{
types::{PyList, PyModule, PyTuple}, types::{PyList, PyModule, PyTuple},
PyAny, PyObject, PyResult, Python, PyAny, PyObject, PyResult, Python,
}; };
use rustpython_parser::ast::StrRef; use nac3parser::ast::StrRef;
use std::{ use std::{
cell::RefCell, cell::RefCell,
collections::{HashMap, HashSet}, collections::{HashMap, HashSet},

View File

@ -1,7 +1,7 @@
[package] [package]
name = "rustpython-ast" name = "nac3ast"
version = "0.1.0" version = "0.1.0"
authors = ["RustPython Team"] authors = ["RustPython Team", "M-Labs"]
edition = "2018" edition = "2018"
[features] [features]

View File

@ -8,11 +8,11 @@ edition = "2018"
num-bigint = "0.3" num-bigint = "0.3"
num-traits = "0.2" num-traits = "0.2"
inkwell = { git = "https://github.com/TheDan64/inkwell", branch = "master", features = ["llvm12-0"] } inkwell = { git = "https://github.com/TheDan64/inkwell", branch = "master", features = ["llvm12-0"] }
rustpython-parser = { git = "https://github.com/m-labs/RustPython", branch = "parser-mod" }
itertools = "0.10.1" itertools = "0.10.1"
crossbeam = "0.8.1" crossbeam = "0.8.1"
parking_lot = "0.11.1" parking_lot = "0.11.1"
rayon = "1.5.1" rayon = "1.5.1"
nac3parser = { path = "../nac3parser" }
[dev-dependencies] [dev-dependencies]
test-case = "1.2.0" test-case = "1.2.0"

View File

@ -7,7 +7,7 @@ use crate::{
}, },
}; };
use rustpython_parser::ast::StrRef; use nac3parser::ast::StrRef;
use std::collections::HashMap; use std::collections::HashMap;
pub struct ConcreteTypeStore { pub struct ConcreteTypeStore {

View File

@ -15,7 +15,7 @@ use inkwell::{
AddressSpace, AddressSpace,
}; };
use itertools::{chain, izip, zip, Itertools}; use itertools::{chain, izip, zip, Itertools};
use rustpython_parser::ast::{ use nac3parser::ast::{
self, Boolop, Comprehension, Constant, Expr, ExprKind, Operator, StrRef, self, Boolop, Comprehension, Constant, Expr, ExprKind, Operator, StrRef,
}; };

View File

@ -4,7 +4,7 @@ use crate::{
typecheck::typedef::{FunSignature, Type}, typecheck::typedef::{FunSignature, Type},
}; };
use inkwell::values::{BasicValueEnum, PointerValue}; use inkwell::values::{BasicValueEnum, PointerValue};
use rustpython_parser::ast::{Expr, Stmt, StrRef}; use nac3parser::ast::{Expr, Stmt, StrRef};
pub trait CodeGenerator { pub trait CodeGenerator {
/// Return the module name for the code generator. /// Return the module name for the code generator.

View File

@ -19,7 +19,7 @@ use inkwell::{
}; };
use itertools::Itertools; use itertools::Itertools;
use parking_lot::{Condvar, Mutex}; use parking_lot::{Condvar, Mutex};
use rustpython_parser::ast::{Stmt, StrRef}; use nac3parser::ast::{Stmt, StrRef};
use std::collections::HashMap; use std::collections::HashMap;
use std::sync::{ use std::sync::{
atomic::{AtomicBool, Ordering}, atomic::{AtomicBool, Ordering},

View File

@ -1,7 +1,7 @@
use super::{expr::destructure_range, CodeGenContext, CodeGenerator}; use super::{expr::destructure_range, CodeGenContext, CodeGenerator};
use crate::typecheck::typedef::Type; use crate::typecheck::typedef::Type;
use inkwell::values::{BasicValue, BasicValueEnum, PointerValue}; use inkwell::values::{BasicValue, BasicValueEnum, PointerValue};
use rustpython_parser::ast::{Expr, ExprKind, Stmt, StmtKind}; use nac3parser::ast::{Expr, ExprKind, Stmt, StmtKind};
pub fn gen_var<'ctx, 'a>(ctx: &mut CodeGenContext<'ctx, 'a>, ty: Type) -> PointerValue<'ctx> { pub fn gen_var<'ctx, 'a>(ctx: &mut CodeGenContext<'ctx, 'a>, ty: Type) -> PointerValue<'ctx> {
// put the alloca in init block // put the alloca in init block

View File

@ -16,7 +16,7 @@ use crate::{
use indoc::indoc; use indoc::indoc;
use inkwell::values::BasicValueEnum; use inkwell::values::BasicValueEnum;
use parking_lot::RwLock; use parking_lot::RwLock;
use rustpython_parser::{ use nac3parser::{
ast::{fold::Fold, StrRef}, ast::{fold::Fold, StrRef},
parser::parse_program, parser::parse_program,
}; };

View File

@ -1,4 +1,4 @@
use rustpython_parser::ast; use nac3parser::ast;
use std::vec::Vec; use std::vec::Vec;
#[derive(Clone, Copy, PartialEq)] #[derive(Clone, Copy, PartialEq)]

View File

@ -10,7 +10,7 @@ use crate::typecheck::{
use crate::{location::Location, typecheck::typedef::TypeEnum}; use crate::{location::Location, typecheck::typedef::TypeEnum};
use itertools::{chain, izip}; use itertools::{chain, izip};
use parking_lot::RwLock; use parking_lot::RwLock;
use rustpython_parser::ast::{Expr, StrRef}; use nac3parser::ast::{Expr, StrRef};
use inkwell::values::BasicValueEnum; use inkwell::values::BasicValueEnum;
#[derive(Clone, PartialEq, Debug)] #[derive(Clone, PartialEq, Debug)]
@ -61,7 +61,7 @@ pub fn parse_type_annotation<T>(
primitives: &PrimitiveStore, primitives: &PrimitiveStore,
expr: &Expr<T>, expr: &Expr<T>,
) -> Result<Type, String> { ) -> Result<Type, String> {
use rustpython_parser::ast::ExprKind::*; use nac3parser::ast::ExprKind::*;
let ids = IDENTIFIER_ID.with(|ids| { let ids = IDENTIFIER_ID.with(|ids| {
*ids *ids
}); });

View File

@ -1,6 +1,6 @@
use std::cell::RefCell; use std::cell::RefCell;
use rustpython_parser::ast::fold::Fold; use nac3parser::ast::fold::Fold;
use crate::{ use crate::{
symbol_resolver::SymbolValue, symbol_resolver::SymbolValue,

View File

@ -16,7 +16,7 @@ use crate::{
}; };
use itertools::{izip, Itertools}; use itertools::{izip, Itertools};
use parking_lot::RwLock; use parking_lot::RwLock;
use rustpython_parser::ast::{self, Stmt, StrRef}; use nac3parser::ast::{self, Stmt, StrRef};
use inkwell::values::BasicValueEnum; use inkwell::values::BasicValueEnum;
#[derive(PartialEq, Eq, PartialOrd, Ord, Clone, Copy, Hash, Debug)] #[derive(PartialEq, Eq, PartialOrd, Ord, Clone, Copy, Hash, Debug)]

View File

@ -10,7 +10,7 @@ use crate::{
}; };
use indoc::indoc; use indoc::indoc;
use parking_lot::Mutex; use parking_lot::Mutex;
use rustpython_parser::{ast::fold::Fold, parser::parse_program}; use nac3parser::{ast::fold::Fold, parser::parse_program};
use std::{collections::HashMap, sync::Arc}; use std::{collections::HashMap, sync::Arc};
use test_case::test_case; use test_case::test_case;

View File

@ -2,7 +2,7 @@ use crate::typecheck::typedef::TypeEnum;
use super::type_inferencer::Inferencer; use super::type_inferencer::Inferencer;
use super::typedef::Type; use super::typedef::Type;
use rustpython_parser::ast::{self, Expr, ExprKind, Stmt, StmtKind, StrRef}; use nac3parser::ast::{self, Expr, ExprKind, Stmt, StmtKind, StrRef};
use std::{collections::HashSet, iter::once}; use std::{collections::HashSet, iter::once};
impl<'a> Inferencer<'a> { impl<'a> Inferencer<'a> {

View File

@ -2,8 +2,8 @@ use crate::typecheck::{
type_inferencer::*, type_inferencer::*,
typedef::{FunSignature, FuncArg, Type, TypeEnum, Unifier}, typedef::{FunSignature, FuncArg, Type, TypeEnum, Unifier},
}; };
use rustpython_parser::ast; use nac3parser::ast;
use rustpython_parser::ast::{Cmpop, Operator, Unaryop}; use nac3parser::ast::{Cmpop, Operator, Unaryop};
use std::borrow::Borrow; use std::borrow::Borrow;
use std::collections::HashMap; use std::collections::HashMap;

View File

@ -7,7 +7,7 @@ use super::typedef::{Call, FunSignature, FuncArg, Type, TypeEnum, Unifier};
use super::{magic_methods::*, typedef::CallId}; use super::{magic_methods::*, typedef::CallId};
use crate::{symbol_resolver::SymbolResolver, toplevel::TopLevelContext}; use crate::{symbol_resolver::SymbolResolver, toplevel::TopLevelContext};
use itertools::izip; use itertools::izip;
use rustpython_parser::ast::{ use nac3parser::ast::{
self, self,
fold::{self, Fold}, fold::{self, Fold},
Arguments, Comprehension, ExprKind, Located, Location, StrRef, Arguments, Comprehension, ExprKind, Located, Location, StrRef,

View File

@ -9,7 +9,7 @@ use indoc::indoc;
use inkwell::values::BasicValueEnum; use inkwell::values::BasicValueEnum;
use itertools::zip; use itertools::zip;
use parking_lot::RwLock; use parking_lot::RwLock;
use rustpython_parser::parser::parse_program; use nac3parser::parser::parse_program;
use test_case::test_case; use test_case::test_case;
struct Resolver { struct Resolver {

View File

@ -5,7 +5,7 @@ use std::rc::Rc;
use std::sync::{Arc, Mutex}; use std::sync::{Arc, Mutex};
use std::{borrow::Cow, collections::HashSet}; use std::{borrow::Cow, collections::HashSet};
use rustpython_parser::ast::StrRef; use nac3parser::ast::StrRef;
use super::unification_table::{UnificationKey, UnificationTable}; use super::unification_table::{UnificationKey, UnificationTable};
use crate::symbol_resolver::SymbolValue; use crate::symbol_resolver::SymbolValue;

View File

@ -1,10 +1,9 @@
[package] [package]
name = "rustpython-parser" name = "nac3parser"
version = "0.1.2" version = "0.1.2"
description = "Parser for python code." description = "Parser for python code."
authors = [ "RustPython Team" ] authors = [ "RustPython Team", "M-Labs" ]
build = "build.rs" build = "build.rs"
repository = "https://github.com/RustPython/RustPython"
license = "MIT" license = "MIT"
edition = "2018" edition = "2018"
@ -12,7 +11,7 @@ edition = "2018"
lalrpop = "0.19.6" lalrpop = "0.19.6"
[dependencies] [dependencies]
rustpython-ast = { path = "../ast" } nac3ast = { path = "../nac3ast" }
lalrpop-util = "0.19.6" lalrpop-util = "0.19.6"
log = "0.4.1" log = "0.4.1"
num-bigint = "0.4.0" num-bigint = "0.4.0"

View File

@ -1,14 +1,14 @@
# RustPython/parser # nac3parser
This directory has the code for python lexing, parsing and generating Abstract Syntax Trees (AST). This directory has the code for python lexing, parsing and generating Abstract Syntax Trees (AST).
This is the RustPython parser with modifications for NAC3.
The steps are: The steps are:
- Lexical analysis: splits the source code into tokens. - Lexical analysis: splits the source code into tokens.
- Parsing and generating the AST: transforms those tokens into an AST. Uses `LALRPOP`, a Rust parser generator framework. - Parsing and generating the AST: transforms those tokens into an AST. Uses `LALRPOP`, a Rust parser generator framework.
This crate is published on [https://docs.rs/rustpython-parser](https://docs.rs/rustpython-parser). The RustPython team wrote [a blog post](https://rustpython.github.io/2020/04/02/thing-explainer-parser.html) with screenshots and an explanation to help you understand the steps by seeing them in action.
We wrote [a blog post](https://rustpython.github.io/2020/04/02/thing-explainer-parser.html) with screenshots and an explanation to help you understand the steps by seeing them in action.
For more information on LALRPOP, here is a link to the [LALRPOP book](https://github.com/lalrpop/lalrpop). For more information on LALRPOP, here is a link to the [LALRPOP book](https://github.com/lalrpop/lalrpop).
@ -54,13 +54,3 @@ Datatypes to support source location information.
**mode.rs** **mode.rs**
Execution mode check. Allowed modes are `exec`, `eval` or `single`. Execution mode check. Allowed modes are `exec`, `eval` or `single`.
## How to use
For example, one could do this:
```
use rustpython_parser::{parser, ast};
let python_source = "print('Hello world')";
let python_ast = parser::parse_expression(python_source).unwrap();
```

View File

@ -8,20 +8,17 @@
//! For example, one could do this: //! For example, one could do this:
//! //!
//! ``` //! ```
//! use rustpython_parser::{parser, ast}; //! use nac3parser::{parser, ast};
//! //!
//! let python_source = "print('Hello world')"; //! let python_source = "print('Hello world')";
//! let python_ast = parser::parse_expression(python_source).unwrap(); //! let python_ast = parser::parse_expression(python_source).unwrap();
//! //!
//! ``` //! ```
#![doc(html_logo_url = "https://raw.githubusercontent.com/RustPython/RustPython/master/logo.png")]
#![doc(html_root_url = "https://docs.rs/rustpython-parser/")]
#[macro_use] #[macro_use]
extern crate log; extern crate log;
use lalrpop_util::lalrpop_mod; use lalrpop_util::lalrpop_mod;
pub use rustpython_ast as ast; pub use nac3ast as ast;
pub mod error; pub mod error;
mod fstring; mod fstring;

View File

@ -32,7 +32,7 @@ pub fn parse_program(source: &str) -> Result<ast::Suite, ParseError> {
/// # Example /// # Example
/// ``` /// ```
/// extern crate num_bigint; /// extern crate num_bigint;
/// use rustpython_parser::{parser, ast}; /// use nac3parser::{parser, ast};
/// let expr = parser::parse_expression("1 + 2").unwrap(); /// let expr = parser::parse_expression("1 + 2").unwrap();
/// ///
/// assert_eq!( /// assert_eq!(

View File

@ -6,6 +6,6 @@ edition = "2018"
[dependencies] [dependencies]
inkwell = { git = "https://github.com/TheDan64/inkwell", branch = "master", features = ["llvm12-0"] } inkwell = { git = "https://github.com/TheDan64/inkwell", branch = "master", features = ["llvm12-0"] }
rustpython-parser = { git = "https://github.com/m-labs/RustPython", branch = "parser-mod" }
parking_lot = "0.11.1" parking_lot = "0.11.1"
nac3parser = { path = "../nac3parser" }
nac3core = { path = "../nac3core" } nac3core = { path = "../nac3core" }

View File

@ -10,7 +10,7 @@ use nac3core::{
}, },
}; };
use parking_lot::{Mutex, RwLock}; use parking_lot::{Mutex, RwLock};
use rustpython_parser::ast::StrRef; use nac3parser::ast::StrRef;
use std::{collections::HashMap, sync::Arc}; use std::{collections::HashMap, sync::Arc};
pub struct ResolverInternal { pub struct ResolverInternal {

View File

@ -4,7 +4,7 @@ use inkwell::{
OptimizationLevel, OptimizationLevel,
}; };
use nac3core::typecheck::type_inferencer::PrimitiveStore; use nac3core::typecheck::type_inferencer::PrimitiveStore;
use rustpython_parser::parser; use nac3parser::parser;
use std::env; use std::env;
use std::fs; use std::fs;
use std::{collections::HashMap, path::Path, sync::Arc, time::SystemTime}; use std::{collections::HashMap, path::Path, sync::Arc, time::SystemTime};