forked from M-Labs/nac3
1
0
Fork 0

integrate nac3parser

This commit is contained in:
Sebastien Bourdeauducq 2021-11-03 17:11:00 +08:00
parent 48ce6bb6c5
commit bf7e2c295a
32 changed files with 73 additions and 86 deletions

65
Cargo.lock generated
View File

@ -489,9 +489,20 @@ version = "0.1.0"
dependencies = [
"inkwell",
"nac3core",
"nac3parser",
"parking_lot",
"pyo3",
"rustpython-parser",
]
[[package]]
name = "nac3ast"
version = "0.1.0"
dependencies = [
"fxhash",
"lazy_static",
"num-bigint 0.4.2",
"parking_lot",
"string-interner",
]
[[package]]
@ -503,22 +514,40 @@ dependencies = [
"inkwell",
"insta",
"itertools",
"nac3parser",
"num-bigint 0.3.3",
"num-traits",
"parking_lot",
"rayon",
"rustpython-parser",
"test-case",
]
[[package]]
name = "nac3parser"
version = "0.1.2"
dependencies = [
"ahash",
"insta",
"lalrpop",
"lalrpop-util",
"log",
"nac3ast",
"num-bigint 0.4.2",
"num-traits",
"phf",
"unic-emoji-char",
"unic-ucd-ident",
"unicode_names2",
]
[[package]]
name = "nac3standalone"
version = "0.1.0"
dependencies = [
"inkwell",
"nac3core",
"nac3parser",
"parking_lot",
"rustpython-parser",
]
[[package]]
@ -898,36 +927,6 @@ dependencies = [
"libloading",
]
[[package]]
name = "rustpython-ast"
version = "0.1.0"
source = "git+https://github.com/m-labs/RustPython?branch=parser-mod#efdf7829ba1a5f87d30df8eaff12a330544f3cbd"
dependencies = [
"fxhash",
"lazy_static",
"num-bigint 0.4.2",
"parking_lot",
"string-interner",
]
[[package]]
name = "rustpython-parser"
version = "0.1.2"
source = "git+https://github.com/m-labs/RustPython?branch=parser-mod#efdf7829ba1a5f87d30df8eaff12a330544f3cbd"
dependencies = [
"ahash",
"lalrpop",
"lalrpop-util",
"log",
"num-bigint 0.4.2",
"num-traits",
"phf",
"rustpython-ast",
"unic-emoji-char",
"unic-ucd-ident",
"unicode_names2",
]
[[package]]
name = "rustversion"
version = "1.0.5"

View File

@ -1,5 +1,7 @@
[workspace]
members = [
"nac3ast",
"nac3parser",
"nac3core",
"nac3standalone",
"nac3artiq",

View File

@ -12,7 +12,7 @@
pkgs.rustPlatform.buildRustPackage {
name = "nac3artiq";
src = self;
cargoSha256 = "sha256-si/yuVrRnXXLp8fMo5ZORHXa5JUqI2YqQ9U4R3UEUL0=";
cargoSha256 = "sha256-o/3QRbe1WEOA7NPyO1bXCLxUxUWBex8bPcApl/aT040=";
nativeBuildInputs = [ pkgs.python3 pkgs.llvm_12 ];
buildInputs = [ pkgs.python3 pkgs.libffi pkgs.libxml2 pkgs.llvm_12 ];
cargoBuildFlags = [ "--package" "nac3artiq" ];

View File

@ -10,7 +10,7 @@ crate-type = ["cdylib"]
[dependencies]
pyo3 = { version = "0.14", features = ["extension-module"] }
rustpython-parser = { git = "https://github.com/m-labs/RustPython", branch = "parser-mod" }
inkwell = { git = "https://github.com/TheDan64/inkwell", branch = "master", features = ["llvm12-0"] }
parking_lot = "0.11"
nac3parser = { path = "../nac3parser" }
nac3core = { path = "../nac3core" }

View File

@ -4,7 +4,7 @@ use nac3core::{
typecheck::typedef::{FunSignature, Type},
};
use rustpython_parser::ast::{Expr, ExprKind, Located, Stmt, StmtKind, StrRef};
use nac3parser::ast::{Expr, ExprKind, Located, Stmt, StmtKind, StrRef};
use inkwell::values::BasicValueEnum;

View File

@ -11,7 +11,7 @@ use inkwell::{
};
use pyo3::prelude::*;
use pyo3::{exceptions, types::PyList};
use rustpython_parser::{
use nac3parser::{
ast::{self, StrRef},
parser::{self, parse_program},
};

View File

@ -14,7 +14,7 @@ use pyo3::{
types::{PyList, PyModule, PyTuple},
PyAny, PyObject, PyResult, Python,
};
use rustpython_parser::ast::StrRef;
use nac3parser::ast::StrRef;
use std::{
cell::RefCell,
collections::{HashMap, HashSet},

View File

@ -1,7 +1,7 @@
[package]
name = "rustpython-ast"
name = "nac3ast"
version = "0.1.0"
authors = ["RustPython Team"]
authors = ["RustPython Team", "M-Labs"]
edition = "2018"
[features]

View File

@ -8,11 +8,11 @@ edition = "2018"
num-bigint = "0.3"
num-traits = "0.2"
inkwell = { git = "https://github.com/TheDan64/inkwell", branch = "master", features = ["llvm12-0"] }
rustpython-parser = { git = "https://github.com/m-labs/RustPython", branch = "parser-mod" }
itertools = "0.10.1"
crossbeam = "0.8.1"
parking_lot = "0.11.1"
rayon = "1.5.1"
nac3parser = { path = "../nac3parser" }
[dev-dependencies]
test-case = "1.2.0"

View File

@ -7,7 +7,7 @@ use crate::{
},
};
use rustpython_parser::ast::StrRef;
use nac3parser::ast::StrRef;
use std::collections::HashMap;
pub struct ConcreteTypeStore {

View File

@ -15,7 +15,7 @@ use inkwell::{
AddressSpace,
};
use itertools::{chain, izip, zip, Itertools};
use rustpython_parser::ast::{
use nac3parser::ast::{
self, Boolop, Comprehension, Constant, Expr, ExprKind, Operator, StrRef,
};

View File

@ -4,7 +4,7 @@ use crate::{
typecheck::typedef::{FunSignature, Type},
};
use inkwell::values::{BasicValueEnum, PointerValue};
use rustpython_parser::ast::{Expr, Stmt, StrRef};
use nac3parser::ast::{Expr, Stmt, StrRef};
pub trait CodeGenerator {
/// Return the module name for the code generator.

View File

@ -19,7 +19,7 @@ use inkwell::{
};
use itertools::Itertools;
use parking_lot::{Condvar, Mutex};
use rustpython_parser::ast::{Stmt, StrRef};
use nac3parser::ast::{Stmt, StrRef};
use std::collections::HashMap;
use std::sync::{
atomic::{AtomicBool, Ordering},

View File

@ -1,7 +1,7 @@
use super::{expr::destructure_range, CodeGenContext, CodeGenerator};
use crate::typecheck::typedef::Type;
use inkwell::values::{BasicValue, BasicValueEnum, PointerValue};
use rustpython_parser::ast::{Expr, ExprKind, Stmt, StmtKind};
use nac3parser::ast::{Expr, ExprKind, Stmt, StmtKind};
pub fn gen_var<'ctx, 'a>(ctx: &mut CodeGenContext<'ctx, 'a>, ty: Type) -> PointerValue<'ctx> {
// put the alloca in init block

View File

@ -16,7 +16,7 @@ use crate::{
use indoc::indoc;
use inkwell::values::BasicValueEnum;
use parking_lot::RwLock;
use rustpython_parser::{
use nac3parser::{
ast::{fold::Fold, StrRef},
parser::parse_program,
};

View File

@ -1,4 +1,4 @@
use rustpython_parser::ast;
use nac3parser::ast;
use std::vec::Vec;
#[derive(Clone, Copy, PartialEq)]

View File

@ -10,7 +10,7 @@ use crate::typecheck::{
use crate::{location::Location, typecheck::typedef::TypeEnum};
use itertools::{chain, izip};
use parking_lot::RwLock;
use rustpython_parser::ast::{Expr, StrRef};
use nac3parser::ast::{Expr, StrRef};
use inkwell::values::BasicValueEnum;
#[derive(Clone, PartialEq, Debug)]
@ -61,7 +61,7 @@ pub fn parse_type_annotation<T>(
primitives: &PrimitiveStore,
expr: &Expr<T>,
) -> Result<Type, String> {
use rustpython_parser::ast::ExprKind::*;
use nac3parser::ast::ExprKind::*;
let ids = IDENTIFIER_ID.with(|ids| {
*ids
});

View File

@ -1,6 +1,6 @@
use std::cell::RefCell;
use rustpython_parser::ast::fold::Fold;
use nac3parser::ast::fold::Fold;
use crate::{
symbol_resolver::SymbolValue,

View File

@ -16,7 +16,7 @@ use crate::{
};
use itertools::{izip, Itertools};
use parking_lot::RwLock;
use rustpython_parser::ast::{self, Stmt, StrRef};
use nac3parser::ast::{self, Stmt, StrRef};
use inkwell::values::BasicValueEnum;
#[derive(PartialEq, Eq, PartialOrd, Ord, Clone, Copy, Hash, Debug)]

View File

@ -10,7 +10,7 @@ use crate::{
};
use indoc::indoc;
use parking_lot::Mutex;
use rustpython_parser::{ast::fold::Fold, parser::parse_program};
use nac3parser::{ast::fold::Fold, parser::parse_program};
use std::{collections::HashMap, sync::Arc};
use test_case::test_case;

View File

@ -2,7 +2,7 @@ use crate::typecheck::typedef::TypeEnum;
use super::type_inferencer::Inferencer;
use super::typedef::Type;
use rustpython_parser::ast::{self, Expr, ExprKind, Stmt, StmtKind, StrRef};
use nac3parser::ast::{self, Expr, ExprKind, Stmt, StmtKind, StrRef};
use std::{collections::HashSet, iter::once};
impl<'a> Inferencer<'a> {

View File

@ -2,8 +2,8 @@ use crate::typecheck::{
type_inferencer::*,
typedef::{FunSignature, FuncArg, Type, TypeEnum, Unifier},
};
use rustpython_parser::ast;
use rustpython_parser::ast::{Cmpop, Operator, Unaryop};
use nac3parser::ast;
use nac3parser::ast::{Cmpop, Operator, Unaryop};
use std::borrow::Borrow;
use std::collections::HashMap;

View File

@ -7,7 +7,7 @@ use super::typedef::{Call, FunSignature, FuncArg, Type, TypeEnum, Unifier};
use super::{magic_methods::*, typedef::CallId};
use crate::{symbol_resolver::SymbolResolver, toplevel::TopLevelContext};
use itertools::izip;
use rustpython_parser::ast::{
use nac3parser::ast::{
self,
fold::{self, Fold},
Arguments, Comprehension, ExprKind, Located, Location, StrRef,

View File

@ -9,7 +9,7 @@ use indoc::indoc;
use inkwell::values::BasicValueEnum;
use itertools::zip;
use parking_lot::RwLock;
use rustpython_parser::parser::parse_program;
use nac3parser::parser::parse_program;
use test_case::test_case;
struct Resolver {

View File

@ -5,7 +5,7 @@ use std::rc::Rc;
use std::sync::{Arc, Mutex};
use std::{borrow::Cow, collections::HashSet};
use rustpython_parser::ast::StrRef;
use nac3parser::ast::StrRef;
use super::unification_table::{UnificationKey, UnificationTable};
use crate::symbol_resolver::SymbolValue;

View File

@ -1,10 +1,9 @@
[package]
name = "rustpython-parser"
name = "nac3parser"
version = "0.1.2"
description = "Parser for python code."
authors = [ "RustPython Team" ]
authors = [ "RustPython Team", "M-Labs" ]
build = "build.rs"
repository = "https://github.com/RustPython/RustPython"
license = "MIT"
edition = "2018"
@ -12,7 +11,7 @@ edition = "2018"
lalrpop = "0.19.6"
[dependencies]
rustpython-ast = { path = "../ast" }
nac3ast = { path = "../nac3ast" }
lalrpop-util = "0.19.6"
log = "0.4.1"
num-bigint = "0.4.0"

View File

@ -1,14 +1,14 @@
# RustPython/parser
# nac3parser
This directory has the code for python lexing, parsing and generating Abstract Syntax Trees (AST).
This is the RustPython parser with modifications for NAC3.
The steps are:
- Lexical analysis: splits the source code into tokens.
- Parsing and generating the AST: transforms those tokens into an AST. Uses `LALRPOP`, a Rust parser generator framework.
This crate is published on [https://docs.rs/rustpython-parser](https://docs.rs/rustpython-parser).
We wrote [a blog post](https://rustpython.github.io/2020/04/02/thing-explainer-parser.html) with screenshots and an explanation to help you understand the steps by seeing them in action.
The RustPython team wrote [a blog post](https://rustpython.github.io/2020/04/02/thing-explainer-parser.html) with screenshots and an explanation to help you understand the steps by seeing them in action.
For more information on LALRPOP, here is a link to the [LALRPOP book](https://github.com/lalrpop/lalrpop).
@ -54,13 +54,3 @@ Datatypes to support source location information.
**mode.rs**
Execution mode check. Allowed modes are `exec`, `eval` or `single`.
## How to use
For example, one could do this:
```
use rustpython_parser::{parser, ast};
let python_source = "print('Hello world')";
let python_ast = parser::parse_expression(python_source).unwrap();
```

View File

@ -8,20 +8,17 @@
//! For example, one could do this:
//!
//! ```
//! use rustpython_parser::{parser, ast};
//! use nac3parser::{parser, ast};
//!
//! let python_source = "print('Hello world')";
//! let python_ast = parser::parse_expression(python_source).unwrap();
//!
//! ```
#![doc(html_logo_url = "https://raw.githubusercontent.com/RustPython/RustPython/master/logo.png")]
#![doc(html_root_url = "https://docs.rs/rustpython-parser/")]
#[macro_use]
extern crate log;
use lalrpop_util::lalrpop_mod;
pub use rustpython_ast as ast;
pub use nac3ast as ast;
pub mod error;
mod fstring;

View File

@ -32,7 +32,7 @@ pub fn parse_program(source: &str) -> Result<ast::Suite, ParseError> {
/// # Example
/// ```
/// extern crate num_bigint;
/// use rustpython_parser::{parser, ast};
/// use nac3parser::{parser, ast};
/// let expr = parser::parse_expression("1 + 2").unwrap();
///
/// assert_eq!(

View File

@ -6,6 +6,6 @@ edition = "2018"
[dependencies]
inkwell = { git = "https://github.com/TheDan64/inkwell", branch = "master", features = ["llvm12-0"] }
rustpython-parser = { git = "https://github.com/m-labs/RustPython", branch = "parser-mod" }
parking_lot = "0.11.1"
nac3parser = { path = "../nac3parser" }
nac3core = { path = "../nac3core" }

View File

@ -10,7 +10,7 @@ use nac3core::{
},
};
use parking_lot::{Mutex, RwLock};
use rustpython_parser::ast::StrRef;
use nac3parser::ast::StrRef;
use std::{collections::HashMap, sync::Arc};
pub struct ResolverInternal {

View File

@ -4,7 +4,7 @@ use inkwell::{
OptimizationLevel,
};
use nac3core::typecheck::type_inferencer::PrimitiveStore;
use rustpython_parser::parser;
use nac3parser::parser;
use std::env;
use std::fs;
use std::{collections::HashMap, path::Path, sync::Arc, time::SystemTime};