From 48ce6bb6c5e78b7a1980a2431351c4808e803100 Mon Sep 17 00:00:00 2001 From: pca006132 Date: Wed, 3 Nov 2021 16:35:16 +0800 Subject: [PATCH] rustpython-parser: string interner, optimizations, thread local cache corresponds to M-Labs RustPython fork at efdf7829ba1a5f87d30df8eaff12a330544f3cbd branch parser-mod --- nac3ast/Cargo.toml | 4 + nac3ast/asdl_rs.py | 2 +- nac3ast/src/ast_gen.rs | 1316 +++++++++++++++++---------------- nac3ast/src/constant.rs | 2 +- nac3ast/src/fold_helpers.rs | 2 + nac3ast/src/lib.rs | 3 + nac3parser/src/function.rs | 2 +- nac3parser/src/lexer.rs | 29 +- nac3parser/src/python.lalrpop | 19 +- nac3parser/src/token.rs | 5 +- 10 files changed, 737 insertions(+), 647 deletions(-) diff --git a/nac3ast/Cargo.toml b/nac3ast/Cargo.toml index 24aa0e124..ab49bf021 100644 --- a/nac3ast/Cargo.toml +++ b/nac3ast/Cargo.toml @@ -11,3 +11,7 @@ fold = [] [dependencies] num-bigint = "0.4.0" +lazy_static = "1.4.0" +parking_lot = "0.11.1" +string-interner = "0.13.0" +fxhash = "0.2.1" diff --git a/nac3ast/asdl_rs.py b/nac3ast/asdl_rs.py index 3c5d6fd56..7b63e595e 100755 --- a/nac3ast/asdl_rs.py +++ b/nac3ast/asdl_rs.py @@ -185,7 +185,7 @@ class StructVisitor(TypeInfoEmitVisitor): self.sum_with_constructors(sum, name, depth) def emit_attrs(self, depth): - self.emit("#[derive(Debug, PartialEq)]", depth) + self.emit("#[derive(Clone, Debug, PartialEq)]", depth) def simple_sum(self, sum, name, depth): rustname = get_rust_type(name) diff --git a/nac3ast/src/ast_gen.rs b/nac3ast/src/ast_gen.rs index 31380e087..812944677 100644 --- a/nac3ast/src/ast_gen.rs +++ b/nac3ast/src/ast_gen.rs @@ -1,11 +1,80 @@ // File automatically generated by ast/asdl_rs.py. -pub use crate::constant::*; pub use crate::location::Location; +pub use crate::constant::*; -type Ident = String; +use std::{fmt, collections::HashMap, cell::RefCell}; +use parking_lot::{Mutex, MutexGuard}; +use string_interner::{DefaultBackend, DefaultSymbol, StringInterner, symbol::SymbolU32}; +use fxhash::FxBuildHasher; -#[derive(Debug, PartialEq)] +pub type Interner = StringInterner, FxBuildHasher>; +lazy_static! { + static ref INTERNER: Mutex = Mutex::new(StringInterner::with_hasher(FxBuildHasher::default())); +} + +thread_local! { + static LOCAL_INTERNER: RefCell> = Default::default(); +} + +#[derive(Eq, PartialEq, Copy, Clone, Hash)] +pub struct StrRef(SymbolU32); + +impl fmt::Debug for StrRef { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + let s: String = (*self).into(); + write!(f, "{:?}", s) + } +} + +impl fmt::Display for StrRef { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + let s: String = (*self).into(); + write!(f, "{}", s) + } +} + +impl From for StrRef { + fn from(s: String) -> Self { + get_str_ref(&mut get_str_ref_lock(), &s) + } +} + +impl From<&str> for StrRef { + fn from(s: &str) -> Self { + // thread local cache + LOCAL_INTERNER.with(|local| { + let mut local = local.borrow_mut(); + local.get(s).copied().unwrap_or_else(|| { + let r = get_str_ref(&mut get_str_ref_lock(), &s); + local.insert(s.to_string(), r); + r + }) + }) + } +} + +impl From for String{ + fn from(s: StrRef) -> Self { + get_str_from_ref(&get_str_ref_lock(), s).to_string() + } +} + +pub fn get_str_ref_lock<'a>() -> MutexGuard<'a, Interner> { + INTERNER.lock() +} + +pub fn get_str_ref(lock: &mut MutexGuard, str: &str) -> StrRef { + StrRef(lock.get_or_intern(str)) +} + +pub fn get_str_from_ref<'a>(lock: &'a MutexGuard, id: StrRef) -> &'a str { + lock.resolve(id.0).unwrap() +} + +type Ident = StrRef; + +#[derive(Clone, Debug, PartialEq)] pub struct Located { pub location: Location, pub custom: U, @@ -14,15 +83,11 @@ pub struct Located { impl Located { pub fn new(location: Location, node: T) -> Self { - Self { - location, - custom: (), - node, - } + Self { location, custom: (), node } } } -#[derive(Debug, PartialEq)] +#[derive(Clone, Debug, PartialEq)] pub enum Mod { Module { body: Vec>, @@ -40,7 +105,7 @@ pub enum Mod { }, } -#[derive(Debug, PartialEq)] +#[derive(Clone, Debug, PartialEq)] pub enum StmtKind { FunctionDef { name: Ident, @@ -158,7 +223,7 @@ pub enum StmtKind { } pub type Stmt = Located, U>; -#[derive(Debug, PartialEq)] +#[derive(Clone, Debug, PartialEq)] pub enum ExprKind { BoolOp { op: Boolop, @@ -275,20 +340,20 @@ pub enum ExprKind { } pub type Expr = Located, U>; -#[derive(Debug, PartialEq)] +#[derive(Clone, Debug, PartialEq)] pub enum ExprContext { Load, Store, Del, } -#[derive(Debug, PartialEq)] +#[derive(Clone, Debug, PartialEq)] pub enum Boolop { And, Or, } -#[derive(Debug, PartialEq)] +#[derive(Clone, Debug, PartialEq)] pub enum Operator { Add, Sub, @@ -305,7 +370,7 @@ pub enum Operator { FloorDiv, } -#[derive(Debug, PartialEq)] +#[derive(Clone, Debug, PartialEq)] pub enum Unaryop { Invert, Not, @@ -313,7 +378,7 @@ pub enum Unaryop { USub, } -#[derive(Debug, PartialEq)] +#[derive(Clone, Debug, PartialEq)] pub enum Cmpop { Eq, NotEq, @@ -327,7 +392,7 @@ pub enum Cmpop { NotIn, } -#[derive(Debug, PartialEq)] +#[derive(Clone, Debug, PartialEq)] pub struct Comprehension { pub target: Box>, pub iter: Box>, @@ -335,7 +400,7 @@ pub struct Comprehension { pub is_async: bool, } -#[derive(Debug, PartialEq)] +#[derive(Clone, Debug, PartialEq)] pub enum ExcepthandlerKind { ExceptHandler { type_: Option>>, @@ -345,7 +410,7 @@ pub enum ExcepthandlerKind { } pub type Excepthandler = Located, U>; -#[derive(Debug, PartialEq)] +#[derive(Clone, Debug, PartialEq)] pub struct Arguments { pub posonlyargs: Vec>, pub args: Vec>, @@ -356,7 +421,7 @@ pub struct Arguments { pub defaults: Vec>, } -#[derive(Debug, PartialEq)] +#[derive(Clone, Debug, PartialEq)] pub struct ArgData { pub arg: Ident, pub annotation: Option>>, @@ -364,30 +429,34 @@ pub struct ArgData { } pub type Arg = Located, U>; -#[derive(Debug, PartialEq)] +#[derive(Clone, Debug, PartialEq)] pub struct KeywordData { pub arg: Option, pub value: Box>, } pub type Keyword = Located, U>; -#[derive(Debug, PartialEq)] +#[derive(Clone, Debug, PartialEq)] pub struct Alias { pub name: Ident, pub asname: Option, } -#[derive(Debug, PartialEq)] +#[derive(Clone, Debug, PartialEq)] pub struct Withitem { pub context_expr: Box>, pub optional_vars: Option>>, } -#[derive(Debug, PartialEq)] +#[derive(Clone, Debug, PartialEq)] pub enum TypeIgnore { - TypeIgnore { lineno: usize, tag: String }, + TypeIgnore { + lineno: usize, + tag: String, + }, } + #[cfg(feature = "fold")] pub mod fold { use super::*; @@ -395,574 +464,633 @@ pub mod fold { pub trait Fold { type TargetU; type Error; - fn map_user(&mut self, user: U) -> Result; - fn fold_mod(&mut self, node: Mod) -> Result, Self::Error> { - fold_mod(self, node) - } - fn fold_stmt(&mut self, node: Stmt) -> Result, Self::Error> { - fold_stmt(self, node) - } - fn fold_expr(&mut self, node: Expr) -> Result, Self::Error> { - fold_expr(self, node) - } - fn fold_expr_context(&mut self, node: ExprContext) -> Result { - fold_expr_context(self, node) - } - fn fold_boolop(&mut self, node: Boolop) -> Result { - fold_boolop(self, node) - } - fn fold_operator(&mut self, node: Operator) -> Result { - fold_operator(self, node) - } - fn fold_unaryop(&mut self, node: Unaryop) -> Result { - fold_unaryop(self, node) - } - fn fold_cmpop(&mut self, node: Cmpop) -> Result { - fold_cmpop(self, node) - } - fn fold_comprehension( - &mut self, - node: Comprehension, - ) -> Result, Self::Error> { - fold_comprehension(self, node) - } - fn fold_excepthandler( - &mut self, - node: Excepthandler, - ) -> Result, Self::Error> { - fold_excepthandler(self, node) - } - fn fold_arguments( - &mut self, - node: Arguments, - ) -> Result, Self::Error> { - fold_arguments(self, node) - } - fn fold_arg(&mut self, node: Arg) -> Result, Self::Error> { - fold_arg(self, node) - } - fn fold_keyword( - &mut self, - node: Keyword, - ) -> Result, Self::Error> { - fold_keyword(self, node) - } - fn fold_alias(&mut self, node: Alias) -> Result { - fold_alias(self, node) - } - fn fold_withitem( - &mut self, - node: Withitem, - ) -> Result, Self::Error> { - fold_withitem(self, node) - } - fn fold_type_ignore(&mut self, node: TypeIgnore) -> Result { - fold_type_ignore(self, node) - } + fn map_user(&mut self, user: U) -> Result; + fn fold_mod(&mut self, node: Mod) -> Result, Self::Error> { + fold_mod(self, node) + } + fn fold_stmt(&mut self, node: Stmt) -> Result, Self::Error> { + fold_stmt(self, node) + } + fn fold_expr(&mut self, node: Expr) -> Result, Self::Error> { + fold_expr(self, node) + } + fn fold_expr_context(&mut self, node: ExprContext) -> Result { + fold_expr_context(self, node) + } + fn fold_boolop(&mut self, node: Boolop) -> Result { + fold_boolop(self, node) + } + fn fold_operator(&mut self, node: Operator) -> Result { + fold_operator(self, node) + } + fn fold_unaryop(&mut self, node: Unaryop) -> Result { + fold_unaryop(self, node) + } + fn fold_cmpop(&mut self, node: Cmpop) -> Result { + fold_cmpop(self, node) + } + fn fold_comprehension(&mut self, node: Comprehension) -> Result, Self::Error> { + fold_comprehension(self, node) + } + fn fold_excepthandler(&mut self, node: Excepthandler) -> Result, Self::Error> { + fold_excepthandler(self, node) + } + fn fold_arguments(&mut self, node: Arguments) -> Result, Self::Error> { + fold_arguments(self, node) + } + fn fold_arg(&mut self, node: Arg) -> Result, Self::Error> { + fold_arg(self, node) + } + fn fold_keyword(&mut self, node: Keyword) -> Result, Self::Error> { + fold_keyword(self, node) + } + fn fold_alias(&mut self, node: Alias) -> Result { + fold_alias(self, node) + } + fn fold_withitem(&mut self, node: Withitem) -> Result, Self::Error> { + fold_withitem(self, node) + } + fn fold_type_ignore(&mut self, node: TypeIgnore) -> Result { + fold_type_ignore(self, node) + } } - fn fold_located + ?Sized, T, MT>( - folder: &mut F, - node: Located, - f: impl FnOnce(&mut F, T) -> Result, - ) -> Result, F::Error> { - Ok(Located { - custom: folder.map_user(node.custom)?, - location: node.location, - node: f(folder, node.node)?, - }) + fn fold_located + ?Sized, T, MT>(folder: &mut F, node: Located, f: impl FnOnce(&mut F, T) -> Result) -> Result, F::Error> { + Ok(Located { custom: folder.map_user(node.custom)?, location: node.location, node: f(folder, node.node)? }) } impl Foldable for Mod { type Mapped = Mod; - fn fold + ?Sized>( - self, - folder: &mut F, - ) -> Result { + fn fold + ?Sized>(self, folder: &mut F) -> Result { folder.fold_mod(self) } } - pub fn fold_mod + ?Sized>( - #[allow(unused)] folder: &mut F, - node: Mod, - ) -> Result, F::Error> { + pub fn fold_mod + ?Sized>(#[allow(unused)] folder: &mut F, node: Mod) -> Result, F::Error> { match node { - Mod::Module { body, type_ignores } => Ok(Mod::Module { - body: Foldable::fold(body, folder)?, - type_ignores: Foldable::fold(type_ignores, folder)?, - }), - Mod::Interactive { body } => Ok(Mod::Interactive { - body: Foldable::fold(body, folder)?, - }), - Mod::Expression { body } => Ok(Mod::Expression { - body: Foldable::fold(body, folder)?, - }), - Mod::FunctionType { argtypes, returns } => Ok(Mod::FunctionType { - argtypes: Foldable::fold(argtypes, folder)?, - returns: Foldable::fold(returns, folder)?, - }), + Mod::Module { body,type_ignores } => { + Ok(Mod::Module { + body: Foldable::fold(body, folder)?, + type_ignores: Foldable::fold(type_ignores, folder)?, + }) + } + Mod::Interactive { body } => { + Ok(Mod::Interactive { + body: Foldable::fold(body, folder)?, + }) + } + Mod::Expression { body } => { + Ok(Mod::Expression { + body: Foldable::fold(body, folder)?, + }) + } + Mod::FunctionType { argtypes,returns } => { + Ok(Mod::FunctionType { + argtypes: Foldable::fold(argtypes, folder)?, + returns: Foldable::fold(returns, folder)?, + }) + } } } impl Foldable for Stmt { type Mapped = Stmt; - fn fold + ?Sized>( - self, - folder: &mut F, - ) -> Result { + fn fold + ?Sized>(self, folder: &mut F) -> Result { folder.fold_stmt(self) } } - pub fn fold_stmt + ?Sized>( - #[allow(unused)] folder: &mut F, - node: Stmt, - ) -> Result, F::Error> { - fold_located(folder, node, |folder, node| match node { - StmtKind::FunctionDef { - name, - args, - body, - decorator_list, - returns, - type_comment, - } => Ok(StmtKind::FunctionDef { - name: Foldable::fold(name, folder)?, - args: Foldable::fold(args, folder)?, - body: Foldable::fold(body, folder)?, - decorator_list: Foldable::fold(decorator_list, folder)?, - returns: Foldable::fold(returns, folder)?, - type_comment: Foldable::fold(type_comment, folder)?, - }), - StmtKind::AsyncFunctionDef { - name, - args, - body, - decorator_list, - returns, - type_comment, - } => Ok(StmtKind::AsyncFunctionDef { - name: Foldable::fold(name, folder)?, - args: Foldable::fold(args, folder)?, - body: Foldable::fold(body, folder)?, - decorator_list: Foldable::fold(decorator_list, folder)?, - returns: Foldable::fold(returns, folder)?, - type_comment: Foldable::fold(type_comment, folder)?, - }), - StmtKind::ClassDef { - name, - bases, - keywords, - body, - decorator_list, - } => Ok(StmtKind::ClassDef { - name: Foldable::fold(name, folder)?, - bases: Foldable::fold(bases, folder)?, - keywords: Foldable::fold(keywords, folder)?, - body: Foldable::fold(body, folder)?, - decorator_list: Foldable::fold(decorator_list, folder)?, - }), - StmtKind::Return { value } => Ok(StmtKind::Return { - value: Foldable::fold(value, folder)?, - }), - StmtKind::Delete { targets } => Ok(StmtKind::Delete { - targets: Foldable::fold(targets, folder)?, - }), - StmtKind::Assign { - targets, - value, - type_comment, - } => Ok(StmtKind::Assign { - targets: Foldable::fold(targets, folder)?, - value: Foldable::fold(value, folder)?, - type_comment: Foldable::fold(type_comment, folder)?, - }), - StmtKind::AugAssign { target, op, value } => Ok(StmtKind::AugAssign { - target: Foldable::fold(target, folder)?, - op: Foldable::fold(op, folder)?, - value: Foldable::fold(value, folder)?, - }), - StmtKind::AnnAssign { - target, - annotation, - value, - simple, - } => Ok(StmtKind::AnnAssign { - target: Foldable::fold(target, folder)?, - annotation: Foldable::fold(annotation, folder)?, - value: Foldable::fold(value, folder)?, - simple: Foldable::fold(simple, folder)?, - }), - StmtKind::For { - target, - iter, - body, - orelse, - type_comment, - } => Ok(StmtKind::For { - target: Foldable::fold(target, folder)?, - iter: Foldable::fold(iter, folder)?, - body: Foldable::fold(body, folder)?, - orelse: Foldable::fold(orelse, folder)?, - type_comment: Foldable::fold(type_comment, folder)?, - }), - StmtKind::AsyncFor { - target, - iter, - body, - orelse, - type_comment, - } => Ok(StmtKind::AsyncFor { - target: Foldable::fold(target, folder)?, - iter: Foldable::fold(iter, folder)?, - body: Foldable::fold(body, folder)?, - orelse: Foldable::fold(orelse, folder)?, - type_comment: Foldable::fold(type_comment, folder)?, - }), - StmtKind::While { test, body, orelse } => Ok(StmtKind::While { - test: Foldable::fold(test, folder)?, - body: Foldable::fold(body, folder)?, - orelse: Foldable::fold(orelse, folder)?, - }), - StmtKind::If { test, body, orelse } => Ok(StmtKind::If { - test: Foldable::fold(test, folder)?, - body: Foldable::fold(body, folder)?, - orelse: Foldable::fold(orelse, folder)?, - }), - StmtKind::With { - items, - body, - type_comment, - } => Ok(StmtKind::With { - items: Foldable::fold(items, folder)?, - body: Foldable::fold(body, folder)?, - type_comment: Foldable::fold(type_comment, folder)?, - }), - StmtKind::AsyncWith { - items, - body, - type_comment, - } => Ok(StmtKind::AsyncWith { - items: Foldable::fold(items, folder)?, - body: Foldable::fold(body, folder)?, - type_comment: Foldable::fold(type_comment, folder)?, - }), - StmtKind::Raise { exc, cause } => Ok(StmtKind::Raise { - exc: Foldable::fold(exc, folder)?, - cause: Foldable::fold(cause, folder)?, - }), - StmtKind::Try { - body, - handlers, - orelse, - finalbody, - } => Ok(StmtKind::Try { - body: Foldable::fold(body, folder)?, - handlers: Foldable::fold(handlers, folder)?, - orelse: Foldable::fold(orelse, folder)?, - finalbody: Foldable::fold(finalbody, folder)?, - }), - StmtKind::Assert { test, msg } => Ok(StmtKind::Assert { - test: Foldable::fold(test, folder)?, - msg: Foldable::fold(msg, folder)?, - }), - StmtKind::Import { names } => Ok(StmtKind::Import { - names: Foldable::fold(names, folder)?, - }), - StmtKind::ImportFrom { - module, - names, - level, - } => Ok(StmtKind::ImportFrom { - module: Foldable::fold(module, folder)?, - names: Foldable::fold(names, folder)?, - level: Foldable::fold(level, folder)?, - }), - StmtKind::Global { names } => Ok(StmtKind::Global { - names: Foldable::fold(names, folder)?, - }), - StmtKind::Nonlocal { names } => Ok(StmtKind::Nonlocal { - names: Foldable::fold(names, folder)?, - }), - StmtKind::Expr { value } => Ok(StmtKind::Expr { - value: Foldable::fold(value, folder)?, - }), - StmtKind::Pass {} => Ok(StmtKind::Pass {}), - StmtKind::Break {} => Ok(StmtKind::Break {}), - StmtKind::Continue {} => Ok(StmtKind::Continue {}), - }) + pub fn fold_stmt + ?Sized>(#[allow(unused)] folder: &mut F, node: Stmt) -> Result, F::Error> { + fold_located(folder, node, |folder, node| { + match node { + StmtKind::FunctionDef { name,args,body,decorator_list,returns,type_comment } => { + Ok(StmtKind::FunctionDef { + name: Foldable::fold(name, folder)?, + args: Foldable::fold(args, folder)?, + body: Foldable::fold(body, folder)?, + decorator_list: Foldable::fold(decorator_list, folder)?, + returns: Foldable::fold(returns, folder)?, + type_comment: Foldable::fold(type_comment, folder)?, + }) + } + StmtKind::AsyncFunctionDef { name,args,body,decorator_list,returns,type_comment } => { + Ok(StmtKind::AsyncFunctionDef { + name: Foldable::fold(name, folder)?, + args: Foldable::fold(args, folder)?, + body: Foldable::fold(body, folder)?, + decorator_list: Foldable::fold(decorator_list, folder)?, + returns: Foldable::fold(returns, folder)?, + type_comment: Foldable::fold(type_comment, folder)?, + }) + } + StmtKind::ClassDef { name,bases,keywords,body,decorator_list } => { + Ok(StmtKind::ClassDef { + name: Foldable::fold(name, folder)?, + bases: Foldable::fold(bases, folder)?, + keywords: Foldable::fold(keywords, folder)?, + body: Foldable::fold(body, folder)?, + decorator_list: Foldable::fold(decorator_list, folder)?, + }) + } + StmtKind::Return { value } => { + Ok(StmtKind::Return { + value: Foldable::fold(value, folder)?, + }) + } + StmtKind::Delete { targets } => { + Ok(StmtKind::Delete { + targets: Foldable::fold(targets, folder)?, + }) + } + StmtKind::Assign { targets,value,type_comment } => { + Ok(StmtKind::Assign { + targets: Foldable::fold(targets, folder)?, + value: Foldable::fold(value, folder)?, + type_comment: Foldable::fold(type_comment, folder)?, + }) + } + StmtKind::AugAssign { target,op,value } => { + Ok(StmtKind::AugAssign { + target: Foldable::fold(target, folder)?, + op: Foldable::fold(op, folder)?, + value: Foldable::fold(value, folder)?, + }) + } + StmtKind::AnnAssign { target,annotation,value,simple } => { + Ok(StmtKind::AnnAssign { + target: Foldable::fold(target, folder)?, + annotation: Foldable::fold(annotation, folder)?, + value: Foldable::fold(value, folder)?, + simple: Foldable::fold(simple, folder)?, + }) + } + StmtKind::For { target,iter,body,orelse,type_comment } => { + Ok(StmtKind::For { + target: Foldable::fold(target, folder)?, + iter: Foldable::fold(iter, folder)?, + body: Foldable::fold(body, folder)?, + orelse: Foldable::fold(orelse, folder)?, + type_comment: Foldable::fold(type_comment, folder)?, + }) + } + StmtKind::AsyncFor { target,iter,body,orelse,type_comment } => { + Ok(StmtKind::AsyncFor { + target: Foldable::fold(target, folder)?, + iter: Foldable::fold(iter, folder)?, + body: Foldable::fold(body, folder)?, + orelse: Foldable::fold(orelse, folder)?, + type_comment: Foldable::fold(type_comment, folder)?, + }) + } + StmtKind::While { test,body,orelse } => { + Ok(StmtKind::While { + test: Foldable::fold(test, folder)?, + body: Foldable::fold(body, folder)?, + orelse: Foldable::fold(orelse, folder)?, + }) + } + StmtKind::If { test,body,orelse } => { + Ok(StmtKind::If { + test: Foldable::fold(test, folder)?, + body: Foldable::fold(body, folder)?, + orelse: Foldable::fold(orelse, folder)?, + }) + } + StmtKind::With { items,body,type_comment } => { + Ok(StmtKind::With { + items: Foldable::fold(items, folder)?, + body: Foldable::fold(body, folder)?, + type_comment: Foldable::fold(type_comment, folder)?, + }) + } + StmtKind::AsyncWith { items,body,type_comment } => { + Ok(StmtKind::AsyncWith { + items: Foldable::fold(items, folder)?, + body: Foldable::fold(body, folder)?, + type_comment: Foldable::fold(type_comment, folder)?, + }) + } + StmtKind::Raise { exc,cause } => { + Ok(StmtKind::Raise { + exc: Foldable::fold(exc, folder)?, + cause: Foldable::fold(cause, folder)?, + }) + } + StmtKind::Try { body,handlers,orelse,finalbody } => { + Ok(StmtKind::Try { + body: Foldable::fold(body, folder)?, + handlers: Foldable::fold(handlers, folder)?, + orelse: Foldable::fold(orelse, folder)?, + finalbody: Foldable::fold(finalbody, folder)?, + }) + } + StmtKind::Assert { test,msg } => { + Ok(StmtKind::Assert { + test: Foldable::fold(test, folder)?, + msg: Foldable::fold(msg, folder)?, + }) + } + StmtKind::Import { names } => { + Ok(StmtKind::Import { + names: Foldable::fold(names, folder)?, + }) + } + StmtKind::ImportFrom { module,names,level } => { + Ok(StmtKind::ImportFrom { + module: Foldable::fold(module, folder)?, + names: Foldable::fold(names, folder)?, + level: Foldable::fold(level, folder)?, + }) + } + StmtKind::Global { names } => { + Ok(StmtKind::Global { + names: Foldable::fold(names, folder)?, + }) + } + StmtKind::Nonlocal { names } => { + Ok(StmtKind::Nonlocal { + names: Foldable::fold(names, folder)?, + }) + } + StmtKind::Expr { value } => { + Ok(StmtKind::Expr { + value: Foldable::fold(value, folder)?, + }) + } + StmtKind::Pass { } => { + Ok(StmtKind::Pass { + }) + } + StmtKind::Break { } => { + Ok(StmtKind::Break { + }) + } + StmtKind::Continue { } => { + Ok(StmtKind::Continue { + }) + } + } + }) } impl Foldable for Expr { type Mapped = Expr; - fn fold + ?Sized>( - self, - folder: &mut F, - ) -> Result { + fn fold + ?Sized>(self, folder: &mut F) -> Result { folder.fold_expr(self) } } - pub fn fold_expr + ?Sized>( - #[allow(unused)] folder: &mut F, - node: Expr, - ) -> Result, F::Error> { - fold_located(folder, node, |folder, node| match node { - ExprKind::BoolOp { op, values } => Ok(ExprKind::BoolOp { - op: Foldable::fold(op, folder)?, - values: Foldable::fold(values, folder)?, - }), - ExprKind::NamedExpr { target, value } => Ok(ExprKind::NamedExpr { - target: Foldable::fold(target, folder)?, - value: Foldable::fold(value, folder)?, - }), - ExprKind::BinOp { left, op, right } => Ok(ExprKind::BinOp { - left: Foldable::fold(left, folder)?, - op: Foldable::fold(op, folder)?, - right: Foldable::fold(right, folder)?, - }), - ExprKind::UnaryOp { op, operand } => Ok(ExprKind::UnaryOp { - op: Foldable::fold(op, folder)?, - operand: Foldable::fold(operand, folder)?, - }), - ExprKind::Lambda { args, body } => Ok(ExprKind::Lambda { - args: Foldable::fold(args, folder)?, - body: Foldable::fold(body, folder)?, - }), - ExprKind::IfExp { test, body, orelse } => Ok(ExprKind::IfExp { - test: Foldable::fold(test, folder)?, - body: Foldable::fold(body, folder)?, - orelse: Foldable::fold(orelse, folder)?, - }), - ExprKind::Dict { keys, values } => Ok(ExprKind::Dict { - keys: Foldable::fold(keys, folder)?, - values: Foldable::fold(values, folder)?, - }), - ExprKind::Set { elts } => Ok(ExprKind::Set { - elts: Foldable::fold(elts, folder)?, - }), - ExprKind::ListComp { elt, generators } => Ok(ExprKind::ListComp { - elt: Foldable::fold(elt, folder)?, - generators: Foldable::fold(generators, folder)?, - }), - ExprKind::SetComp { elt, generators } => Ok(ExprKind::SetComp { - elt: Foldable::fold(elt, folder)?, - generators: Foldable::fold(generators, folder)?, - }), - ExprKind::DictComp { - key, - value, - generators, - } => Ok(ExprKind::DictComp { - key: Foldable::fold(key, folder)?, - value: Foldable::fold(value, folder)?, - generators: Foldable::fold(generators, folder)?, - }), - ExprKind::GeneratorExp { elt, generators } => Ok(ExprKind::GeneratorExp { - elt: Foldable::fold(elt, folder)?, - generators: Foldable::fold(generators, folder)?, - }), - ExprKind::Await { value } => Ok(ExprKind::Await { - value: Foldable::fold(value, folder)?, - }), - ExprKind::Yield { value } => Ok(ExprKind::Yield { - value: Foldable::fold(value, folder)?, - }), - ExprKind::YieldFrom { value } => Ok(ExprKind::YieldFrom { - value: Foldable::fold(value, folder)?, - }), - ExprKind::Compare { - left, - ops, - comparators, - } => Ok(ExprKind::Compare { - left: Foldable::fold(left, folder)?, - ops: Foldable::fold(ops, folder)?, - comparators: Foldable::fold(comparators, folder)?, - }), - ExprKind::Call { - func, - args, - keywords, - } => Ok(ExprKind::Call { - func: Foldable::fold(func, folder)?, - args: Foldable::fold(args, folder)?, - keywords: Foldable::fold(keywords, folder)?, - }), - ExprKind::FormattedValue { - value, - conversion, - format_spec, - } => Ok(ExprKind::FormattedValue { - value: Foldable::fold(value, folder)?, - conversion: Foldable::fold(conversion, folder)?, - format_spec: Foldable::fold(format_spec, folder)?, - }), - ExprKind::JoinedStr { values } => Ok(ExprKind::JoinedStr { - values: Foldable::fold(values, folder)?, - }), - ExprKind::Constant { value, kind } => Ok(ExprKind::Constant { - value: Foldable::fold(value, folder)?, - kind: Foldable::fold(kind, folder)?, - }), - ExprKind::Attribute { value, attr, ctx } => Ok(ExprKind::Attribute { - value: Foldable::fold(value, folder)?, - attr: Foldable::fold(attr, folder)?, - ctx: Foldable::fold(ctx, folder)?, - }), - ExprKind::Subscript { value, slice, ctx } => Ok(ExprKind::Subscript { - value: Foldable::fold(value, folder)?, - slice: Foldable::fold(slice, folder)?, - ctx: Foldable::fold(ctx, folder)?, - }), - ExprKind::Starred { value, ctx } => Ok(ExprKind::Starred { - value: Foldable::fold(value, folder)?, - ctx: Foldable::fold(ctx, folder)?, - }), - ExprKind::Name { id, ctx } => Ok(ExprKind::Name { - id: Foldable::fold(id, folder)?, - ctx: Foldable::fold(ctx, folder)?, - }), - ExprKind::List { elts, ctx } => Ok(ExprKind::List { - elts: Foldable::fold(elts, folder)?, - ctx: Foldable::fold(ctx, folder)?, - }), - ExprKind::Tuple { elts, ctx } => Ok(ExprKind::Tuple { - elts: Foldable::fold(elts, folder)?, - ctx: Foldable::fold(ctx, folder)?, - }), - ExprKind::Slice { lower, upper, step } => Ok(ExprKind::Slice { - lower: Foldable::fold(lower, folder)?, - upper: Foldable::fold(upper, folder)?, - step: Foldable::fold(step, folder)?, - }), - }) + pub fn fold_expr + ?Sized>(#[allow(unused)] folder: &mut F, node: Expr) -> Result, F::Error> { + fold_located(folder, node, |folder, node| { + match node { + ExprKind::BoolOp { op,values } => { + Ok(ExprKind::BoolOp { + op: Foldable::fold(op, folder)?, + values: Foldable::fold(values, folder)?, + }) + } + ExprKind::NamedExpr { target,value } => { + Ok(ExprKind::NamedExpr { + target: Foldable::fold(target, folder)?, + value: Foldable::fold(value, folder)?, + }) + } + ExprKind::BinOp { left,op,right } => { + Ok(ExprKind::BinOp { + left: Foldable::fold(left, folder)?, + op: Foldable::fold(op, folder)?, + right: Foldable::fold(right, folder)?, + }) + } + ExprKind::UnaryOp { op,operand } => { + Ok(ExprKind::UnaryOp { + op: Foldable::fold(op, folder)?, + operand: Foldable::fold(operand, folder)?, + }) + } + ExprKind::Lambda { args,body } => { + Ok(ExprKind::Lambda { + args: Foldable::fold(args, folder)?, + body: Foldable::fold(body, folder)?, + }) + } + ExprKind::IfExp { test,body,orelse } => { + Ok(ExprKind::IfExp { + test: Foldable::fold(test, folder)?, + body: Foldable::fold(body, folder)?, + orelse: Foldable::fold(orelse, folder)?, + }) + } + ExprKind::Dict { keys,values } => { + Ok(ExprKind::Dict { + keys: Foldable::fold(keys, folder)?, + values: Foldable::fold(values, folder)?, + }) + } + ExprKind::Set { elts } => { + Ok(ExprKind::Set { + elts: Foldable::fold(elts, folder)?, + }) + } + ExprKind::ListComp { elt,generators } => { + Ok(ExprKind::ListComp { + elt: Foldable::fold(elt, folder)?, + generators: Foldable::fold(generators, folder)?, + }) + } + ExprKind::SetComp { elt,generators } => { + Ok(ExprKind::SetComp { + elt: Foldable::fold(elt, folder)?, + generators: Foldable::fold(generators, folder)?, + }) + } + ExprKind::DictComp { key,value,generators } => { + Ok(ExprKind::DictComp { + key: Foldable::fold(key, folder)?, + value: Foldable::fold(value, folder)?, + generators: Foldable::fold(generators, folder)?, + }) + } + ExprKind::GeneratorExp { elt,generators } => { + Ok(ExprKind::GeneratorExp { + elt: Foldable::fold(elt, folder)?, + generators: Foldable::fold(generators, folder)?, + }) + } + ExprKind::Await { value } => { + Ok(ExprKind::Await { + value: Foldable::fold(value, folder)?, + }) + } + ExprKind::Yield { value } => { + Ok(ExprKind::Yield { + value: Foldable::fold(value, folder)?, + }) + } + ExprKind::YieldFrom { value } => { + Ok(ExprKind::YieldFrom { + value: Foldable::fold(value, folder)?, + }) + } + ExprKind::Compare { left,ops,comparators } => { + Ok(ExprKind::Compare { + left: Foldable::fold(left, folder)?, + ops: Foldable::fold(ops, folder)?, + comparators: Foldable::fold(comparators, folder)?, + }) + } + ExprKind::Call { func,args,keywords } => { + Ok(ExprKind::Call { + func: Foldable::fold(func, folder)?, + args: Foldable::fold(args, folder)?, + keywords: Foldable::fold(keywords, folder)?, + }) + } + ExprKind::FormattedValue { value,conversion,format_spec } => { + Ok(ExprKind::FormattedValue { + value: Foldable::fold(value, folder)?, + conversion: Foldable::fold(conversion, folder)?, + format_spec: Foldable::fold(format_spec, folder)?, + }) + } + ExprKind::JoinedStr { values } => { + Ok(ExprKind::JoinedStr { + values: Foldable::fold(values, folder)?, + }) + } + ExprKind::Constant { value,kind } => { + Ok(ExprKind::Constant { + value: Foldable::fold(value, folder)?, + kind: Foldable::fold(kind, folder)?, + }) + } + ExprKind::Attribute { value,attr,ctx } => { + Ok(ExprKind::Attribute { + value: Foldable::fold(value, folder)?, + attr: Foldable::fold(attr, folder)?, + ctx: Foldable::fold(ctx, folder)?, + }) + } + ExprKind::Subscript { value,slice,ctx } => { + Ok(ExprKind::Subscript { + value: Foldable::fold(value, folder)?, + slice: Foldable::fold(slice, folder)?, + ctx: Foldable::fold(ctx, folder)?, + }) + } + ExprKind::Starred { value,ctx } => { + Ok(ExprKind::Starred { + value: Foldable::fold(value, folder)?, + ctx: Foldable::fold(ctx, folder)?, + }) + } + ExprKind::Name { id,ctx } => { + Ok(ExprKind::Name { + id: Foldable::fold(id, folder)?, + ctx: Foldable::fold(ctx, folder)?, + }) + } + ExprKind::List { elts,ctx } => { + Ok(ExprKind::List { + elts: Foldable::fold(elts, folder)?, + ctx: Foldable::fold(ctx, folder)?, + }) + } + ExprKind::Tuple { elts,ctx } => { + Ok(ExprKind::Tuple { + elts: Foldable::fold(elts, folder)?, + ctx: Foldable::fold(ctx, folder)?, + }) + } + ExprKind::Slice { lower,upper,step } => { + Ok(ExprKind::Slice { + lower: Foldable::fold(lower, folder)?, + upper: Foldable::fold(upper, folder)?, + step: Foldable::fold(step, folder)?, + }) + } + } + }) } impl Foldable for ExprContext { type Mapped = ExprContext; - fn fold + ?Sized>( - self, - folder: &mut F, - ) -> Result { + fn fold + ?Sized>(self, folder: &mut F) -> Result { folder.fold_expr_context(self) } } - pub fn fold_expr_context + ?Sized>( - #[allow(unused)] folder: &mut F, - node: ExprContext, - ) -> Result { + pub fn fold_expr_context + ?Sized>(#[allow(unused)] folder: &mut F, node: ExprContext) -> Result { match node { - ExprContext::Load {} => Ok(ExprContext::Load {}), - ExprContext::Store {} => Ok(ExprContext::Store {}), - ExprContext::Del {} => Ok(ExprContext::Del {}), + ExprContext::Load { } => { + Ok(ExprContext::Load { + }) + } + ExprContext::Store { } => { + Ok(ExprContext::Store { + }) + } + ExprContext::Del { } => { + Ok(ExprContext::Del { + }) + } } } impl Foldable for Boolop { type Mapped = Boolop; - fn fold + ?Sized>( - self, - folder: &mut F, - ) -> Result { + fn fold + ?Sized>(self, folder: &mut F) -> Result { folder.fold_boolop(self) } } - pub fn fold_boolop + ?Sized>( - #[allow(unused)] folder: &mut F, - node: Boolop, - ) -> Result { + pub fn fold_boolop + ?Sized>(#[allow(unused)] folder: &mut F, node: Boolop) -> Result { match node { - Boolop::And {} => Ok(Boolop::And {}), - Boolop::Or {} => Ok(Boolop::Or {}), + Boolop::And { } => { + Ok(Boolop::And { + }) + } + Boolop::Or { } => { + Ok(Boolop::Or { + }) + } } } impl Foldable for Operator { type Mapped = Operator; - fn fold + ?Sized>( - self, - folder: &mut F, - ) -> Result { + fn fold + ?Sized>(self, folder: &mut F) -> Result { folder.fold_operator(self) } } - pub fn fold_operator + ?Sized>( - #[allow(unused)] folder: &mut F, - node: Operator, - ) -> Result { + pub fn fold_operator + ?Sized>(#[allow(unused)] folder: &mut F, node: Operator) -> Result { match node { - Operator::Add {} => Ok(Operator::Add {}), - Operator::Sub {} => Ok(Operator::Sub {}), - Operator::Mult {} => Ok(Operator::Mult {}), - Operator::MatMult {} => Ok(Operator::MatMult {}), - Operator::Div {} => Ok(Operator::Div {}), - Operator::Mod {} => Ok(Operator::Mod {}), - Operator::Pow {} => Ok(Operator::Pow {}), - Operator::LShift {} => Ok(Operator::LShift {}), - Operator::RShift {} => Ok(Operator::RShift {}), - Operator::BitOr {} => Ok(Operator::BitOr {}), - Operator::BitXor {} => Ok(Operator::BitXor {}), - Operator::BitAnd {} => Ok(Operator::BitAnd {}), - Operator::FloorDiv {} => Ok(Operator::FloorDiv {}), + Operator::Add { } => { + Ok(Operator::Add { + }) + } + Operator::Sub { } => { + Ok(Operator::Sub { + }) + } + Operator::Mult { } => { + Ok(Operator::Mult { + }) + } + Operator::MatMult { } => { + Ok(Operator::MatMult { + }) + } + Operator::Div { } => { + Ok(Operator::Div { + }) + } + Operator::Mod { } => { + Ok(Operator::Mod { + }) + } + Operator::Pow { } => { + Ok(Operator::Pow { + }) + } + Operator::LShift { } => { + Ok(Operator::LShift { + }) + } + Operator::RShift { } => { + Ok(Operator::RShift { + }) + } + Operator::BitOr { } => { + Ok(Operator::BitOr { + }) + } + Operator::BitXor { } => { + Ok(Operator::BitXor { + }) + } + Operator::BitAnd { } => { + Ok(Operator::BitAnd { + }) + } + Operator::FloorDiv { } => { + Ok(Operator::FloorDiv { + }) + } } } impl Foldable for Unaryop { type Mapped = Unaryop; - fn fold + ?Sized>( - self, - folder: &mut F, - ) -> Result { + fn fold + ?Sized>(self, folder: &mut F) -> Result { folder.fold_unaryop(self) } } - pub fn fold_unaryop + ?Sized>( - #[allow(unused)] folder: &mut F, - node: Unaryop, - ) -> Result { + pub fn fold_unaryop + ?Sized>(#[allow(unused)] folder: &mut F, node: Unaryop) -> Result { match node { - Unaryop::Invert {} => Ok(Unaryop::Invert {}), - Unaryop::Not {} => Ok(Unaryop::Not {}), - Unaryop::UAdd {} => Ok(Unaryop::UAdd {}), - Unaryop::USub {} => Ok(Unaryop::USub {}), + Unaryop::Invert { } => { + Ok(Unaryop::Invert { + }) + } + Unaryop::Not { } => { + Ok(Unaryop::Not { + }) + } + Unaryop::UAdd { } => { + Ok(Unaryop::UAdd { + }) + } + Unaryop::USub { } => { + Ok(Unaryop::USub { + }) + } } } impl Foldable for Cmpop { type Mapped = Cmpop; - fn fold + ?Sized>( - self, - folder: &mut F, - ) -> Result { + fn fold + ?Sized>(self, folder: &mut F) -> Result { folder.fold_cmpop(self) } } - pub fn fold_cmpop + ?Sized>( - #[allow(unused)] folder: &mut F, - node: Cmpop, - ) -> Result { + pub fn fold_cmpop + ?Sized>(#[allow(unused)] folder: &mut F, node: Cmpop) -> Result { match node { - Cmpop::Eq {} => Ok(Cmpop::Eq {}), - Cmpop::NotEq {} => Ok(Cmpop::NotEq {}), - Cmpop::Lt {} => Ok(Cmpop::Lt {}), - Cmpop::LtE {} => Ok(Cmpop::LtE {}), - Cmpop::Gt {} => Ok(Cmpop::Gt {}), - Cmpop::GtE {} => Ok(Cmpop::GtE {}), - Cmpop::Is {} => Ok(Cmpop::Is {}), - Cmpop::IsNot {} => Ok(Cmpop::IsNot {}), - Cmpop::In {} => Ok(Cmpop::In {}), - Cmpop::NotIn {} => Ok(Cmpop::NotIn {}), + Cmpop::Eq { } => { + Ok(Cmpop::Eq { + }) + } + Cmpop::NotEq { } => { + Ok(Cmpop::NotEq { + }) + } + Cmpop::Lt { } => { + Ok(Cmpop::Lt { + }) + } + Cmpop::LtE { } => { + Ok(Cmpop::LtE { + }) + } + Cmpop::Gt { } => { + Ok(Cmpop::Gt { + }) + } + Cmpop::GtE { } => { + Ok(Cmpop::GtE { + }) + } + Cmpop::Is { } => { + Ok(Cmpop::Is { + }) + } + Cmpop::IsNot { } => { + Ok(Cmpop::IsNot { + }) + } + Cmpop::In { } => { + Ok(Cmpop::In { + }) + } + Cmpop::NotIn { } => { + Ok(Cmpop::NotIn { + }) + } } } impl Foldable for Comprehension { type Mapped = Comprehension; - fn fold + ?Sized>( - self, - folder: &mut F, - ) -> Result { + fn fold + ?Sized>(self, folder: &mut F) -> Result { folder.fold_comprehension(self) } } - pub fn fold_comprehension + ?Sized>( - #[allow(unused)] folder: &mut F, - node: Comprehension, - ) -> Result, F::Error> { - let Comprehension { - target, - iter, - ifs, - is_async, - } = node; + pub fn fold_comprehension + ?Sized>(#[allow(unused)] folder: &mut F, node: Comprehension) -> Result, F::Error> { + let Comprehension { target,iter,ifs,is_async } = node; Ok(Comprehension { target: Foldable::fold(target, folder)?, iter: Foldable::fold(iter, folder)?, @@ -972,49 +1100,31 @@ pub mod fold { } impl Foldable for Excepthandler { type Mapped = Excepthandler; - fn fold + ?Sized>( - self, - folder: &mut F, - ) -> Result { + fn fold + ?Sized>(self, folder: &mut F) -> Result { folder.fold_excepthandler(self) } } - pub fn fold_excepthandler + ?Sized>( - #[allow(unused)] folder: &mut F, - node: Excepthandler, - ) -> Result, F::Error> { - fold_located(folder, node, |folder, node| match node { - ExcepthandlerKind::ExceptHandler { type_, name, body } => { + pub fn fold_excepthandler + ?Sized>(#[allow(unused)] folder: &mut F, node: Excepthandler) -> Result, F::Error> { + fold_located(folder, node, |folder, node| { + match node { + ExcepthandlerKind::ExceptHandler { type_,name,body } => { Ok(ExcepthandlerKind::ExceptHandler { type_: Foldable::fold(type_, folder)?, name: Foldable::fold(name, folder)?, body: Foldable::fold(body, folder)?, }) } - }) + } + }) } impl Foldable for Arguments { type Mapped = Arguments; - fn fold + ?Sized>( - self, - folder: &mut F, - ) -> Result { + fn fold + ?Sized>(self, folder: &mut F) -> Result { folder.fold_arguments(self) } } - pub fn fold_arguments + ?Sized>( - #[allow(unused)] folder: &mut F, - node: Arguments, - ) -> Result, F::Error> { - let Arguments { - posonlyargs, - args, - vararg, - kwonlyargs, - kw_defaults, - kwarg, - defaults, - } = node; + pub fn fold_arguments + ?Sized>(#[allow(unused)] folder: &mut F, node: Arguments) -> Result, F::Error> { + let Arguments { posonlyargs,args,vararg,kwonlyargs,kw_defaults,kwarg,defaults } = node; Ok(Arguments { posonlyargs: Foldable::fold(posonlyargs, folder)?, args: Foldable::fold(args, folder)?, @@ -1027,65 +1137,43 @@ pub mod fold { } impl Foldable for Arg { type Mapped = Arg; - fn fold + ?Sized>( - self, - folder: &mut F, - ) -> Result { + fn fold + ?Sized>(self, folder: &mut F) -> Result { folder.fold_arg(self) } } - pub fn fold_arg + ?Sized>( - #[allow(unused)] folder: &mut F, - node: Arg, - ) -> Result, F::Error> { - fold_located(folder, node, |folder, node| { - let ArgData { - arg, - annotation, - type_comment, - } = node; - Ok(ArgData { - arg: Foldable::fold(arg, folder)?, - annotation: Foldable::fold(annotation, folder)?, - type_comment: Foldable::fold(type_comment, folder)?, - }) + pub fn fold_arg + ?Sized>(#[allow(unused)] folder: &mut F, node: Arg) -> Result, F::Error> { + fold_located(folder, node, |folder, node| { + let ArgData { arg,annotation,type_comment } = node; + Ok(ArgData { + arg: Foldable::fold(arg, folder)?, + annotation: Foldable::fold(annotation, folder)?, + type_comment: Foldable::fold(type_comment, folder)?, }) + }) } impl Foldable for Keyword { type Mapped = Keyword; - fn fold + ?Sized>( - self, - folder: &mut F, - ) -> Result { + fn fold + ?Sized>(self, folder: &mut F) -> Result { folder.fold_keyword(self) } } - pub fn fold_keyword + ?Sized>( - #[allow(unused)] folder: &mut F, - node: Keyword, - ) -> Result, F::Error> { - fold_located(folder, node, |folder, node| { - let KeywordData { arg, value } = node; - Ok(KeywordData { - arg: Foldable::fold(arg, folder)?, - value: Foldable::fold(value, folder)?, - }) + pub fn fold_keyword + ?Sized>(#[allow(unused)] folder: &mut F, node: Keyword) -> Result, F::Error> { + fold_located(folder, node, |folder, node| { + let KeywordData { arg,value } = node; + Ok(KeywordData { + arg: Foldable::fold(arg, folder)?, + value: Foldable::fold(value, folder)?, }) + }) } impl Foldable for Alias { type Mapped = Alias; - fn fold + ?Sized>( - self, - folder: &mut F, - ) -> Result { + fn fold + ?Sized>(self, folder: &mut F) -> Result { folder.fold_alias(self) } } - pub fn fold_alias + ?Sized>( - #[allow(unused)] folder: &mut F, - node: Alias, - ) -> Result { - let Alias { name, asname } = node; + pub fn fold_alias + ?Sized>(#[allow(unused)] folder: &mut F, node: Alias) -> Result { + let Alias { name,asname } = node; Ok(Alias { name: Foldable::fold(name, folder)?, asname: Foldable::fold(asname, folder)?, @@ -1093,21 +1181,12 @@ pub mod fold { } impl Foldable for Withitem { type Mapped = Withitem; - fn fold + ?Sized>( - self, - folder: &mut F, - ) -> Result { + fn fold + ?Sized>(self, folder: &mut F) -> Result { folder.fold_withitem(self) } } - pub fn fold_withitem + ?Sized>( - #[allow(unused)] folder: &mut F, - node: Withitem, - ) -> Result, F::Error> { - let Withitem { - context_expr, - optional_vars, - } = node; + pub fn fold_withitem + ?Sized>(#[allow(unused)] folder: &mut F, node: Withitem) -> Result, F::Error> { + let Withitem { context_expr,optional_vars } = node; Ok(Withitem { context_expr: Foldable::fold(context_expr, folder)?, optional_vars: Foldable::fold(optional_vars, folder)?, @@ -1115,22 +1194,19 @@ pub mod fold { } impl Foldable for TypeIgnore { type Mapped = TypeIgnore; - fn fold + ?Sized>( - self, - folder: &mut F, - ) -> Result { + fn fold + ?Sized>(self, folder: &mut F) -> Result { folder.fold_type_ignore(self) } } - pub fn fold_type_ignore + ?Sized>( - #[allow(unused)] folder: &mut F, - node: TypeIgnore, - ) -> Result { + pub fn fold_type_ignore + ?Sized>(#[allow(unused)] folder: &mut F, node: TypeIgnore) -> Result { match node { - TypeIgnore::TypeIgnore { lineno, tag } => Ok(TypeIgnore::TypeIgnore { - lineno: Foldable::fold(lineno, folder)?, - tag: Foldable::fold(tag, folder)?, - }), + TypeIgnore::TypeIgnore { lineno,tag } => { + Ok(TypeIgnore::TypeIgnore { + lineno: Foldable::fold(lineno, folder)?, + tag: Foldable::fold(tag, folder)?, + }) + } } } } + diff --git a/nac3ast/src/constant.rs b/nac3ast/src/constant.rs index 0a627783f..3e5ed8d72 100644 --- a/nac3ast/src/constant.rs +++ b/nac3ast/src/constant.rs @@ -1,6 +1,6 @@ use num_bigint::BigInt; -#[derive(Debug, PartialEq)] +#[derive(Clone, Debug, PartialEq)] pub enum Constant { None, Bool(bool), diff --git a/nac3ast/src/fold_helpers.rs b/nac3ast/src/fold_helpers.rs index 26a08b2fb..5ff83d0b1 100644 --- a/nac3ast/src/fold_helpers.rs +++ b/nac3ast/src/fold_helpers.rs @@ -1,5 +1,6 @@ use crate::constant; use crate::fold::Fold; +use crate::StrRef; pub(crate) trait Foldable { type Mapped; @@ -67,6 +68,7 @@ simple_fold!( usize, String, bool, + StrRef, constant::Constant, constant::ConversionFlag ); diff --git a/nac3ast/src/lib.rs b/nac3ast/src/lib.rs index 83e862315..10abbe014 100644 --- a/nac3ast/src/lib.rs +++ b/nac3ast/src/lib.rs @@ -1,3 +1,6 @@ +#[macro_use] +extern crate lazy_static; + mod ast_gen; mod constant; #[cfg(feature = "fold")] diff --git a/nac3parser/src/function.rs b/nac3parser/src/function.rs index 68d890f20..a6fa07e0e 100644 --- a/nac3parser/src/function.rs +++ b/nac3parser/src/function.rs @@ -70,7 +70,7 @@ pub fn parse_args(func_args: Vec) -> Result bool { - c == '_' || is_xid_start(c) + match c { + '_' | 'a'..='z' | 'A'..='Z' => true, + '+' | '-' | '*' | '/' | '=' | ' ' | '<' | '>' => false, + c => is_xid_start(c), + } } fn is_identifier_continuation(&self) -> bool { if let Some(c) = self.chr0 { match c { - '_' | '0'..='9' => true, + '_' | '0'..='9' | 'a'..='z' | 'A'..='Z' => true, + '+' | '-' | '*' | '/' | '=' | ' ' | '<' | '>' => false, c => is_xid_continue(c), } } else { @@ -784,9 +789,7 @@ where let tok_end = self.get_pos(); self.emit(( tok_start, - Tok::Name { - name: c.to_string(), - }, + Tok::Name { name: c.to_string().into() }, tok_end, )); } else { @@ -1402,7 +1405,7 @@ mod tests { tokens, vec![ Tok::Name { - name: String::from("avariable"), + name: String::from("avariable").into(), }, Tok::Equal, Tok::Int { @@ -1433,7 +1436,7 @@ mod tests { vec![ Tok::Def, Tok::Name { - name: String::from("foo"), + name: String::from("foo").into(), }, Tok::Lpar, Tok::Rpar, @@ -1469,7 +1472,7 @@ mod tests { vec![ Tok::Def, Tok::Name { - name: String::from("foo"), + name: String::from("foo").into(), }, Tok::Lpar, Tok::Rpar, @@ -1478,7 +1481,7 @@ mod tests { Tok::Indent, Tok::If, Tok::Name { - name: String::from("x"), + name: String::from("x").into(), }, Tok::Colon, Tok::Newline, @@ -1507,7 +1510,7 @@ mod tests { vec![ Tok::Def, Tok::Name { - name: String::from("foo"), + name: String::from("foo").into(), }, Tok::Lpar, Tok::Rpar, @@ -1516,7 +1519,7 @@ mod tests { Tok::Indent, Tok::If, Tok::Name { - name: String::from("x"), + name: String::from("x").into(), }, Tok::Colon, Tok::Newline, @@ -1556,7 +1559,7 @@ mod tests { tokens, vec![ Tok::Name { - name: String::from("x"), + name: String::from("x").into(), }, Tok::Equal, Tok::Lsqb, diff --git a/nac3parser/src/python.lalrpop b/nac3parser/src/python.lalrpop index c76f26904..7a9947900 100644 --- a/nac3parser/src/python.lalrpop +++ b/nac3parser/src/python.lalrpop @@ -246,7 +246,7 @@ ImportStatement: ast::Stmt = { location, node: ast::StmtKind::ImportFrom { level, - module, + module: module.map(|s| s.into()), names }, } @@ -272,24 +272,25 @@ ImportAsNames: Vec = { "(" >> ","? ")" => i, "*" => { // Star import all - vec![ast::Alias { name: "*".to_string(), asname: None }] + vec![ast::Alias { name: "*".into(), asname: None }] }, }; #[inline] ImportAsAlias: ast::Alias = { - => ast::Alias { name, asname: a.map(|a| a.1) }, + => ast::Alias { name: name.into(), asname: a.map(|a| a.1) }, }; // A name like abc or abc.def.ghi DottedName: String = { - => n, + => n.into(), => { - let mut r = n.to_string(); + let lock = ast::get_str_ref_lock(); + let mut r = ast::get_str_from_ref(&lock, n).to_string(); for x in n2 { r.push_str("."); - r.push_str(&x.1); + r.push_str(&ast::get_str_from_ref(&lock, x.1)); } r }, @@ -1157,7 +1158,7 @@ FunctionArgument: (Option<(ast::Location, Option)>, ast::Expr) = { }; (None, expr) }, - "=" => (Some((location, Some(i))), e), + "=" => (Some((location, Some(i.into()))), e), "*" => { let expr = ast::Expr::new( location, @@ -1199,7 +1200,7 @@ Bytes: Vec = { }, }; -Identifier: String = => s; +Identifier: ast::StrRef = => s; // Hook external lexer: extern { @@ -1299,7 +1300,7 @@ extern { complex => lexer::Tok::Complex { real: , imag: }, string => lexer::Tok::String { value: , is_fstring: }, bytes => lexer::Tok::Bytes { value: > }, - name => lexer::Tok::Name { name: }, + name => lexer::Tok::Name { name: }, "\n" => lexer::Tok::Newline, ";" => lexer::Tok::Semi, } diff --git a/nac3parser/src/token.rs b/nac3parser/src/token.rs index 1cedc66a6..73249db2e 100644 --- a/nac3parser/src/token.rs +++ b/nac3parser/src/token.rs @@ -2,11 +2,12 @@ //! Loosely based on token.h from CPython source: use num_bigint::BigInt; use std::fmt::{self, Write}; +use crate::ast; /// Python source code can be tokenized in a sequence of these tokens. #[derive(Clone, Debug, PartialEq)] pub enum Tok { - Name { name: String }, + Name { name: ast::StrRef }, Int { value: BigInt }, Float { value: f64 }, Complex { real: f64, imag: f64 }, @@ -110,7 +111,7 @@ impl fmt::Display for Tok { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { use Tok::*; match self { - Name { name } => write!(f, "'{}'", name), + Name { name } => write!(f, "'{}'", ast::get_str_from_ref(&ast::get_str_ref_lock(), *name)), Int { value } => write!(f, "'{}'", value), Float { value } => write!(f, "'{}'", value), Complex { real, imag } => write!(f, "{}j{}", real, imag),