// See also: file:///usr/share/doc/python/html/reference/grammar.html?highlight=grammar
// See also: https://github.com/antlr/grammars-v4/blob/master/python3/Python3.g4
// See also: file:///usr/share/doc/python/html/reference/compound_stmts.html#function-definitions
// See also: https://greentreesnakes.readthedocs.io/en/latest/nodes.html#keyword
use std::iter::FromIterator;
use crate::ast::{self, Constant};
use crate::fstring::parse_located_fstring;
use crate::function::{ArgumentList, parse_args, parse_params};
use crate::error::LexicalError;
use crate::error::LexicalErrorType;
use crate::lexer;
use crate::config_comment_helper::*;
use lalrpop_util::ParseError;
use num_bigint::BigInt;
grammar;
// This is a hack to reduce the amount of lalrpop tables generated:
// For each public entry point, a full parse table is generated.
// By having only a single pub function, we reduce this to one.
pub Top: ast::Mod = {
StartModule => ast::Mod::Module { body, type_ignores: vec![] },
StartInteractive => ast::Mod::Interactive { body },
StartExpression ("\n")* => ast::Mod::Expression { body: Box::new(body) },
};
Program: ast::Suite = {
=> {
lines.into_iter().flatten().collect()
},
};
// A file line either has a declaration, or an empty newline:
FileLine: ast::Suite = {
Statement,
"\n" => vec![],
};
Suite: ast::Suite = {
SimpleStatement,
"\n" Indent Dedent => s.into_iter().flatten().collect(),
};
Statement: ast::Suite = {
SimpleStatement,
=> vec![s],
};
SimpleStatement: ast::Suite = {
";"? "\n" =>? {
let mut statements = vec![s1];
statements.extend(s2.into_iter().map(|e| e.1));
handle_small_stmt(&mut statements, nac3com_above, nac3com_end, com_loc)?;
Ok(statements)
}
};
SmallStatement: ast::Stmt = {
ExpressionStatement,
PassStatement,
DelStatement,
FlowStatement,
ImportStatement,
GlobalStatement,
NonlocalStatement,
AssertStatement,
};
PassStatement: ast::Stmt = {
"pass" => {
ast::Stmt {
location,
custom: (),
node: ast::StmtKind::Pass { config_comment: vec![] },
}
},
};
DelStatement: ast::Stmt = {
"del" => {
ast::Stmt {
location,
custom: (),
node: ast::StmtKind::Delete { targets, config_comment: vec![] },
}
},
};
ExpressionStatement: ast::Stmt = {
=> {
// Just an expression, no assignment:
if suffix.is_empty() {
ast::Stmt {
custom: (),
location,
node: ast::StmtKind::Expr { value: Box::new(expression), config_comment: vec![] }
}
} else {
let mut targets = vec![expression];
let mut values = suffix;
while values.len() > 1 {
targets.push(values.remove(0));
}
let value = Box::new(values.into_iter().next().unwrap());
ast::Stmt {
custom: (),
location,
node: ast::StmtKind::Assign { targets, value, type_comment: None, config_comment: vec![] },
}
}
},
=> {
ast::Stmt {
custom: (),
location,
node: ast::StmtKind::AugAssign {
target: Box::new(target),
op,
value: Box::new(rhs),
config_comment: vec![],
},
}
},
":" => {
let simple = matches!(target.node, ast::ExprKind::Name { .. });
ast::Stmt {
custom: (),
location,
node: ast::StmtKind::AnnAssign {
target: Box::new(target),
annotation: Box::new(annotation),
value: rhs.map(Box::new),
simple,
config_comment: vec![],
},
}
},
};
AssignSuffix: ast::Expr = {
"=" => e
};
TestListOrYieldExpr: ast::Expr = {
TestList,
YieldExpr
}
#[inline]
TestOrStarExprList: ast::Expr = {
// as far as I can tell, these were the same
TestList
};
TestOrStarNamedExprList: ast::Expr = {
GenericList
};
TestOrStarExpr: ast::Expr = {
Test,
StarExpr,
};
TestOrStarNamedExpr: ast::Expr = {
NamedExpressionTest,
StarExpr,
};
AugAssign: ast::Operator = {
"+=" => ast::Operator::Add,
"-=" => ast::Operator::Sub,
"*=" => ast::Operator::Mult,
"@=" => ast::Operator::MatMult,
"/=" => ast::Operator::Div,
"%=" => ast::Operator::Mod,
"&=" => ast::Operator::BitAnd,
"|=" => ast::Operator::BitOr,
"^=" => ast::Operator::BitXor,
"<<=" => ast::Operator::LShift,
">>=" => ast::Operator::RShift,
"**=" => ast::Operator::Pow,
"//=" => ast::Operator::FloorDiv,
};
FlowStatement: ast::Stmt = {
"break" => {
ast::Stmt {
custom: (),
location,
node: ast::StmtKind::Break { config_comment: vec![] },
}
},
"continue" => {
ast::Stmt {
custom: (),
location,
node: ast::StmtKind::Continue { config_comment: vec![] },
}
},
"return" => {
ast::Stmt {
custom: (),
location,
node: ast::StmtKind::Return { value: value.map(Box::new), config_comment: vec![] },
}
},
=> {
ast::Stmt {
custom: (),
location,
node: ast::StmtKind::Expr { value: Box::new(expression), config_comment: vec![] },
}
},
RaiseStatement,
};
RaiseStatement: ast::Stmt = {
"raise" => {
ast::Stmt {
custom: (),
location,
node: ast::StmtKind::Raise { exc: None, cause: None, config_comment: vec![] },
}
},
"raise" => {
ast::Stmt {
custom: (),
location,
node: ast::StmtKind::Raise { exc: Some(Box::new(t)), cause: c.map(|x| Box::new(x.1)), config_comment: vec![] },
}
},
};
ImportStatement: ast::Stmt = {
"import" >> => {
ast::Stmt {
custom: (),
location,
node: ast::StmtKind::Import { names, config_comment: vec![] },
}
},
"from" "import" => {
let (level, module) = source;
ast::Stmt {
custom: (),
location,
node: ast::StmtKind::ImportFrom {
level,
module: module.map(|s| s.into()),
names,
config_comment: vec![]
},
}
},
};
ImportFromLocation: (usize, Option) = {
=> {
(dots.iter().sum(), Some(name))
},
=> {
(dots.iter().sum(), None)
},
};
ImportDots: usize = {
"..." => 3,
"." => 1,
};
ImportAsNames: Vec = {
>> => i,
"(" >> ","? ")" => i,
"*" => {
// Star import all
vec![ast::Alias { name: "*".into(), asname: None }]
},
};
#[inline]
ImportAsAlias: ast::Alias = {
=> ast::Alias { name: name.into(), asname: a.map(|a| a.1) },
};
// A name like abc or abc.def.ghi
DottedName: String = {
=> n.into(),
=> {
let lock = ast::get_str_ref_lock();
let mut r = ast::get_str_from_ref(&lock, n).to_string();
for x in n2 {
r.push_str(".");
r.push_str(&ast::get_str_from_ref(&lock, x.1));
}
r
},
};
GlobalStatement: ast::Stmt = {
"global" > => {
ast::Stmt {
custom: (),
location,
node: ast::StmtKind::Global { names, config_comment: vec![] }
}
},
};
NonlocalStatement: ast::Stmt = {
"nonlocal" > => {
ast::Stmt {
custom: (),
location,
node: ast::StmtKind::Nonlocal { names, config_comment: vec![] }
}
},
};
AssertStatement: ast::Stmt = {
"assert" => {
ast::Stmt {
custom: (),
location,
node: ast::StmtKind::Assert {
test: Box::new(test),
msg: msg.map(|e| Box::new(e.1)),
config_comment: vec![],
}
}
},
};
CompoundStatement: ast::Stmt = {
IfStatement,
WhileStatement,
ForStatement,
TryStatement,
WithStatement,
FuncDef,
ClassDef,
};
IfStatement: ast::Stmt = {
"if" ":" =>? {
// Determine last else:
let mut last = s3.map(|s| s.2).unwrap_or_default();
// handle elif:
for i in s2.into_iter().rev() {
let x = ast::Stmt {
custom: (),
location: i.0,
node: ast::StmtKind::If { test: Box::new(i.2), body: i.4, orelse: last, config_comment: vec![] },
};
last = vec![x];
}
Ok(ast::Stmt {
custom: (),
location,
node: ast::StmtKind::If {
test: Box::new(test),
body,
orelse: last,
config_comment: make_config_comment(location, stmt_loc, nac3com_above, nac3com_end)?
}
})
},
};
WhileStatement: ast::Stmt = {
"while" ":" =>? {
let orelse = s2.map(|s| s.2).unwrap_or_default();
Ok(ast::Stmt {
custom: (),
location,
node: ast::StmtKind::While {
test: Box::new(test),
body,
orelse,
config_comment: make_config_comment(location, stmt_loc, nac3com_above, nac3com_end)?
},
})
},
};
ForStatement: ast::Stmt = {
"for" "in" ":" =>? {
let orelse = s2.map(|s| s.2).unwrap_or_default();
let target = Box::new(target);
let iter = Box::new(iter);
let type_comment = None;
let node = if is_async.is_some() {
ast::StmtKind::AsyncFor {
target,
iter,
body,
orelse,
type_comment,
config_comment: make_config_comment(location, stmt_loc, nac3com_above, nac3com_end)?
}
} else {
ast::StmtKind::For {
target,
iter,
body,
orelse,
type_comment,
config_comment: make_config_comment(location, stmt_loc, nac3com_above, nac3com_end)?
}
};
Ok(ast::Stmt::new(location, node))
},
};
TryStatement: ast::Stmt = {
"try" ":" =>? {
let orelse = else_suite.map(|s| s.2).unwrap_or_default();
let finalbody = finally.map(|s| s.2).unwrap_or_default();
Ok(ast::Stmt {
custom: (),
location,
node: ast::StmtKind::Try {
body,
handlers,
orelse,
finalbody,
config_comment: make_config_comment(location, stmt_loc, nac3com_above, nac3com_end)?
},
})
},
"try" ":" =>? {
let handlers = vec![];
let orelse = vec![];
let finalbody = finally.2;
Ok(ast::Stmt {
custom: (),
location,
node: ast::StmtKind::Try {
body,
handlers,
orelse,
finalbody,
config_comment: make_config_comment(location, stmt_loc, nac3com_above, nac3com_end)?
},
})
},
};
ExceptClause: ast::Excepthandler = {
"except" ":" => {
ast::Excepthandler::new(
location,
ast::ExcepthandlerKind::ExceptHandler {
type_: typ.map(Box::new),
name: None,
body,
},
)
},
"except" ":" => {
ast::Excepthandler::new(
location,
ast::ExcepthandlerKind::ExceptHandler {
type_: Some(Box::new(x.0)),
name: Some(x.2),
body,
},
)
},
};
WithStatement: ast::Stmt = {
"with" > ":" =>? {
let type_comment = None;
let node = if is_async.is_some() {
ast::StmtKind::AsyncWith {
items,
body,
type_comment,
config_comment: make_config_comment(location, stmt_loc, nac3com_above, nac3com_end)?
}
} else {
ast::StmtKind::With {
items,
body,
type_comment,
config_comment: make_config_comment(location, stmt_loc, nac3com_above, nac3com_end)?
}
};
Ok(ast::Stmt::new(location, node))
},
};
WithItem: ast::Withitem = {
=> {
let optional_vars = n.map(|val| Box::new(val.1));
let context_expr = Box::new(context_expr);
ast::Withitem { context_expr, optional_vars }
},
};
FuncDef: ast::Stmt = {
"def" " Test)?> ":" =>? {
let args = Box::new(args);
let returns = r.map(|x| Box::new(x.1));
let type_comment = None;
let node = if is_async.is_some() {
ast::StmtKind::AsyncFunctionDef {
name,
args,
body,
decorator_list,
returns,
type_comment,
config_comment: make_config_comment(location, stmt_loc, nac3com_above, nac3com_end)?
}
} else {
ast::StmtKind::FunctionDef {
name,
args,
body,
decorator_list,
returns,
type_comment,
config_comment: make_config_comment(location, stmt_loc, nac3com_above, nac3com_end)?
}
};
Ok(ast::Stmt::new(location, node))
},
};
Parameters: ast::Arguments = {
"(" )?> ")" => {
a.unwrap_or_else(|| ast::Arguments {
posonlyargs: vec![],
args: vec![],
vararg: None,
kwonlyargs: vec![],
kw_defaults: vec![],
kwarg: None,
defaults: vec![]
})
}
};
// Note that this is a macro which is used once for function defs, and
// once for lambda defs.
ParameterList: ast::Arguments = {
> )?> ","? =>? {
let (posonlyargs, args, defaults) = parse_params(param1)?;
// Now gather rest of parameters:
let (vararg, kwonlyargs, kw_defaults, kwarg) = args2.map_or((None, vec![], vec![], None), |x| x.1);
Ok(ast::Arguments {
posonlyargs,
args,
kwonlyargs,
vararg,
kwarg,
defaults,
kw_defaults,
})
},
> )> ","? =>? {
let (posonlyargs, args, defaults) = parse_params(param1)?;
// Now gather rest of parameters:
let vararg = None;
let kwonlyargs = vec![];
let kw_defaults = vec![];
let kwarg = kw.1;
Ok(ast::Arguments {
posonlyargs,
args,
kwonlyargs,
vararg,
kwarg,
defaults,
kw_defaults,
})
},
> ","? => {
let (vararg, kwonlyargs, kw_defaults, kwarg) = params;
ast::Arguments {
posonlyargs: vec![],
args: vec![],
kwonlyargs,
vararg,
kwarg,
defaults: vec![],
kw_defaults,
}
},
> ","? => {
ast::Arguments {
posonlyargs: vec![],
args: vec![],
kwonlyargs: vec![],
vararg: None,
kwarg,
defaults: vec![],
kw_defaults: vec![],
}
},
};
// Use inline here to make sure the "," is not creating an ambiguity.
#[inline]
ParameterDefs: (Vec<(ast::Arg, Option)>, Vec<(ast::Arg, Option)>) = {
>> => {
(vec![], args)
},
>> "," "/" )*> => {
(pos_args, args.into_iter().map(|e| e.1).collect())
},
};
ParameterDef: (ast::Arg, Option) = {
=> (i, None),
"=" => (i, Some(e)),
};
UntypedParameter: ast::Arg = {
=> ast::Arg::new(
location,
ast::ArgData { arg, annotation: None, type_comment: None },
),
};
TypedParameter: ast::Arg = {
=> {
let annotation = a.map(|x| Box::new(x.1));
ast::Arg::new(location, ast::ArgData { arg, annotation, type_comment: None })
},
};
// Use inline here to make sure the "," is not creating an ambiguity.
// TODO: figure out another grammar that makes this inline no longer required.
#[inline]
ParameterListStarArgs: (Option>, Vec, Vec