Aadityavardhan
/
nac3_sca
forked from M-Labs/nac3
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

nac3parser, ast: add comment support core changes

escape-analysis
ychenfo 2021-11-04 15:00:27 +08:00
parent 32d1fe811b
commit 3b1cc02d06
6 changed files with 354 additions and 82 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

48
nac3ast/Python.asdl
View File

@ -10,43 +10,45 @@ module Python
stmt = FunctionDef(identifier name, arguments args,
stmt* body, expr* decorator_list, expr? returns,
string? type_comment)
string? type_comment, identifier* config_comment)
| AsyncFunctionDef(identifier name, arguments args,
stmt* body, expr* decorator_list, expr? returns,
string? type_comment)
string? type_comment, identifier* config_comment)
| ClassDef(identifier name,
expr* bases,
keyword* keywords,
stmt* body,
expr* decorator_list)
| Return(expr? value)
expr* decorator_list, identifier* config_comment)
| Return(expr? value, identifier* config_comment)
| Delete(expr* targets)
| Assign(expr* targets, expr value, string? type_comment)
| AugAssign(expr target, operator op, expr value)
| Delete(expr* targets, identifier* config_comment)
| Assign(expr* targets, expr value, string? type_comment, identifier* config_comment)
| AugAssign(expr target, operator op, expr value, identifier* config_comment)
-- 'simple' indicates that we annotate simple name without parens
| AnnAssign(expr target, expr annotation, expr? value, bool simple)
| AnnAssign(expr target, expr annotation, expr? value, bool simple, identifier* config_comment)
-- use 'orelse' because else is a keyword in target languages
| For(expr target, expr iter, stmt* body, stmt* orelse, string? type_comment)
| AsyncFor(expr target, expr iter, stmt* body, stmt* orelse, string? type_comment)
| While(expr test, stmt* body, stmt* orelse)
| If(expr test, stmt* body, stmt* orelse)
| With(withitem* items, stmt* body, string? type_comment)
| AsyncWith(withitem* items, stmt* body, string? type_comment)
| For(expr target, expr iter, stmt* body, stmt* orelse, string? type_comment, identifier* config_comment)
| AsyncFor(expr target, expr iter, stmt* body, stmt* orelse, string? type_comment, identifier* config_comment)
| While(expr test, stmt* body, stmt* orelse, identifier* config_comment)
| If(expr test, stmt* body, stmt* orelse, identifier* config_comment)
| With(withitem* items, stmt* body, string? type_comment, identifier* config_comment)
| AsyncWith(withitem* items, stmt* body, string? type_comment, identifier* config_comment)
| Raise(expr? exc, expr? cause)
| Try(stmt* body, excepthandler* handlers, stmt* orelse, stmt* finalbody)
| Assert(expr test, expr? msg)
| Raise(expr? exc, expr? cause, identifier* config_comment)
| Try(stmt* body, excepthandler* handlers, stmt* orelse, stmt* finalbody, identifier* config_comment)
| Assert(expr test, expr? msg, identifier* config_comment)
| Import(alias* names)
| ImportFrom(identifier? module, alias* names, int level)
| Import(alias* names, identifier* config_comment)
| ImportFrom(identifier? module, alias* names, int level, identifier* config_comment)
| Global(identifier* names)
| Nonlocal(identifier* names)
| Expr(expr value)
| Pass | Break | Continue
| Global(identifier* names, identifier* config_comment)
| Nonlocal(identifier* names, identifier* config_comment)
| Expr(expr value, identifier* config_comment)
| Pass(identifier* config_comment)
| Break(identifier* config_comment)
| Continue(identifier* config_comment)
-- col_offset is the byte offset in the utf8 string the parser uses
attributes (int lineno, int col_offset, int? end_lineno, int? end_col_offset)

85
nac3parser/src/config_comment_helper.rs Normal file
View File

@ -0,0 +1,85 @@
use lalrpop_util::ParseError;
use nac3ast::*;
use crate::ast::Ident;
use crate::ast::Location;
use crate::token::Tok;
use crate::error::*;
pub fn make_config_comment(
com_loc: Location,
stmt_loc: Location,
nac3com_above: Vec<(Ident, Tok)>,
nac3com_end: Option<Ident>
) -> Result<Vec<Ident>, ParseError<Location, Tok, LexicalError>> {
if com_loc.column() != stmt_loc.column() {
return Err(ParseError::User {
error: LexicalError {
location: com_loc,
error: LexicalErrorType::OtherError(
format!(
"config comment at top must have the same indentation with what it applies, comment at {}, statement at {}",
com_loc,
stmt_loc,
)
)
}
})
};
Ok(
nac3com_above
.into_iter()
.map(|(com, _)| com)
.chain(nac3com_end.map_or_else(|| vec![].into_iter(), |com| vec![com].into_iter()))
.collect()
)
}
pub fn handle_small_stmt<U>(stmts: &mut [Stmt<U>], nac3com_above: Vec<(Ident, Tok)>, nac3com_end: Option<Ident>, com_above_loc: Location) -> Result<(), ParseError<Location, Tok, LexicalError>> {
if com_above_loc.column() != stmts[0].location.column() {
return Err(ParseError::User {
error: LexicalError {
location: com_above_loc,
error: LexicalErrorType::OtherError(
format!(
"config comment at top must have the same indentation with what it applies, comment at {}, statement at {}",
com_above_loc,
stmts[0].location,
)
)
}
})
}
apply_config_comments(
&mut stmts[0],
nac3com_above
.into_iter()
.map(|(com, _)| com).collect()
);
apply_config_comments(
stmts.last_mut().unwrap(),
nac3com_end.map_or_else(Vec::new, |com| vec![com])
);
Ok(())
}
fn apply_config_comments<U>(stmt: &mut Stmt<U>, comments: Vec<Ident>) {
match &mut stmt.node {
StmtKind::Pass { config_comment, .. }
| StmtKind::Delete { config_comment, .. }
| StmtKind::Expr { config_comment, .. }
| StmtKind::Assign { config_comment, .. }
| StmtKind::AugAssign { config_comment, .. }
| StmtKind::AnnAssign { config_comment, .. }
| StmtKind::Break { config_comment, .. }
| StmtKind::Continue { config_comment, .. }
| StmtKind::Return { config_comment, .. }
| StmtKind::Raise { config_comment, .. }
| StmtKind::Import { config_comment, .. }
| StmtKind::ImportFrom { config_comment, .. }
| StmtKind::Global { config_comment, .. }
| StmtKind::Nonlocal { config_comment, .. }
| StmtKind::Assert { config_comment, .. } => config_comment.extend(comments),
_ => { unreachable!("only small statements should call this function") }
}
}

146
nac3parser/src/lexer.rs
View File

@ -65,6 +65,7 @@ pub struct Lexer<T: Iterator<Item = char>> {
chr1: Option<char>,
chr2: Option<char>,
location: Location,
config_comment_prefix: Option<&'static str>
}
pub static KEYWORDS: phf::Map<&'static str, Tok> = phf::phf_map! {
@ -196,6 +197,7 @@ where
location: start,
chr1: None,
chr2: None,
config_comment_prefix: Some(" nac3:")
};
lxr.next_char();
lxr.next_char();
@ -415,17 +417,45 @@ where
}
}
/// Skip everything until end of line
fn lex_comment(&mut self) {
/// Skip everything until end of line, may produce nac3 pseudocomment
fn lex_comment(&mut self) -> Option<Spanned> {
self.next_char();
// if possibly nac3 pseudocomment, special handling for `# nac3:`
let (mut prefix, mut is_comment) = self
.config_comment_prefix
.map_or_else(|| ("".chars(), false), |v| (v.chars(), true));
// for the correct location of config comment
let mut start_loc = self.location;
start_loc.go_left();
loop {
match self.chr0 {
Some('\n') => return,
Some(_) => {}
None => return,
Some('\n') => return None,
None => return None,
Some(c) => {
if let (true, Some(p)) = (is_comment, prefix.next()) {
is_comment = is_comment && c == p
} else {
// done checking prefix, if is comment then return the spanned
if is_comment {
let mut content = String::new();
loop {
match self.chr0 {
Some('\n') | None => break,
Some(c) => content.push(c),
}
self.next_char();
}
return Some((
start_loc,
Tok::ConfigComment { content: content.trim().into() },
self.location
));
}
}
}
}
self.next_char();
}
};
}
fn unicode_literal(&mut self, literal_number: usize) -> Result<char, LexicalError> {
@ -658,10 +688,11 @@ where
}
/// Given we are at the start of a line, count the number of spaces and/or tabs until the first character.
fn eat_indentation(&mut self) -> Result<IndentationLevel, LexicalError> {
fn eat_indentation(&mut self) -> Result<(IndentationLevel, Option<Spanned>), LexicalError> {
// Determine indentation:
let mut spaces: usize = 0;
let mut tabs: usize = 0;
let mut nac3comment: Option<Spanned> = None;
loop {
match self.chr0 {
Some(' ') => {
@ -693,7 +724,14 @@ where
tabs += 1;
}
Some('#') => {
self.lex_comment();
nac3comment = self.lex_comment();
// if is nac3comment, we need to add newline, so it is not begin of line
// and we should break from the loop, else in the next loop it will be
// regarded as a empty line
if nac3comment.is_some() {
self.at_begin_of_line = false;
break;
}
spaces = 0;
tabs = 0;
}
@ -722,11 +760,12 @@ where
}
}
Ok(IndentationLevel { tabs, spaces })
Ok((IndentationLevel { tabs, spaces }, nac3comment))
}
fn handle_indentations(&mut self) -> Result<(), LexicalError> {
let indentation_level = self.eat_indentation()?;
let eat_result = self.eat_indentation()?;
let indentation_level = eat_result.0;
if self.nesting == 0 {
// Determine indent or dedent:
@ -770,6 +809,10 @@ where
}
}
}
};
if let Some(comment) = eat_result.1 {
self.emit(comment);
}
Ok(())
@ -833,7 +876,9 @@ where
self.emit(number);
}
'#' => {
self.lex_comment();
if let Some(c) = self.lex_comment() {
self.emit(c);
};
}
'"' | '\'' => {
let string = self.lex_string(false, false, false, false)?;
@ -1287,6 +1332,85 @@ mod tests {
lexer.map(|x| x.unwrap().1).collect()
}
#[test]
fn test_nac3comment() {
let src = "\
a: int32
# nac3:
b: int64";
let tokens = lex_source(src);
assert_eq!(
tokens,
vec![
Tok::Name { name: "a".into() },
Tok::Colon,
Tok::Name { name: "int32".into() },
Tok::Newline,
Tok::ConfigComment { content: "".into() },
Tok::Newline,
Tok::Name { name: "b".into() },
Tok::Colon,
Tok::Name { name: "int64".into() },
Tok::Newline,
]
);
}
#[test]
fn test_class_lex_with_nac3comment() {
use Tok::*;
let source = "\
class Foo(A, B):
# normal comment
# nac3: no indent
# nac3: correct indent
b: int32
a: int32 # nac3: no need indent
def __init__(self):
pass";
let tokens = lex_source(source);
assert_eq!(
tokens,
vec![
Class,
Name { name: "Foo".into() },
Lpar,
Name { name: "A".into() },
Comma,
Name { name: "B".into() },
Rpar,
Colon,
Newline,
ConfigComment { content: "no indent".into() },
Newline,
Indent,
ConfigComment { content: "correct indent".into() },
Newline,
Name { name: "b".into() },
Colon,
Name { name: "int32".into() },
Newline,
Name { name: "a".into() },
Colon,
Name { name: "int32".into() },
ConfigComment { content: "no need indent".into() },
Newline,
Def,
Name { name: "__init__".into() },
Lpar,
Name { name: "self".into() },
Rpar,
Colon,
Newline,
Indent,
Pass,
Newline,
Dedent,
Dedent
]
)
}
#[test]
fn test_newline_processor() {
// Escape \ followed by \n (by removal):

1
nac3parser/src/lib.rs
View File

@ -32,3 +32,4 @@ lalrpop_mod!(
python
);
pub mod token;
pub mod config_comment_helper;

154
nac3parser/src/python.lalrpop
View File

@ -9,8 +9,11 @@ use crate::ast;
use crate::fstring::parse_located_fstring;
use crate::function::{ArgumentList, parse_args, parse_params};
use crate::error::LexicalError;
use crate::error::LexicalErrorType;
use crate::lexer;
use crate::config_comment_helper::*;
use lalrpop_util::ParseError;
use num_bigint::BigInt;
grammar;
@ -47,10 +50,11 @@ Statement: ast::Suite = {
};
SimpleStatement: ast::Suite = {
<s1:SmallStatement> <s2:(";" SmallStatement)*> ";"? "\n" => {
<com_loc:@L> <nac3com_above:(config_comment "\n")*> <s1:SmallStatement> <s2:(";" SmallStatement)*> ";"? <nac3com_end:config_comment?> "\n" =>? {
let mut statements = vec![s1];
statements.extend(s2.into_iter().map(|e| e.1));
statements
handle_small_stmt(&mut statements, nac3com_above, nac3com_end, com_loc)?;
Ok(statements)
}
};
@ -70,7 +74,7 @@ PassStatement: ast::Stmt = {
ast::Stmt {
location,
custom: (),
node: ast::StmtKind::Pass,
node: ast::StmtKind::Pass { config_comment: vec![] },
}
},
};
@ -80,7 +84,7 @@ DelStatement: ast::Stmt = {
ast::Stmt {
location,
custom: (),
node: ast::StmtKind::Delete { targets },
node: ast::StmtKind::Delete { targets, config_comment: vec![] },
}
},
};
@ -92,7 +96,7 @@ ExpressionStatement: ast::Stmt = {
ast::Stmt {
custom: (),
location,
node: ast::StmtKind::Expr { value: Box::new(expression) }
node: ast::StmtKind::Expr { value: Box::new(expression), config_comment: vec![] }
}
} else {
let mut targets = vec![expression];
@ -107,7 +111,7 @@ ExpressionStatement: ast::Stmt = {
ast::Stmt {
custom: (),
location,
node: ast::StmtKind::Assign { targets, value, type_comment: None },
node: ast::StmtKind::Assign { targets, value, type_comment: None, config_comment: vec![] },
}
}
},
@ -118,7 +122,8 @@ ExpressionStatement: ast::Stmt = {
node: ast::StmtKind::AugAssign {
target: Box::new(target),
op,
value: Box::new(rhs)
value: Box::new(rhs),
config_comment: vec![],
},
}
},
@ -132,6 +137,7 @@ ExpressionStatement: ast::Stmt = {
annotation: Box::new(annotation),
value: rhs.map(Box::new),
simple,
config_comment: vec![],
},
}
},
@ -187,28 +193,28 @@ FlowStatement: ast::Stmt = {
ast::Stmt {
custom: (),
location,
node: ast::StmtKind::Break,
node: ast::StmtKind::Break { config_comment: vec![] },
}
},
<location:@L> "continue" => {
ast::Stmt {
custom: (),
location,
node: ast::StmtKind::Continue,
node: ast::StmtKind::Continue { config_comment: vec![] },
}
},
<location:@L> "return" <value:TestList?> => {
ast::Stmt {
custom: (),
location,
node: ast::StmtKind::Return { value: value.map(Box::new) },
node: ast::StmtKind::Return { value: value.map(Box::new), config_comment: vec![] },
}
},
<location:@L> <expression:YieldExpr> => {
ast::Stmt {
custom: (),
location,
node: ast::StmtKind::Expr { value: Box::new(expression) },
node: ast::StmtKind::Expr { value: Box::new(expression), config_comment: vec![] },
}
},
RaiseStatement,
@ -219,14 +225,14 @@ RaiseStatement: ast::Stmt = {
ast::Stmt {
custom: (),
location,
node: ast::StmtKind::Raise { exc: None, cause: None },
node: ast::StmtKind::Raise { exc: None, cause: None, config_comment: vec![] },
}
},
<location:@L> "raise" <t:Test> <c:("from" Test)?> => {
ast::Stmt {
custom: (),
location,
node: ast::StmtKind::Raise { exc: Some(Box::new(t)), cause: c.map(|x| Box::new(x.1)) },
node: ast::StmtKind::Raise { exc: Some(Box::new(t)), cause: c.map(|x| Box::new(x.1)), config_comment: vec![] },
}
},
};
@ -236,7 +242,7 @@ ImportStatement: ast::Stmt = {
ast::Stmt {
custom: (),
location,
node: ast::StmtKind::Import { names },
node: ast::StmtKind::Import { names, config_comment: vec![] },
}
},
<location:@L> "from" <source:ImportFromLocation> "import" <names: ImportAsNames> => {
@ -247,7 +253,8 @@ ImportStatement: ast::Stmt = {
node: ast::StmtKind::ImportFrom {
level,
module: module.map(|s| s.into()),
names
names,
config_comment: vec![]
},
}
},
@ -301,7 +308,7 @@ GlobalStatement: ast::Stmt = {
ast::Stmt {
custom: (),
location,
node: ast::StmtKind::Global { names }
node: ast::StmtKind::Global { names, config_comment: vec![] }
}
},
};
@ -311,7 +318,7 @@ NonlocalStatement: ast::Stmt = {
ast::Stmt {
custom: (),
location,
node: ast::StmtKind::Nonlocal { names }
node: ast::StmtKind::Nonlocal { names, config_comment: vec![] }
}
},
};
@ -323,7 +330,8 @@ AssertStatement: ast::Stmt = {
location,
node: ast::StmtKind::Assert {
test: Box::new(test),
msg: msg.map(|e| Box::new(e.1))
msg: msg.map(|e| Box::new(e.1)),
config_comment: vec![],
}
}
},
@ -340,7 +348,7 @@ CompoundStatement: ast::Stmt = {
};
IfStatement: ast::Stmt = {
<location:@L> "if" <test:NamedExpressionTest> ":" <body:Suite> <s2:(@L "elif" NamedExpressionTest ":" Suite)*> <s3:("else" ":" Suite)?> => {
<location:@L> <nac3com_above:(config_comment "\n")*> <stmt_loc:@L> "if" <test:NamedExpressionTest> ":" <nac3com_end:config_comment?> <body:Suite> <s2:(@L "elif" NamedExpressionTest ":" Suite)*> <s3:("else" ":" Suite)?> =>? {
// Determine last else:
let mut last = s3.map(|s| s.2).unwrap_or_default();
@ -349,54 +357,74 @@ IfStatement: ast::Stmt = {
let x = ast::Stmt {
custom: (),
location: i.0,
node: ast::StmtKind::If { test: Box::new(i.2), body: i.4, orelse: last },
node: ast::StmtKind::If { test: Box::new(i.2), body: i.4, orelse: last, config_comment: vec![] },
};
last = vec![x];
}
ast::Stmt {
Ok(ast::Stmt {
custom: (),
location,
node: ast::StmtKind::If { test: Box::new(test), body, orelse: last }
}
node: ast::StmtKind::If {
test: Box::new(test),
body,
orelse: last,
config_comment: make_config_comment(location, stmt_loc, nac3com_above, nac3com_end)?
}
})
},
};
WhileStatement: ast::Stmt = {
<location:@L> "while" <test:NamedExpressionTest> ":" <body:Suite> <s2:("else" ":" Suite)?> => {
<location:@L> <nac3com_above:(config_comment "\n")*> <stmt_loc:@L> "while" <test:NamedExpressionTest> ":" <nac3com_end:config_comment?> <body:Suite> <s2:("else" ":" Suite)?> =>? {
let orelse = s2.map(|s| s.2).unwrap_or_default();
ast::Stmt {
Ok(ast::Stmt {
custom: (),
location,
node: ast::StmtKind::While {
test: Box::new(test),
body,
orelse
orelse,
config_comment: make_config_comment(location, stmt_loc, nac3com_above, nac3com_end)?
},
}
})
},
};
ForStatement: ast::Stmt = {
<location:@L> <is_async:"async"?> "for" <target:ExpressionList> "in" <iter:TestList> ":" <body:Suite> <s2:("else" ":" Suite)?> => {
<location:@L> <nac3com_above:(config_comment "\n")*> <stmt_loc:@L> <is_async:"async"?> "for" <target:ExpressionList> "in" <iter:TestList> ":" <nac3com_end:config_comment?> <body:Suite> <s2:("else" ":" Suite)?> =>? {
let orelse = s2.map(|s| s.2).unwrap_or_default();
let target = Box::new(target);
let iter = Box::new(iter);
let type_comment = None;
let node = if is_async.is_some() {
ast::StmtKind::AsyncFor { target, iter, body, orelse, type_comment }
ast::StmtKind::AsyncFor {
target,
iter,
body,
orelse,
type_comment,
config_comment: make_config_comment(location, stmt_loc, nac3com_above, nac3com_end)?
}
} else {
ast::StmtKind::For { target, iter, body, orelse, type_comment }
ast::StmtKind::For {
target,
iter,
body,
orelse,
type_comment,
config_comment: make_config_comment(location, stmt_loc, nac3com_above, nac3com_end)?
}
};
ast::Stmt::new(location, node)
Ok(ast::Stmt::new(location, node))
},
};
TryStatement: ast::Stmt = {
<location:@L> "try" ":" <body:Suite> <handlers:ExceptClause+> <else_suite:("else" ":" Suite)?> <finally:("finally" ":" Suite)?> => {
<location:@L> <nac3com_above:(config_comment "\n")*> <stmt_loc:@L> "try" ":" <nac3com_end:config_comment?> <body:Suite> <handlers:ExceptClause+> <else_suite:("else" ":" Suite)?> <finally:("finally" ":" Suite)?> =>? {
let orelse = else_suite.map(|s| s.2).unwrap_or_default();
let finalbody = finally.map(|s| s.2).unwrap_or_default();
ast::Stmt {
Ok(ast::Stmt {
custom: (),
location,
node: ast::StmtKind::Try {
@ -404,14 +432,15 @@ TryStatement: ast::Stmt = {
handlers,
orelse,
finalbody,
config_comment: make_config_comment(location, stmt_loc, nac3com_above, nac3com_end)?
},
}
})
},
<location:@L> "try" ":" <body:Suite> <finally:("finally" ":" Suite)> => {
<location:@L> <nac3com_above:(config_comment "\n")*> <stmt_loc:@L> "try" ":" <nac3com_end:config_comment?> <body:Suite> <finally:("finally" ":" Suite)> =>? {
let handlers = vec![];
let orelse = vec![];
let finalbody = finally.2;
ast::Stmt {
Ok(ast::Stmt {
custom: (),
location,
node: ast::StmtKind::Try {
@ -419,8 +448,9 @@ TryStatement: ast::Stmt = {
handlers,
orelse,
finalbody,
config_comment: make_config_comment(location, stmt_loc, nac3com_above, nac3com_end)?
},
}
})
},
};
@ -448,14 +478,24 @@ ExceptClause: ast::Excepthandler = {
};
WithStatement: ast::Stmt = {
<location:@L> <is_async:"async"?> "with" <items:OneOrMore<WithItem>> ":" <body:Suite> => {
<location:@L> <nac3com_above:(config_comment "\n")*> <stmt_loc:@L> <is_async:"async"?> "with" <items:OneOrMore<WithItem>> ":" <nac3com_end:config_comment?> <body:Suite> =>? {
let type_comment = None;
let node = if is_async.is_some() {
ast::StmtKind::AsyncWith { items, body, type_comment }
ast::StmtKind::AsyncWith {
items,
body,
type_comment,
config_comment: make_config_comment(location, stmt_loc, nac3com_above, nac3com_end)?
}
} else {
ast::StmtKind::With { items, body, type_comment }
ast::StmtKind::With {
items,
body,
type_comment,
config_comment: make_config_comment(location, stmt_loc, nac3com_above, nac3com_end)?
}
};
ast::Stmt::new(location, node)
Ok(ast::Stmt::new(location, node))
},
};
@ -468,16 +508,32 @@ WithItem: ast::Withitem = {
};
FuncDef: ast::Stmt = {
<decorator_list:Decorator*> <location:@L> <is_async:"async"?> "def" <name:Identifier> <args:Parameters> <r:("->" Test)?> ":" <body:Suite> => {
<decorator_list:Decorator*> <location:@L> <nac3com_above:(config_comment "\n")*> <stmt_loc:@L> <is_async:"async"?> "def" <name:Identifier> <args:Parameters> <r:("->" Test)?> ":" <nac3com_end:config_comment?> <body:Suite> =>? {
let args = Box::new(args);
let returns = r.map(|x| Box::new(x.1));
let type_comment = None;
let node = if is_async.is_some() {
ast::StmtKind::AsyncFunctionDef { name, args, body, decorator_list, returns, type_comment }
ast::StmtKind::AsyncFunctionDef {
name,
args,
body,
decorator_list,
returns,
type_comment,
config_comment: make_config_comment(location, stmt_loc, nac3com_above, nac3com_end)?
}
} else {
ast::StmtKind::FunctionDef { name, args, body, decorator_list, returns, type_comment }
ast::StmtKind::FunctionDef {
name,
args,
body,
decorator_list,
returns,
type_comment,
config_comment: make_config_comment(location, stmt_loc, nac3com_above, nac3com_end)?
}
};
ast::Stmt::new(location, node)
Ok(ast::Stmt::new(location, node))
},
};
@ -615,12 +671,12 @@ KwargParameter<ArgType>: Option<Box<ast::Arg>> = {
};
ClassDef: ast::Stmt = {
<decorator_list:Decorator*> <location:@L> "class" <name:Identifier> <a:("(" ArgumentList ")")?> ":" <body:Suite> => {
<decorator_list:Decorator*> <location:@L> <nac3com_above:(config_comment "\n")*> <stmt_loc:@L> "class" <name:Identifier> <a:("(" ArgumentList ")")?> ":" <nac3com_end:config_comment?> <body:Suite> =>? {
let (bases, keywords) = match a {
Some((_, arg, _)) => (arg.args, arg.keywords),
None => (vec![], vec![]),
};
ast::Stmt {
Ok(ast::Stmt {
custom: (),
location,
node: ast::StmtKind::ClassDef {
@ -629,8 +685,9 @@ ClassDef: ast::Stmt = {
keywords,
body,
decorator_list,
config_comment: make_config_comment(location, stmt_loc, nac3com_above, nac3com_end)?
},
}
})
},
};
@ -1301,6 +1358,7 @@ extern {
string => lexer::Tok::String { value: <String>, is_fstring: <bool> },
bytes => lexer::Tok::Bytes { value: <Vec<u8>> },
name => lexer::Tok::Name { name: <ast::StrRef> },
config_comment => lexer::Tok::ConfigComment { content: <ast::StrRef> },
"\n" => lexer::Tok::Newline,
";" => lexer::Tok::Semi,
}

2
nac3parser/src/token.rs
View File

@ -13,6 +13,7 @@ pub enum Tok {
Complex { real: f64, imag: f64 },
String { value: String, is_fstring: bool },
Bytes { value: Vec<u8> },
ConfigComment { content: ast::StrRef },
Newline,
Indent,
Dedent,
@ -134,6 +135,7 @@ impl fmt::Display for Tok {
}
f.write_str("\"")
}
ConfigComment { content } => write!(f, "ConfigComment: '{}'", ast::get_str_from_ref(&ast::get_str_ref_lock(), *content)),
Newline => f.write_str("Newline"),
Indent => f.write_str("Indent"),
Dedent => f.write_str("Dedent"),