From 3b1cc02d06ae4e534ebe3086f18660db07f7ec11 Mon Sep 17 00:00:00 2001 From: ychenfo Date: Thu, 4 Nov 2021 15:00:27 +0800 Subject: [PATCH 1/4] nac3parser, ast: add comment support core changes --- nac3ast/Python.asdl | 48 ++++---- nac3parser/src/config_comment_helper.rs | 85 +++++++++++++ nac3parser/src/lexer.rs | 146 ++++++++++++++++++++-- nac3parser/src/lib.rs | 1 + nac3parser/src/python.lalrpop | 154 ++++++++++++++++-------- nac3parser/src/token.rs | 2 + 6 files changed, 354 insertions(+), 82 deletions(-) create mode 100644 nac3parser/src/config_comment_helper.rs diff --git a/nac3ast/Python.asdl b/nac3ast/Python.asdl index b3abe1628..0bc56c81e 100644 --- a/nac3ast/Python.asdl +++ b/nac3ast/Python.asdl @@ -10,43 +10,45 @@ module Python stmt = FunctionDef(identifier name, arguments args, stmt* body, expr* decorator_list, expr? returns, - string? type_comment) + string? type_comment, identifier* config_comment) | AsyncFunctionDef(identifier name, arguments args, stmt* body, expr* decorator_list, expr? returns, - string? type_comment) + string? type_comment, identifier* config_comment) | ClassDef(identifier name, expr* bases, keyword* keywords, stmt* body, - expr* decorator_list) - | Return(expr? value) + expr* decorator_list, identifier* config_comment) + | Return(expr? value, identifier* config_comment) - | Delete(expr* targets) - | Assign(expr* targets, expr value, string? type_comment) - | AugAssign(expr target, operator op, expr value) + | Delete(expr* targets, identifier* config_comment) + | Assign(expr* targets, expr value, string? type_comment, identifier* config_comment) + | AugAssign(expr target, operator op, expr value, identifier* config_comment) -- 'simple' indicates that we annotate simple name without parens - | AnnAssign(expr target, expr annotation, expr? value, bool simple) + | AnnAssign(expr target, expr annotation, expr? value, bool simple, identifier* config_comment) -- use 'orelse' because else is a keyword in target languages - | For(expr target, expr iter, stmt* body, stmt* orelse, string? type_comment) - | AsyncFor(expr target, expr iter, stmt* body, stmt* orelse, string? type_comment) - | While(expr test, stmt* body, stmt* orelse) - | If(expr test, stmt* body, stmt* orelse) - | With(withitem* items, stmt* body, string? type_comment) - | AsyncWith(withitem* items, stmt* body, string? type_comment) + | For(expr target, expr iter, stmt* body, stmt* orelse, string? type_comment, identifier* config_comment) + | AsyncFor(expr target, expr iter, stmt* body, stmt* orelse, string? type_comment, identifier* config_comment) + | While(expr test, stmt* body, stmt* orelse, identifier* config_comment) + | If(expr test, stmt* body, stmt* orelse, identifier* config_comment) + | With(withitem* items, stmt* body, string? type_comment, identifier* config_comment) + | AsyncWith(withitem* items, stmt* body, string? type_comment, identifier* config_comment) - | Raise(expr? exc, expr? cause) - | Try(stmt* body, excepthandler* handlers, stmt* orelse, stmt* finalbody) - | Assert(expr test, expr? msg) + | Raise(expr? exc, expr? cause, identifier* config_comment) + | Try(stmt* body, excepthandler* handlers, stmt* orelse, stmt* finalbody, identifier* config_comment) + | Assert(expr test, expr? msg, identifier* config_comment) - | Import(alias* names) - | ImportFrom(identifier? module, alias* names, int level) + | Import(alias* names, identifier* config_comment) + | ImportFrom(identifier? module, alias* names, int level, identifier* config_comment) - | Global(identifier* names) - | Nonlocal(identifier* names) - | Expr(expr value) - | Pass | Break | Continue + | Global(identifier* names, identifier* config_comment) + | Nonlocal(identifier* names, identifier* config_comment) + | Expr(expr value, identifier* config_comment) + | Pass(identifier* config_comment) + | Break(identifier* config_comment) + | Continue(identifier* config_comment) -- col_offset is the byte offset in the utf8 string the parser uses attributes (int lineno, int col_offset, int? end_lineno, int? end_col_offset) diff --git a/nac3parser/src/config_comment_helper.rs b/nac3parser/src/config_comment_helper.rs new file mode 100644 index 000000000..588f3e38f --- /dev/null +++ b/nac3parser/src/config_comment_helper.rs @@ -0,0 +1,85 @@ +use lalrpop_util::ParseError; +use nac3ast::*; +use crate::ast::Ident; +use crate::ast::Location; +use crate::token::Tok; +use crate::error::*; + +pub fn make_config_comment( + com_loc: Location, + stmt_loc: Location, + nac3com_above: Vec<(Ident, Tok)>, + nac3com_end: Option +) -> Result, ParseError> { + if com_loc.column() != stmt_loc.column() { + return Err(ParseError::User { + error: LexicalError { + location: com_loc, + error: LexicalErrorType::OtherError( + format!( + "config comment at top must have the same indentation with what it applies, comment at {}, statement at {}", + com_loc, + stmt_loc, + ) + ) + } + }) + }; + Ok( + nac3com_above + .into_iter() + .map(|(com, _)| com) + .chain(nac3com_end.map_or_else(|| vec![].into_iter(), |com| vec![com].into_iter())) + .collect() + ) +} + +pub fn handle_small_stmt(stmts: &mut [Stmt], nac3com_above: Vec<(Ident, Tok)>, nac3com_end: Option, com_above_loc: Location) -> Result<(), ParseError> { + if com_above_loc.column() != stmts[0].location.column() { + return Err(ParseError::User { + error: LexicalError { + location: com_above_loc, + error: LexicalErrorType::OtherError( + format!( + "config comment at top must have the same indentation with what it applies, comment at {}, statement at {}", + com_above_loc, + stmts[0].location, + ) + ) + } + }) + } + apply_config_comments( + &mut stmts[0], + nac3com_above + .into_iter() + .map(|(com, _)| com).collect() + ); + apply_config_comments( + stmts.last_mut().unwrap(), + nac3com_end.map_or_else(Vec::new, |com| vec![com]) + ); + Ok(()) +} + +fn apply_config_comments(stmt: &mut Stmt, comments: Vec) { + match &mut stmt.node { + StmtKind::Pass { config_comment, .. } + | StmtKind::Delete { config_comment, .. } + | StmtKind::Expr { config_comment, .. } + | StmtKind::Assign { config_comment, .. } + | StmtKind::AugAssign { config_comment, .. } + | StmtKind::AnnAssign { config_comment, .. } + | StmtKind::Break { config_comment, .. } + | StmtKind::Continue { config_comment, .. } + | StmtKind::Return { config_comment, .. } + | StmtKind::Raise { config_comment, .. } + | StmtKind::Import { config_comment, .. } + | StmtKind::ImportFrom { config_comment, .. } + | StmtKind::Global { config_comment, .. } + | StmtKind::Nonlocal { config_comment, .. } + | StmtKind::Assert { config_comment, .. } => config_comment.extend(comments), + + _ => { unreachable!("only small statements should call this function") } + } +} diff --git a/nac3parser/src/lexer.rs b/nac3parser/src/lexer.rs index 1936eecfa..9166abd42 100644 --- a/nac3parser/src/lexer.rs +++ b/nac3parser/src/lexer.rs @@ -65,6 +65,7 @@ pub struct Lexer> { chr1: Option, chr2: Option, location: Location, + config_comment_prefix: Option<&'static str> } pub static KEYWORDS: phf::Map<&'static str, Tok> = phf::phf_map! { @@ -196,6 +197,7 @@ where location: start, chr1: None, chr2: None, + config_comment_prefix: Some(" nac3:") }; lxr.next_char(); lxr.next_char(); @@ -415,17 +417,45 @@ where } } - /// Skip everything until end of line - fn lex_comment(&mut self) { + /// Skip everything until end of line, may produce nac3 pseudocomment + fn lex_comment(&mut self) -> Option { self.next_char(); + // if possibly nac3 pseudocomment, special handling for `# nac3:` + let (mut prefix, mut is_comment) = self + .config_comment_prefix + .map_or_else(|| ("".chars(), false), |v| (v.chars(), true)); + // for the correct location of config comment + let mut start_loc = self.location; + start_loc.go_left(); loop { match self.chr0 { - Some('\n') => return, - Some(_) => {} - None => return, + Some('\n') => return None, + None => return None, + Some(c) => { + if let (true, Some(p)) = (is_comment, prefix.next()) { + is_comment = is_comment && c == p + } else { + // done checking prefix, if is comment then return the spanned + if is_comment { + let mut content = String::new(); + loop { + match self.chr0 { + Some('\n') | None => break, + Some(c) => content.push(c), + } + self.next_char(); + } + return Some(( + start_loc, + Tok::ConfigComment { content: content.trim().into() }, + self.location + )); + } + } + } } self.next_char(); - } + }; } fn unicode_literal(&mut self, literal_number: usize) -> Result { @@ -658,10 +688,11 @@ where } /// Given we are at the start of a line, count the number of spaces and/or tabs until the first character. - fn eat_indentation(&mut self) -> Result { + fn eat_indentation(&mut self) -> Result<(IndentationLevel, Option), LexicalError> { // Determine indentation: let mut spaces: usize = 0; let mut tabs: usize = 0; + let mut nac3comment: Option = None; loop { match self.chr0 { Some(' ') => { @@ -693,7 +724,14 @@ where tabs += 1; } Some('#') => { - self.lex_comment(); + nac3comment = self.lex_comment(); + // if is nac3comment, we need to add newline, so it is not begin of line + // and we should break from the loop, else in the next loop it will be + // regarded as a empty line + if nac3comment.is_some() { + self.at_begin_of_line = false; + break; + } spaces = 0; tabs = 0; } @@ -722,11 +760,12 @@ where } } - Ok(IndentationLevel { tabs, spaces }) + Ok((IndentationLevel { tabs, spaces }, nac3comment)) } fn handle_indentations(&mut self) -> Result<(), LexicalError> { - let indentation_level = self.eat_indentation()?; + let eat_result = self.eat_indentation()?; + let indentation_level = eat_result.0; if self.nesting == 0 { // Determine indent or dedent: @@ -770,6 +809,10 @@ where } } } + }; + + if let Some(comment) = eat_result.1 { + self.emit(comment); } Ok(()) @@ -833,7 +876,9 @@ where self.emit(number); } '#' => { - self.lex_comment(); + if let Some(c) = self.lex_comment() { + self.emit(c); + }; } '"' | '\'' => { let string = self.lex_string(false, false, false, false)?; @@ -1287,6 +1332,85 @@ mod tests { lexer.map(|x| x.unwrap().1).collect() } + #[test] + fn test_nac3comment() { + let src = "\ +a: int32 +# nac3: +b: int64"; + let tokens = lex_source(src); + assert_eq!( + tokens, + vec![ + Tok::Name { name: "a".into() }, + Tok::Colon, + Tok::Name { name: "int32".into() }, + Tok::Newline, + Tok::ConfigComment { content: "".into() }, + Tok::Newline, + Tok::Name { name: "b".into() }, + Tok::Colon, + Tok::Name { name: "int64".into() }, + Tok::Newline, + ] + ); + } + + #[test] + fn test_class_lex_with_nac3comment() { + use Tok::*; + let source = "\ +class Foo(A, B): +# normal comment +# nac3: no indent + # nac3: correct indent + b: int32 + a: int32 # nac3: no need indent + def __init__(self): + pass"; + let tokens = lex_source(source); + assert_eq!( + tokens, + vec![ + Class, + Name { name: "Foo".into() }, + Lpar, + Name { name: "A".into() }, + Comma, + Name { name: "B".into() }, + Rpar, + Colon, + Newline, + ConfigComment { content: "no indent".into() }, + Newline, + Indent, + ConfigComment { content: "correct indent".into() }, + Newline, + Name { name: "b".into() }, + Colon, + Name { name: "int32".into() }, + Newline, + Name { name: "a".into() }, + Colon, + Name { name: "int32".into() }, + ConfigComment { content: "no need indent".into() }, + Newline, + Def, + Name { name: "__init__".into() }, + Lpar, + Name { name: "self".into() }, + Rpar, + Colon, + Newline, + Indent, + Pass, + Newline, + Dedent, + Dedent + ] + ) + } + #[test] fn test_newline_processor() { // Escape \ followed by \n (by removal): diff --git a/nac3parser/src/lib.rs b/nac3parser/src/lib.rs index cab63529a..5e2530593 100644 --- a/nac3parser/src/lib.rs +++ b/nac3parser/src/lib.rs @@ -32,3 +32,4 @@ lalrpop_mod!( python ); pub mod token; +pub mod config_comment_helper; diff --git a/nac3parser/src/python.lalrpop b/nac3parser/src/python.lalrpop index 7a9947900..5ca7a56b7 100644 --- a/nac3parser/src/python.lalrpop +++ b/nac3parser/src/python.lalrpop @@ -9,8 +9,11 @@ use crate::ast; use crate::fstring::parse_located_fstring; use crate::function::{ArgumentList, parse_args, parse_params}; use crate::error::LexicalError; +use crate::error::LexicalErrorType; use crate::lexer; +use crate::config_comment_helper::*; +use lalrpop_util::ParseError; use num_bigint::BigInt; grammar; @@ -47,10 +50,11 @@ Statement: ast::Suite = { }; SimpleStatement: ast::Suite = { - ";"? "\n" => { + ";"? "\n" =>? { let mut statements = vec![s1]; statements.extend(s2.into_iter().map(|e| e.1)); - statements + handle_small_stmt(&mut statements, nac3com_above, nac3com_end, com_loc)?; + Ok(statements) } }; @@ -70,7 +74,7 @@ PassStatement: ast::Stmt = { ast::Stmt { location, custom: (), - node: ast::StmtKind::Pass, + node: ast::StmtKind::Pass { config_comment: vec![] }, } }, }; @@ -80,7 +84,7 @@ DelStatement: ast::Stmt = { ast::Stmt { location, custom: (), - node: ast::StmtKind::Delete { targets }, + node: ast::StmtKind::Delete { targets, config_comment: vec![] }, } }, }; @@ -92,7 +96,7 @@ ExpressionStatement: ast::Stmt = { ast::Stmt { custom: (), location, - node: ast::StmtKind::Expr { value: Box::new(expression) } + node: ast::StmtKind::Expr { value: Box::new(expression), config_comment: vec![] } } } else { let mut targets = vec![expression]; @@ -107,7 +111,7 @@ ExpressionStatement: ast::Stmt = { ast::Stmt { custom: (), location, - node: ast::StmtKind::Assign { targets, value, type_comment: None }, + node: ast::StmtKind::Assign { targets, value, type_comment: None, config_comment: vec![] }, } } }, @@ -118,7 +122,8 @@ ExpressionStatement: ast::Stmt = { node: ast::StmtKind::AugAssign { target: Box::new(target), op, - value: Box::new(rhs) + value: Box::new(rhs), + config_comment: vec![], }, } }, @@ -132,6 +137,7 @@ ExpressionStatement: ast::Stmt = { annotation: Box::new(annotation), value: rhs.map(Box::new), simple, + config_comment: vec![], }, } }, @@ -187,28 +193,28 @@ FlowStatement: ast::Stmt = { ast::Stmt { custom: (), location, - node: ast::StmtKind::Break, + node: ast::StmtKind::Break { config_comment: vec![] }, } }, "continue" => { ast::Stmt { custom: (), location, - node: ast::StmtKind::Continue, + node: ast::StmtKind::Continue { config_comment: vec![] }, } }, "return" => { ast::Stmt { custom: (), location, - node: ast::StmtKind::Return { value: value.map(Box::new) }, + node: ast::StmtKind::Return { value: value.map(Box::new), config_comment: vec![] }, } }, => { ast::Stmt { custom: (), location, - node: ast::StmtKind::Expr { value: Box::new(expression) }, + node: ast::StmtKind::Expr { value: Box::new(expression), config_comment: vec![] }, } }, RaiseStatement, @@ -219,14 +225,14 @@ RaiseStatement: ast::Stmt = { ast::Stmt { custom: (), location, - node: ast::StmtKind::Raise { exc: None, cause: None }, + node: ast::StmtKind::Raise { exc: None, cause: None, config_comment: vec![] }, } }, "raise" => { ast::Stmt { custom: (), location, - node: ast::StmtKind::Raise { exc: Some(Box::new(t)), cause: c.map(|x| Box::new(x.1)) }, + node: ast::StmtKind::Raise { exc: Some(Box::new(t)), cause: c.map(|x| Box::new(x.1)), config_comment: vec![] }, } }, }; @@ -236,7 +242,7 @@ ImportStatement: ast::Stmt = { ast::Stmt { custom: (), location, - node: ast::StmtKind::Import { names }, + node: ast::StmtKind::Import { names, config_comment: vec![] }, } }, "from" "import" => { @@ -247,7 +253,8 @@ ImportStatement: ast::Stmt = { node: ast::StmtKind::ImportFrom { level, module: module.map(|s| s.into()), - names + names, + config_comment: vec![] }, } }, @@ -301,7 +308,7 @@ GlobalStatement: ast::Stmt = { ast::Stmt { custom: (), location, - node: ast::StmtKind::Global { names } + node: ast::StmtKind::Global { names, config_comment: vec![] } } }, }; @@ -311,7 +318,7 @@ NonlocalStatement: ast::Stmt = { ast::Stmt { custom: (), location, - node: ast::StmtKind::Nonlocal { names } + node: ast::StmtKind::Nonlocal { names, config_comment: vec![] } } }, }; @@ -323,7 +330,8 @@ AssertStatement: ast::Stmt = { location, node: ast::StmtKind::Assert { test: Box::new(test), - msg: msg.map(|e| Box::new(e.1)) + msg: msg.map(|e| Box::new(e.1)), + config_comment: vec![], } } }, @@ -340,7 +348,7 @@ CompoundStatement: ast::Stmt = { }; IfStatement: ast::Stmt = { - "if" ":" => { + "if" ":" =>? { // Determine last else: let mut last = s3.map(|s| s.2).unwrap_or_default(); @@ -349,54 +357,74 @@ IfStatement: ast::Stmt = { let x = ast::Stmt { custom: (), location: i.0, - node: ast::StmtKind::If { test: Box::new(i.2), body: i.4, orelse: last }, + node: ast::StmtKind::If { test: Box::new(i.2), body: i.4, orelse: last, config_comment: vec![] }, }; last = vec![x]; } - ast::Stmt { + Ok(ast::Stmt { custom: (), location, - node: ast::StmtKind::If { test: Box::new(test), body, orelse: last } - } + node: ast::StmtKind::If { + test: Box::new(test), + body, + orelse: last, + config_comment: make_config_comment(location, stmt_loc, nac3com_above, nac3com_end)? + } + }) }, }; WhileStatement: ast::Stmt = { - "while" ":" => { + "while" ":" =>? { let orelse = s2.map(|s| s.2).unwrap_or_default(); - ast::Stmt { + Ok(ast::Stmt { custom: (), location, node: ast::StmtKind::While { test: Box::new(test), body, - orelse + orelse, + config_comment: make_config_comment(location, stmt_loc, nac3com_above, nac3com_end)? }, - } + }) }, }; ForStatement: ast::Stmt = { - "for" "in" ":" => { + "for" "in" ":" =>? { let orelse = s2.map(|s| s.2).unwrap_or_default(); let target = Box::new(target); let iter = Box::new(iter); let type_comment = None; let node = if is_async.is_some() { - ast::StmtKind::AsyncFor { target, iter, body, orelse, type_comment } + ast::StmtKind::AsyncFor { + target, + iter, + body, + orelse, + type_comment, + config_comment: make_config_comment(location, stmt_loc, nac3com_above, nac3com_end)? + } } else { - ast::StmtKind::For { target, iter, body, orelse, type_comment } + ast::StmtKind::For { + target, + iter, + body, + orelse, + type_comment, + config_comment: make_config_comment(location, stmt_loc, nac3com_above, nac3com_end)? + } }; - ast::Stmt::new(location, node) + Ok(ast::Stmt::new(location, node)) }, }; TryStatement: ast::Stmt = { - "try" ":" => { + "try" ":" =>? { let orelse = else_suite.map(|s| s.2).unwrap_or_default(); let finalbody = finally.map(|s| s.2).unwrap_or_default(); - ast::Stmt { + Ok(ast::Stmt { custom: (), location, node: ast::StmtKind::Try { @@ -404,14 +432,15 @@ TryStatement: ast::Stmt = { handlers, orelse, finalbody, + config_comment: make_config_comment(location, stmt_loc, nac3com_above, nac3com_end)? }, - } + }) }, - "try" ":" => { + "try" ":" =>? { let handlers = vec![]; let orelse = vec![]; let finalbody = finally.2; - ast::Stmt { + Ok(ast::Stmt { custom: (), location, node: ast::StmtKind::Try { @@ -419,8 +448,9 @@ TryStatement: ast::Stmt = { handlers, orelse, finalbody, + config_comment: make_config_comment(location, stmt_loc, nac3com_above, nac3com_end)? }, - } + }) }, }; @@ -448,14 +478,24 @@ ExceptClause: ast::Excepthandler = { }; WithStatement: ast::Stmt = { - "with" > ":" => { + "with" > ":" =>? { let type_comment = None; let node = if is_async.is_some() { - ast::StmtKind::AsyncWith { items, body, type_comment } + ast::StmtKind::AsyncWith { + items, + body, + type_comment, + config_comment: make_config_comment(location, stmt_loc, nac3com_above, nac3com_end)? + } } else { - ast::StmtKind::With { items, body, type_comment } + ast::StmtKind::With { + items, + body, + type_comment, + config_comment: make_config_comment(location, stmt_loc, nac3com_above, nac3com_end)? + } }; - ast::Stmt::new(location, node) + Ok(ast::Stmt::new(location, node)) }, }; @@ -468,16 +508,32 @@ WithItem: ast::Withitem = { }; FuncDef: ast::Stmt = { - "def" " Test)?> ":" => { + "def" " Test)?> ":" =>? { let args = Box::new(args); let returns = r.map(|x| Box::new(x.1)); let type_comment = None; let node = if is_async.is_some() { - ast::StmtKind::AsyncFunctionDef { name, args, body, decorator_list, returns, type_comment } + ast::StmtKind::AsyncFunctionDef { + name, + args, + body, + decorator_list, + returns, + type_comment, + config_comment: make_config_comment(location, stmt_loc, nac3com_above, nac3com_end)? + } } else { - ast::StmtKind::FunctionDef { name, args, body, decorator_list, returns, type_comment } + ast::StmtKind::FunctionDef { + name, + args, + body, + decorator_list, + returns, + type_comment, + config_comment: make_config_comment(location, stmt_loc, nac3com_above, nac3com_end)? + } }; - ast::Stmt::new(location, node) + Ok(ast::Stmt::new(location, node)) }, }; @@ -615,12 +671,12 @@ KwargParameter: Option> = { }; ClassDef: ast::Stmt = { - "class" ":" => { + "class" ":" =>? { let (bases, keywords) = match a { Some((_, arg, _)) => (arg.args, arg.keywords), None => (vec![], vec![]), }; - ast::Stmt { + Ok(ast::Stmt { custom: (), location, node: ast::StmtKind::ClassDef { @@ -629,8 +685,9 @@ ClassDef: ast::Stmt = { keywords, body, decorator_list, + config_comment: make_config_comment(location, stmt_loc, nac3com_above, nac3com_end)? }, - } + }) }, }; @@ -1301,6 +1358,7 @@ extern { string => lexer::Tok::String { value: , is_fstring: }, bytes => lexer::Tok::Bytes { value: > }, name => lexer::Tok::Name { name: }, + config_comment => lexer::Tok::ConfigComment { content: }, "\n" => lexer::Tok::Newline, ";" => lexer::Tok::Semi, } diff --git a/nac3parser/src/token.rs b/nac3parser/src/token.rs index 73249db2e..163412aa7 100644 --- a/nac3parser/src/token.rs +++ b/nac3parser/src/token.rs @@ -13,6 +13,7 @@ pub enum Tok { Complex { real: f64, imag: f64 }, String { value: String, is_fstring: bool }, Bytes { value: Vec }, + ConfigComment { content: ast::StrRef }, Newline, Indent, Dedent, @@ -134,6 +135,7 @@ impl fmt::Display for Tok { } f.write_str("\"") } + ConfigComment { content } => write!(f, "ConfigComment: '{}'", ast::get_str_from_ref(&ast::get_str_ref_lock(), *content)), Newline => f.write_str("Newline"), Indent => f.write_str("Indent"), Dedent => f.write_str("Dedent"), From 694c7e945c9a7996b0400c4bb1ab440fa43f5e61 Mon Sep 17 00:00:00 2001 From: ychenfo Date: Thu, 4 Nov 2021 15:01:50 +0800 Subject: [PATCH 2/4] nac3ast: generated ast with comment fields --- nac3ast/src/ast_gen.rs | 111 ++++++++++++++++++++++++++++++----------- 1 file changed, 82 insertions(+), 29 deletions(-) diff --git a/nac3ast/src/ast_gen.rs b/nac3ast/src/ast_gen.rs index 812944677..f3a2d0a38 100644 --- a/nac3ast/src/ast_gen.rs +++ b/nac3ast/src/ast_gen.rs @@ -72,7 +72,7 @@ pub fn get_str_from_ref<'a>(lock: &'a MutexGuard, id: StrRef) -> &'a s lock.resolve(id.0).unwrap() } -type Ident = StrRef; +pub type Ident = StrRef; #[derive(Clone, Debug, PartialEq)] pub struct Located { @@ -114,6 +114,7 @@ pub enum StmtKind { decorator_list: Vec>, returns: Option>>, type_comment: Option, + config_comment: Vec, }, AsyncFunctionDef { name: Ident, @@ -122,6 +123,7 @@ pub enum StmtKind { decorator_list: Vec>, returns: Option>>, type_comment: Option, + config_comment: Vec, }, ClassDef { name: Ident, @@ -129,28 +131,34 @@ pub enum StmtKind { keywords: Vec>, body: Vec>, decorator_list: Vec>, + config_comment: Vec, }, Return { value: Option>>, + config_comment: Vec, }, Delete { targets: Vec>, + config_comment: Vec, }, Assign { targets: Vec>, value: Box>, type_comment: Option, + config_comment: Vec, }, AugAssign { target: Box>, op: Operator, value: Box>, + config_comment: Vec, }, AnnAssign { target: Box>, annotation: Box>, value: Option>>, simple: bool, + config_comment: Vec, }, For { target: Box>, @@ -158,6 +166,7 @@ pub enum StmtKind { body: Vec>, orelse: Vec>, type_comment: Option, + config_comment: Vec, }, AsyncFor { target: Box>, @@ -165,61 +174,80 @@ pub enum StmtKind { body: Vec>, orelse: Vec>, type_comment: Option, + config_comment: Vec, }, While { test: Box>, body: Vec>, orelse: Vec>, + config_comment: Vec, }, If { test: Box>, body: Vec>, orelse: Vec>, + config_comment: Vec, }, With { items: Vec>, body: Vec>, type_comment: Option, + config_comment: Vec, }, AsyncWith { items: Vec>, body: Vec>, type_comment: Option, + config_comment: Vec, }, Raise { exc: Option>>, cause: Option>>, + config_comment: Vec, }, Try { body: Vec>, handlers: Vec>, orelse: Vec>, finalbody: Vec>, + config_comment: Vec, }, Assert { test: Box>, msg: Option>>, + config_comment: Vec, }, Import { names: Vec, + config_comment: Vec, }, ImportFrom { module: Option, names: Vec, level: usize, + config_comment: Vec, }, Global { names: Vec, + config_comment: Vec, }, Nonlocal { names: Vec, + config_comment: Vec, }, Expr { value: Box>, + config_comment: Vec, + }, + Pass { + config_comment: Vec, + }, + Break { + config_comment: Vec, + }, + Continue { + config_comment: Vec, }, - Pass, - Break, - Continue, } pub type Stmt = Located, U>; @@ -558,7 +586,7 @@ pub mod fold { pub fn fold_stmt + ?Sized>(#[allow(unused)] folder: &mut F, node: Stmt) -> Result, F::Error> { fold_located(folder, node, |folder, node| { match node { - StmtKind::FunctionDef { name,args,body,decorator_list,returns,type_comment } => { + StmtKind::FunctionDef { name,args,body,decorator_list,returns,type_comment,config_comment } => { Ok(StmtKind::FunctionDef { name: Foldable::fold(name, folder)?, args: Foldable::fold(args, folder)?, @@ -566,9 +594,10 @@ pub mod fold { decorator_list: Foldable::fold(decorator_list, folder)?, returns: Foldable::fold(returns, folder)?, type_comment: Foldable::fold(type_comment, folder)?, + config_comment: Foldable::fold(config_comment, folder)?, }) } - StmtKind::AsyncFunctionDef { name,args,body,decorator_list,returns,type_comment } => { + StmtKind::AsyncFunctionDef { name,args,body,decorator_list,returns,type_comment,config_comment } => { Ok(StmtKind::AsyncFunctionDef { name: Foldable::fold(name, folder)?, args: Foldable::fold(args, folder)?, @@ -576,152 +605,176 @@ pub mod fold { decorator_list: Foldable::fold(decorator_list, folder)?, returns: Foldable::fold(returns, folder)?, type_comment: Foldable::fold(type_comment, folder)?, + config_comment: Foldable::fold(config_comment, folder)?, }) } - StmtKind::ClassDef { name,bases,keywords,body,decorator_list } => { + StmtKind::ClassDef { name,bases,keywords,body,decorator_list,config_comment } => { Ok(StmtKind::ClassDef { name: Foldable::fold(name, folder)?, bases: Foldable::fold(bases, folder)?, keywords: Foldable::fold(keywords, folder)?, body: Foldable::fold(body, folder)?, decorator_list: Foldable::fold(decorator_list, folder)?, + config_comment: Foldable::fold(config_comment, folder)?, }) } - StmtKind::Return { value } => { + StmtKind::Return { value,config_comment } => { Ok(StmtKind::Return { value: Foldable::fold(value, folder)?, + config_comment: Foldable::fold(config_comment, folder)?, }) } - StmtKind::Delete { targets } => { + StmtKind::Delete { targets,config_comment } => { Ok(StmtKind::Delete { targets: Foldable::fold(targets, folder)?, + config_comment: Foldable::fold(config_comment, folder)?, }) } - StmtKind::Assign { targets,value,type_comment } => { + StmtKind::Assign { targets,value,type_comment,config_comment } => { Ok(StmtKind::Assign { targets: Foldable::fold(targets, folder)?, value: Foldable::fold(value, folder)?, type_comment: Foldable::fold(type_comment, folder)?, + config_comment: Foldable::fold(config_comment, folder)?, }) } - StmtKind::AugAssign { target,op,value } => { + StmtKind::AugAssign { target,op,value,config_comment } => { Ok(StmtKind::AugAssign { target: Foldable::fold(target, folder)?, op: Foldable::fold(op, folder)?, value: Foldable::fold(value, folder)?, + config_comment: Foldable::fold(config_comment, folder)?, }) } - StmtKind::AnnAssign { target,annotation,value,simple } => { + StmtKind::AnnAssign { target,annotation,value,simple,config_comment } => { Ok(StmtKind::AnnAssign { target: Foldable::fold(target, folder)?, annotation: Foldable::fold(annotation, folder)?, value: Foldable::fold(value, folder)?, simple: Foldable::fold(simple, folder)?, + config_comment: Foldable::fold(config_comment, folder)?, }) } - StmtKind::For { target,iter,body,orelse,type_comment } => { + StmtKind::For { target,iter,body,orelse,type_comment,config_comment } => { Ok(StmtKind::For { target: Foldable::fold(target, folder)?, iter: Foldable::fold(iter, folder)?, body: Foldable::fold(body, folder)?, orelse: Foldable::fold(orelse, folder)?, type_comment: Foldable::fold(type_comment, folder)?, + config_comment: Foldable::fold(config_comment, folder)?, }) } - StmtKind::AsyncFor { target,iter,body,orelse,type_comment } => { + StmtKind::AsyncFor { target,iter,body,orelse,type_comment,config_comment } => { Ok(StmtKind::AsyncFor { target: Foldable::fold(target, folder)?, iter: Foldable::fold(iter, folder)?, body: Foldable::fold(body, folder)?, orelse: Foldable::fold(orelse, folder)?, type_comment: Foldable::fold(type_comment, folder)?, + config_comment: Foldable::fold(config_comment, folder)?, }) } - StmtKind::While { test,body,orelse } => { + StmtKind::While { test,body,orelse,config_comment } => { Ok(StmtKind::While { test: Foldable::fold(test, folder)?, body: Foldable::fold(body, folder)?, orelse: Foldable::fold(orelse, folder)?, + config_comment: Foldable::fold(config_comment, folder)?, }) } - StmtKind::If { test,body,orelse } => { + StmtKind::If { test,body,orelse,config_comment } => { Ok(StmtKind::If { test: Foldable::fold(test, folder)?, body: Foldable::fold(body, folder)?, orelse: Foldable::fold(orelse, folder)?, + config_comment: Foldable::fold(config_comment, folder)?, }) } - StmtKind::With { items,body,type_comment } => { + StmtKind::With { items,body,type_comment,config_comment } => { Ok(StmtKind::With { items: Foldable::fold(items, folder)?, body: Foldable::fold(body, folder)?, type_comment: Foldable::fold(type_comment, folder)?, + config_comment: Foldable::fold(config_comment, folder)?, }) } - StmtKind::AsyncWith { items,body,type_comment } => { + StmtKind::AsyncWith { items,body,type_comment,config_comment } => { Ok(StmtKind::AsyncWith { items: Foldable::fold(items, folder)?, body: Foldable::fold(body, folder)?, type_comment: Foldable::fold(type_comment, folder)?, + config_comment: Foldable::fold(config_comment, folder)?, }) } - StmtKind::Raise { exc,cause } => { + StmtKind::Raise { exc,cause,config_comment } => { Ok(StmtKind::Raise { exc: Foldable::fold(exc, folder)?, cause: Foldable::fold(cause, folder)?, + config_comment: Foldable::fold(config_comment, folder)?, }) } - StmtKind::Try { body,handlers,orelse,finalbody } => { + StmtKind::Try { body,handlers,orelse,finalbody,config_comment } => { Ok(StmtKind::Try { body: Foldable::fold(body, folder)?, handlers: Foldable::fold(handlers, folder)?, orelse: Foldable::fold(orelse, folder)?, finalbody: Foldable::fold(finalbody, folder)?, + config_comment: Foldable::fold(config_comment, folder)?, }) } - StmtKind::Assert { test,msg } => { + StmtKind::Assert { test,msg,config_comment } => { Ok(StmtKind::Assert { test: Foldable::fold(test, folder)?, msg: Foldable::fold(msg, folder)?, + config_comment: Foldable::fold(config_comment, folder)?, }) } - StmtKind::Import { names } => { + StmtKind::Import { names,config_comment } => { Ok(StmtKind::Import { names: Foldable::fold(names, folder)?, + config_comment: Foldable::fold(config_comment, folder)?, }) } - StmtKind::ImportFrom { module,names,level } => { + StmtKind::ImportFrom { module,names,level,config_comment } => { Ok(StmtKind::ImportFrom { module: Foldable::fold(module, folder)?, names: Foldable::fold(names, folder)?, level: Foldable::fold(level, folder)?, + config_comment: Foldable::fold(config_comment, folder)?, }) } - StmtKind::Global { names } => { + StmtKind::Global { names,config_comment } => { Ok(StmtKind::Global { names: Foldable::fold(names, folder)?, + config_comment: Foldable::fold(config_comment, folder)?, }) } - StmtKind::Nonlocal { names } => { + StmtKind::Nonlocal { names,config_comment } => { Ok(StmtKind::Nonlocal { names: Foldable::fold(names, folder)?, + config_comment: Foldable::fold(config_comment, folder)?, }) } - StmtKind::Expr { value } => { + StmtKind::Expr { value,config_comment } => { Ok(StmtKind::Expr { value: Foldable::fold(value, folder)?, + config_comment: Foldable::fold(config_comment, folder)?, }) } - StmtKind::Pass { } => { + StmtKind::Pass { config_comment } => { Ok(StmtKind::Pass { + config_comment: Foldable::fold(config_comment, folder)?, }) } - StmtKind::Break { } => { + StmtKind::Break { config_comment } => { Ok(StmtKind::Break { + config_comment: Foldable::fold(config_comment, folder)?, }) } - StmtKind::Continue { } => { + StmtKind::Continue { config_comment } => { Ok(StmtKind::Continue { + config_comment: Foldable::fold(config_comment, folder)?, }) } } From b239806558bddd020fc320d89b63c4b7fe058d3e Mon Sep 17 00:00:00 2001 From: ychenfo Date: Thu, 4 Nov 2021 15:02:51 +0800 Subject: [PATCH 3/4] nac3core: adapt to ast change due to comment support --- nac3core/src/codegen/stmt.rs | 14 +++++++------- nac3core/src/toplevel/composer.rs | 4 ++-- nac3core/src/toplevel/helper.rs | 2 +- nac3core/src/typecheck/function_check.rs | 8 ++++---- nac3core/src/typecheck/type_inferencer/mod.rs | 11 ++++++----- 5 files changed, 20 insertions(+), 19 deletions(-) diff --git a/nac3core/src/codegen/stmt.rs b/nac3core/src/codegen/stmt.rs index 0ad9cdd1a..3688b7b0b 100644 --- a/nac3core/src/codegen/stmt.rs +++ b/nac3core/src/codegen/stmt.rs @@ -186,7 +186,7 @@ pub fn gen_while<'ctx, 'a, G: CodeGenerator + ?Sized>( ctx: &mut CodeGenContext<'ctx, 'a>, stmt: &Stmt>, ) { - if let StmtKind::While { test, body, orelse } = &stmt.node { + if let StmtKind::While { test, body, orelse, .. } = &stmt.node { let current = ctx.builder.get_insert_block().unwrap().get_parent().unwrap(); let test_bb = ctx.ctx.append_basic_block(current, "test"); let body_bb = ctx.ctx.append_basic_block(current, "body"); @@ -228,7 +228,7 @@ pub fn gen_if<'ctx, 'a, G: CodeGenerator + ?Sized>( ctx: &mut CodeGenContext<'ctx, 'a>, stmt: &Stmt>, ) -> bool { - if let StmtKind::If { test, body, orelse } = &stmt.node { + if let StmtKind::If { test, body, orelse, .. } = &stmt.node { let current = ctx.builder.get_insert_block().unwrap().get_parent().unwrap(); let test_bb = ctx.ctx.append_basic_block(current, "test"); let body_bb = ctx.ctx.append_basic_block(current, "body"); @@ -306,11 +306,11 @@ pub fn gen_stmt<'ctx, 'a, G: CodeGenerator + ?Sized>( stmt: &Stmt>, ) -> bool { match &stmt.node { - StmtKind::Pass => {} - StmtKind::Expr { value } => { + StmtKind::Pass { .. } => {} + StmtKind::Expr { value, .. } => { generator.gen_expr(ctx, value); } - StmtKind::Return { value } => { + StmtKind::Return { value, .. } => { let value = value.as_ref().map(|v| generator.gen_expr(ctx, v).unwrap()); let value = value.as_ref().map(|v| v as &dyn BasicValue); ctx.builder.build_return(value); @@ -328,11 +328,11 @@ pub fn gen_stmt<'ctx, 'a, G: CodeGenerator + ?Sized>( generator.gen_assign(ctx, target, value); } } - StmtKind::Continue => { + StmtKind::Continue { .. } => { ctx.builder.build_unconditional_branch(ctx.loop_bb.unwrap().0); return true; } - StmtKind::Break => { + StmtKind::Break { .. }=> { ctx.builder.build_unconditional_branch(ctx.loop_bb.unwrap().1); return true; } diff --git a/nac3core/src/toplevel/composer.rs b/nac3core/src/toplevel/composer.rs index 604262ddd..343a68fe5 100644 --- a/nac3core/src/toplevel/composer.rs +++ b/nac3core/src/toplevel/composer.rs @@ -1395,8 +1395,8 @@ impl TopLevelComposer { return Err("unsupported statement type in class definition body".into()); } } - ast::StmtKind::Pass => {} - ast::StmtKind::Expr { value: _ } => {} // typically a docstring; ignoring all expressions matches CPython behavior + ast::StmtKind::Pass { .. } => {} + ast::StmtKind::Expr { value: _, .. } => {} // typically a docstring; ignoring all expressions matches CPython behavior _ => return Err("unsupported statement type in class definition body".into()), } } diff --git a/nac3core/src/toplevel/helper.rs b/nac3core/src/toplevel/helper.rs index ec4bd1849..df7922347 100644 --- a/nac3core/src/toplevel/helper.rs +++ b/nac3core/src/toplevel/helper.rs @@ -337,7 +337,7 @@ impl TopLevelComposer { ast::StmtKind::With { body, .. } => { result.extend(Self::get_all_assigned_field(body.as_slice())?); } - ast::StmtKind::Pass => {} + ast::StmtKind::Pass { .. } => {} ast::StmtKind::Assert { .. } => {} ast::StmtKind::Expr { .. } => {} diff --git a/nac3core/src/typecheck/function_check.rs b/nac3core/src/typecheck/function_check.rs index 5b6812973..8b4323935 100644 --- a/nac3core/src/typecheck/function_check.rs +++ b/nac3core/src/typecheck/function_check.rs @@ -196,7 +196,7 @@ impl<'a> Inferencer<'a> { } Ok(false) } - StmtKind::If { test, body, orelse } => { + StmtKind::If { test, body, orelse, .. } => { self.check_expr(test, defined_identifiers)?; self.should_have_value(test)?; let mut body_identifiers = defined_identifiers.clone(); @@ -211,7 +211,7 @@ impl<'a> Inferencer<'a> { } Ok(body_returned && orelse_returned) } - StmtKind::While { test, body, orelse } => { + StmtKind::While { test, body, orelse, .. } => { self.check_expr(test, defined_identifiers)?; self.should_have_value(test)?; let mut defined_identifiers = defined_identifiers.clone(); @@ -230,7 +230,7 @@ impl<'a> Inferencer<'a> { self.check_block(body, &mut new_defined_identifiers)?; Ok(false) } - StmtKind::Expr { value } => { + StmtKind::Expr { value, .. } => { self.check_expr(value, defined_identifiers)?; Ok(false) } @@ -250,7 +250,7 @@ impl<'a> Inferencer<'a> { } Ok(false) } - StmtKind::Return { value } => { + StmtKind::Return { value, .. } => { if let Some(value) = value { self.check_expr(value, defined_identifiers)?; self.should_have_value(value)?; diff --git a/nac3core/src/typecheck/type_inferencer/mod.rs b/nac3core/src/typecheck/type_inferencer/mod.rs index 1a8fa6385..26f544d89 100644 --- a/nac3core/src/typecheck/type_inferencer/mod.rs +++ b/nac3core/src/typecheck/type_inferencer/mod.rs @@ -80,7 +80,7 @@ impl<'a> fold::Fold<()> for Inferencer<'a> { fn fold_stmt(&mut self, node: ast::Stmt<()>) -> Result, Self::Error> { let stmt = match node.node { // we don't want fold over type annotation - ast::StmtKind::AnnAssign { target, annotation, value, simple } => { + ast::StmtKind::AnnAssign { target, annotation, value, simple, config_comment } => { self.infer_pattern(&target)?; let target = Box::new(self.fold_expr(*target)?); let value = if let Some(v) = value { @@ -105,14 +105,14 @@ impl<'a> fold::Fold<()> for Inferencer<'a> { Located { location: node.location, custom: None, - node: ast::StmtKind::AnnAssign { target, annotation, value, simple }, + node: ast::StmtKind::AnnAssign { target, annotation, value, simple, config_comment }, } } ast::StmtKind::For { ref target, .. } => { self.infer_pattern(target)?; fold::fold_stmt(self, node)? } - ast::StmtKind::Assign { ref targets, .. } => { + ast::StmtKind::Assign { ref targets, ref config_comment, .. } => { if targets.iter().all(|t| matches!(t.node, ast::ExprKind::Name { .. })) { if let ast::StmtKind::Assign { targets, value, .. } = node.node { let value = self.fold_expr(*value)?; @@ -158,6 +158,7 @@ impl<'a> fold::Fold<()> for Inferencer<'a> { targets, value: Box::new(value), type_comment: None, + config_comment: config_comment.clone() }, custom: None, }); @@ -198,7 +199,7 @@ impl<'a> fold::Fold<()> for Inferencer<'a> { } } ast::StmtKind::AnnAssign { .. } | ast::StmtKind::Expr { .. } => {} - ast::StmtKind::Break | ast::StmtKind::Continue | ast::StmtKind::Pass => {} + ast::StmtKind::Break { .. } | ast::StmtKind::Continue { .. } | ast::StmtKind::Pass { .. } => {} ast::StmtKind::With { items, .. } => { for item in items.iter() { let ty = item.context_expr.custom.unwrap(); @@ -272,7 +273,7 @@ impl<'a> fold::Fold<()> for Inferencer<'a> { } } } - ast::StmtKind::Return { value } => match (value, self.function_data.return_type) { + ast::StmtKind::Return { value, .. } => match (value, self.function_data.return_type) { (Some(v), Some(v1)) => { self.unify(v.custom.unwrap(), v1, &v.location)?; } From 032e1d84cff332cc604b0bd52c8b152026f561e9 Mon Sep 17 00:00:00 2001 From: ychenfo Date: Thu, 4 Nov 2021 15:03:34 +0800 Subject: [PATCH 4/4] nac3parser: add and fix tests due to comment support --- nac3parser/src/parser.rs | 63 +++ ...ser__parser__tests__comment_ambiguity.snap | 448 ++++++++++++++++++ ...c3parser__parser__tests__more_comment.snap | 313 ++++++++++++ ...ac3parser__parser__tests__parse_class.snap | 14 +- ...er__parser__tests__parse_if_elif_else.snap | 7 +- ...c3parser__parser__tests__parse_kwargs.snap | 3 +- ...c3parser__parser__tests__parse_lambda.snap | 4 +- ...3parser__parser__tests__parse_print_2.snap | 3 +- ...ser__parser__tests__parse_print_hello.snap | 4 +- ...c3parser__parser__tests__parse_tuples.snap | 3 +- ...parser__parser__tests__sample_comment.snap | 75 +++ 11 files changed, 928 insertions(+), 9 deletions(-) create mode 100644 nac3parser/src/snapshots/nac3parser__parser__tests__comment_ambiguity.snap create mode 100644 nac3parser/src/snapshots/nac3parser__parser__tests__more_comment.snap create mode 100644 nac3parser/src/snapshots/nac3parser__parser__tests__sample_comment.snap diff --git a/nac3parser/src/parser.rs b/nac3parser/src/parser.rs index 3df481b2e..825a14c77 100644 --- a/nac3parser/src/parser.rs +++ b/nac3parser/src/parser.rs @@ -164,4 +164,67 @@ class Foo(A, B): let parse_ast = parse_expression(&source).unwrap(); insta::assert_debug_snapshot!(parse_ast); } + + #[test] + fn test_more_comment() { + let source = "\ +a: int # nac3: sf1 +# nac3: sdf4 +for i in (1, '12'): # nac3: sf2 + a: int +# nac3: 3 +# nac3: 5 +while i < 2: # nac3: 4 + # nac3: real pass + pass + # nac3: expr1 + # nac3: expr3 + 1 + 2 # nac3: expr2 + # nac3: if3 + # nac3: if1 + if 1: # nac3: if2 + 3"; + insta::assert_debug_snapshot!(parse_program(&source).unwrap()); + } + + #[test] + fn test_sample_comment() { + let source = "\ +# nac3: while1 +# nac3: while2 +# normal comment +while test: # nac3: while3 + # nac3: simple assign0 + a = 3 # nac3: simple assign1 +"; + insta::assert_debug_snapshot!(parse_program(&source).unwrap()); + } + + #[test] + fn test_comment_ambiguity() { + let source = "\ +if a: d; # nac3: for d +if b: c # nac3: for c +if d: # nac3: for if d + b; b + 3; # nac3: for b + 3 +a = 3; a + 3; b = a; # nac3: notif +# nac3: smallsingle1 +# nac3: smallsingle3 +aa = 3 # nac3: smallsingle2 +if a: # nac3: small2 + a +for i in a: # nac3: for1 + pass +"; + insta::assert_debug_snapshot!(parse_program(&source).unwrap()); + } + + #[test] + fn test_comment_should_fail() { + let source = "\ +if a: # nac3: something +a = 3 +"; + assert!(parse_program(&source).is_err()); + } } diff --git a/nac3parser/src/snapshots/nac3parser__parser__tests__comment_ambiguity.snap b/nac3parser/src/snapshots/nac3parser__parser__tests__comment_ambiguity.snap new file mode 100644 index 000000000..e46deac7b --- /dev/null +++ b/nac3parser/src/snapshots/nac3parser__parser__tests__comment_ambiguity.snap @@ -0,0 +1,448 @@ +--- +source: nac3parser/src/parser.rs +expression: parse_program(&source).unwrap() + +--- +[ + Located { + location: Location { + row: 1, + column: 1, + }, + custom: (), + node: If { + test: Located { + location: Location { + row: 1, + column: 4, + }, + custom: (), + node: Name { + id: "a", + ctx: Load, + }, + }, + body: [ + Located { + location: Location { + row: 1, + column: 7, + }, + custom: (), + node: Expr { + value: Located { + location: Location { + row: 1, + column: 7, + }, + custom: (), + node: Name { + id: "d", + ctx: Load, + }, + }, + config_comment: [ + "for d", + ], + }, + }, + ], + orelse: [], + config_comment: [], + }, + }, + Located { + location: Location { + row: 2, + column: 1, + }, + custom: (), + node: If { + test: Located { + location: Location { + row: 2, + column: 4, + }, + custom: (), + node: Name { + id: "b", + ctx: Load, + }, + }, + body: [ + Located { + location: Location { + row: 2, + column: 7, + }, + custom: (), + node: Expr { + value: Located { + location: Location { + row: 2, + column: 7, + }, + custom: (), + node: Name { + id: "c", + ctx: Load, + }, + }, + config_comment: [ + "for c", + ], + }, + }, + ], + orelse: [], + config_comment: [], + }, + }, + Located { + location: Location { + row: 3, + column: 1, + }, + custom: (), + node: If { + test: Located { + location: Location { + row: 3, + column: 4, + }, + custom: (), + node: Name { + id: "d", + ctx: Load, + }, + }, + body: [ + Located { + location: Location { + row: 4, + column: 5, + }, + custom: (), + node: Expr { + value: Located { + location: Location { + row: 4, + column: 5, + }, + custom: (), + node: Name { + id: "b", + ctx: Load, + }, + }, + config_comment: [], + }, + }, + Located { + location: Location { + row: 4, + column: 8, + }, + custom: (), + node: Expr { + value: Located { + location: Location { + row: 4, + column: 10, + }, + custom: (), + node: BinOp { + left: Located { + location: Location { + row: 4, + column: 8, + }, + custom: (), + node: Name { + id: "b", + ctx: Load, + }, + }, + op: Add, + right: Located { + location: Location { + row: 4, + column: 12, + }, + custom: (), + node: Constant { + value: Int( + 3, + ), + kind: None, + }, + }, + }, + }, + config_comment: [ + "for b + 3", + ], + }, + }, + ], + orelse: [], + config_comment: [ + "for if d", + ], + }, + }, + Located { + location: Location { + row: 5, + column: 1, + }, + custom: (), + node: Assign { + targets: [ + Located { + location: Location { + row: 5, + column: 1, + }, + custom: (), + node: Name { + id: "a", + ctx: Load, + }, + }, + ], + value: Located { + location: Location { + row: 5, + column: 5, + }, + custom: (), + node: Constant { + value: Int( + 3, + ), + kind: None, + }, + }, + type_comment: None, + config_comment: [], + }, + }, + Located { + location: Location { + row: 5, + column: 8, + }, + custom: (), + node: Expr { + value: Located { + location: Location { + row: 5, + column: 10, + }, + custom: (), + node: BinOp { + left: Located { + location: Location { + row: 5, + column: 8, + }, + custom: (), + node: Name { + id: "a", + ctx: Load, + }, + }, + op: Add, + right: Located { + location: Location { + row: 5, + column: 12, + }, + custom: (), + node: Constant { + value: Int( + 3, + ), + kind: None, + }, + }, + }, + }, + config_comment: [], + }, + }, + Located { + location: Location { + row: 5, + column: 15, + }, + custom: (), + node: Assign { + targets: [ + Located { + location: Location { + row: 5, + column: 15, + }, + custom: (), + node: Name { + id: "b", + ctx: Load, + }, + }, + ], + value: Located { + location: Location { + row: 5, + column: 19, + }, + custom: (), + node: Name { + id: "a", + ctx: Load, + }, + }, + type_comment: None, + config_comment: [ + "notif", + ], + }, + }, + Located { + location: Location { + row: 8, + column: 1, + }, + custom: (), + node: Assign { + targets: [ + Located { + location: Location { + row: 8, + column: 1, + }, + custom: (), + node: Name { + id: "aa", + ctx: Load, + }, + }, + ], + value: Located { + location: Location { + row: 8, + column: 6, + }, + custom: (), + node: Constant { + value: Int( + 3, + ), + kind: None, + }, + }, + type_comment: None, + config_comment: [ + "smallsingle1", + "smallsingle3", + "smallsingle2", + ], + }, + }, + Located { + location: Location { + row: 9, + column: 1, + }, + custom: (), + node: If { + test: Located { + location: Location { + row: 9, + column: 4, + }, + custom: (), + node: Name { + id: "a", + ctx: Load, + }, + }, + body: [ + Located { + location: Location { + row: 10, + column: 5, + }, + custom: (), + node: Expr { + value: Located { + location: Location { + row: 10, + column: 5, + }, + custom: (), + node: Name { + id: "a", + ctx: Load, + }, + }, + config_comment: [], + }, + }, + ], + orelse: [], + config_comment: [ + "small2", + ], + }, + }, + Located { + location: Location { + row: 11, + column: 1, + }, + custom: (), + node: For { + target: Located { + location: Location { + row: 11, + column: 5, + }, + custom: (), + node: Name { + id: "i", + ctx: Load, + }, + }, + iter: Located { + location: Location { + row: 11, + column: 10, + }, + custom: (), + node: Name { + id: "a", + ctx: Load, + }, + }, + body: [ + Located { + location: Location { + row: 12, + column: 5, + }, + custom: (), + node: Pass { + config_comment: [], + }, + }, + ], + orelse: [], + type_comment: None, + config_comment: [ + "for1", + ], + }, + }, +] diff --git a/nac3parser/src/snapshots/nac3parser__parser__tests__more_comment.snap b/nac3parser/src/snapshots/nac3parser__parser__tests__more_comment.snap new file mode 100644 index 000000000..790bb7f35 --- /dev/null +++ b/nac3parser/src/snapshots/nac3parser__parser__tests__more_comment.snap @@ -0,0 +1,313 @@ +--- +source: nac3parser/src/parser.rs +expression: parse_program(&source).unwrap() + +--- +[ + Located { + location: Location { + row: 1, + column: 1, + }, + custom: (), + node: AnnAssign { + target: Located { + location: Location { + row: 1, + column: 1, + }, + custom: (), + node: Name { + id: "a", + ctx: Load, + }, + }, + annotation: Located { + location: Location { + row: 1, + column: 4, + }, + custom: (), + node: Name { + id: "int", + ctx: Load, + }, + }, + value: None, + simple: true, + config_comment: [ + "sf1", + ], + }, + }, + Located { + location: Location { + row: 2, + column: 1, + }, + custom: (), + node: For { + target: Located { + location: Location { + row: 3, + column: 5, + }, + custom: (), + node: Name { + id: "i", + ctx: Load, + }, + }, + iter: Located { + location: Location { + row: 3, + column: 11, + }, + custom: (), + node: Tuple { + elts: [ + Located { + location: Location { + row: 3, + column: 11, + }, + custom: (), + node: Constant { + value: Int( + 1, + ), + kind: None, + }, + }, + Located { + location: Location { + row: 3, + column: 15, + }, + custom: (), + node: Constant { + value: Str( + "12", + ), + kind: None, + }, + }, + ], + ctx: Load, + }, + }, + body: [ + Located { + location: Location { + row: 4, + column: 5, + }, + custom: (), + node: AnnAssign { + target: Located { + location: Location { + row: 4, + column: 5, + }, + custom: (), + node: Name { + id: "a", + ctx: Load, + }, + }, + annotation: Located { + location: Location { + row: 4, + column: 8, + }, + custom: (), + node: Name { + id: "int", + ctx: Load, + }, + }, + value: None, + simple: true, + config_comment: [], + }, + }, + ], + orelse: [], + type_comment: None, + config_comment: [ + "sdf4", + "sf2", + ], + }, + }, + Located { + location: Location { + row: 5, + column: 1, + }, + custom: (), + node: While { + test: Located { + location: Location { + row: 7, + column: 9, + }, + custom: (), + node: Compare { + left: Located { + location: Location { + row: 7, + column: 7, + }, + custom: (), + node: Name { + id: "i", + ctx: Load, + }, + }, + ops: [ + Lt, + ], + comparators: [ + Located { + location: Location { + row: 7, + column: 11, + }, + custom: (), + node: Constant { + value: Int( + 2, + ), + kind: None, + }, + }, + ], + }, + }, + body: [ + Located { + location: Location { + row: 9, + column: 5, + }, + custom: (), + node: Pass { + config_comment: [ + "real pass", + ], + }, + }, + Located { + location: Location { + row: 12, + column: 5, + }, + custom: (), + node: Expr { + value: Located { + location: Location { + row: 12, + column: 7, + }, + custom: (), + node: BinOp { + left: Located { + location: Location { + row: 12, + column: 5, + }, + custom: (), + node: Constant { + value: Int( + 1, + ), + kind: None, + }, + }, + op: Add, + right: Located { + location: Location { + row: 12, + column: 9, + }, + custom: (), + node: Constant { + value: Int( + 2, + ), + kind: None, + }, + }, + }, + }, + config_comment: [ + "expr1", + "expr3", + "expr2", + ], + }, + }, + Located { + location: Location { + row: 13, + column: 5, + }, + custom: (), + node: If { + test: Located { + location: Location { + row: 15, + column: 8, + }, + custom: (), + node: Constant { + value: Int( + 1, + ), + kind: None, + }, + }, + body: [ + Located { + location: Location { + row: 16, + column: 9, + }, + custom: (), + node: Expr { + value: Located { + location: Location { + row: 16, + column: 9, + }, + custom: (), + node: Constant { + value: Int( + 3, + ), + kind: None, + }, + }, + config_comment: [], + }, + }, + ], + orelse: [], + config_comment: [ + "if3", + "if1", + "if2", + ], + }, + }, + ], + orelse: [], + config_comment: [ + "3", + "5", + "4", + ], + }, + }, +] diff --git a/nac3parser/src/snapshots/nac3parser__parser__tests__parse_class.snap b/nac3parser/src/snapshots/nac3parser__parser__tests__parse_class.snap index 869669e9a..46501f0cb 100644 --- a/nac3parser/src/snapshots/nac3parser__parser__tests__parse_class.snap +++ b/nac3parser/src/snapshots/nac3parser__parser__tests__parse_class.snap @@ -1,6 +1,7 @@ --- -source: parser/src/parser.rs +source: nac3parser/src/parser.rs expression: parse_program(&source).unwrap() + --- [ Located { @@ -74,12 +75,15 @@ expression: parse_program(&source).unwrap() column: 3, }, custom: (), - node: Pass, + node: Pass { + config_comment: [], + }, }, ], decorator_list: [], returns: None, type_comment: None, + config_comment: [], }, }, Located { @@ -145,16 +149,20 @@ expression: parse_program(&source).unwrap() column: 3, }, custom: (), - node: Pass, + node: Pass { + config_comment: [], + }, }, ], decorator_list: [], returns: None, type_comment: None, + config_comment: [], }, }, ], decorator_list: [], + config_comment: [], }, }, ] diff --git a/nac3parser/src/snapshots/nac3parser__parser__tests__parse_if_elif_else.snap b/nac3parser/src/snapshots/nac3parser__parser__tests__parse_if_elif_else.snap index 4b34c0e42..13f0bca8a 100644 --- a/nac3parser/src/snapshots/nac3parser__parser__tests__parse_if_elif_else.snap +++ b/nac3parser/src/snapshots/nac3parser__parser__tests__parse_if_elif_else.snap @@ -1,5 +1,5 @@ --- -source: parser/src/parser.rs +source: nac3parser/src/parser.rs expression: parse_ast --- @@ -45,6 +45,7 @@ expression: parse_ast kind: None, }, }, + config_comment: [], }, }, ], @@ -90,6 +91,7 @@ expression: parse_ast kind: None, }, }, + config_comment: [], }, }, ], @@ -114,12 +116,15 @@ expression: parse_ast kind: None, }, }, + config_comment: [], }, }, ], + config_comment: [], }, }, ], + config_comment: [], }, }, ] diff --git a/nac3parser/src/snapshots/nac3parser__parser__tests__parse_kwargs.snap b/nac3parser/src/snapshots/nac3parser__parser__tests__parse_kwargs.snap index d31a3921a..6dd94aaee 100644 --- a/nac3parser/src/snapshots/nac3parser__parser__tests__parse_kwargs.snap +++ b/nac3parser/src/snapshots/nac3parser__parser__tests__parse_kwargs.snap @@ -1,5 +1,5 @@ --- -source: parser/src/parser.rs +source: nac3parser/src/parser.rs expression: parse_ast --- @@ -73,6 +73,7 @@ expression: parse_ast ], }, }, + config_comment: [], }, }, ] diff --git a/nac3parser/src/snapshots/nac3parser__parser__tests__parse_lambda.snap b/nac3parser/src/snapshots/nac3parser__parser__tests__parse_lambda.snap index 2f70292d6..a9aca7e92 100644 --- a/nac3parser/src/snapshots/nac3parser__parser__tests__parse_lambda.snap +++ b/nac3parser/src/snapshots/nac3parser__parser__tests__parse_lambda.snap @@ -1,6 +1,7 @@ --- -source: parser/src/parser.rs +source: nac3parser/src/parser.rs expression: parse_ast + --- [ Located { @@ -85,6 +86,7 @@ expression: parse_ast }, }, }, + config_comment: [], }, }, ] diff --git a/nac3parser/src/snapshots/nac3parser__parser__tests__parse_print_2.snap b/nac3parser/src/snapshots/nac3parser__parser__tests__parse_print_2.snap index 6fe4cad36..92e1c9491 100644 --- a/nac3parser/src/snapshots/nac3parser__parser__tests__parse_print_2.snap +++ b/nac3parser/src/snapshots/nac3parser__parser__tests__parse_print_2.snap @@ -1,5 +1,5 @@ --- -source: parser/src/parser.rs +source: nac3parser/src/parser.rs expression: parse_ast --- @@ -60,6 +60,7 @@ expression: parse_ast keywords: [], }, }, + config_comment: [], }, }, ] diff --git a/nac3parser/src/snapshots/nac3parser__parser__tests__parse_print_hello.snap b/nac3parser/src/snapshots/nac3parser__parser__tests__parse_print_hello.snap index b4e78d8d0..7c9a72ca7 100644 --- a/nac3parser/src/snapshots/nac3parser__parser__tests__parse_print_hello.snap +++ b/nac3parser/src/snapshots/nac3parser__parser__tests__parse_print_hello.snap @@ -1,6 +1,7 @@ --- -source: parser/src/parser.rs +source: nac3parser/src/parser.rs expression: parse_ast + --- [ Located { @@ -46,6 +47,7 @@ expression: parse_ast keywords: [], }, }, + config_comment: [], }, }, ] diff --git a/nac3parser/src/snapshots/nac3parser__parser__tests__parse_tuples.snap b/nac3parser/src/snapshots/nac3parser__parser__tests__parse_tuples.snap index 26243f562..66f2fdd64 100644 --- a/nac3parser/src/snapshots/nac3parser__parser__tests__parse_tuples.snap +++ b/nac3parser/src/snapshots/nac3parser__parser__tests__parse_tuples.snap @@ -1,5 +1,5 @@ --- -source: parser/src/parser.rs +source: nac3parser/src/parser.rs expression: parse_program(&source).unwrap() --- @@ -86,6 +86,7 @@ expression: parse_program(&source).unwrap() }, }, type_comment: None, + config_comment: [], }, }, ] diff --git a/nac3parser/src/snapshots/nac3parser__parser__tests__sample_comment.snap b/nac3parser/src/snapshots/nac3parser__parser__tests__sample_comment.snap new file mode 100644 index 000000000..8e34a64f4 --- /dev/null +++ b/nac3parser/src/snapshots/nac3parser__parser__tests__sample_comment.snap @@ -0,0 +1,75 @@ +--- +source: nac3parser/src/parser.rs +expression: parse_program(&source).unwrap() + +--- +[ + Located { + location: Location { + row: 1, + column: 1, + }, + custom: (), + node: While { + test: Located { + location: Location { + row: 4, + column: 7, + }, + custom: (), + node: Name { + id: "test", + ctx: Load, + }, + }, + body: [ + Located { + location: Location { + row: 6, + column: 5, + }, + custom: (), + node: Assign { + targets: [ + Located { + location: Location { + row: 6, + column: 5, + }, + custom: (), + node: Name { + id: "a", + ctx: Load, + }, + }, + ], + value: Located { + location: Location { + row: 6, + column: 9, + }, + custom: (), + node: Constant { + value: Int( + 3, + ), + kind: None, + }, + }, + type_comment: None, + config_comment: [ + "simple assign0", + "simple assign1", + ], + }, + }, + ], + orelse: [], + config_comment: [ + "while1", + "while2", + "while3", + ], + }, + }, +]