forked from M-Labs/nac3
pca006132
48ce6bb6c5
corresponds to M-Labs RustPython fork at efdf7829ba1a5f87d30df8eaff12a330544f3cbd branch parser-mod
229 lines
6.6 KiB
Rust
229 lines
6.6 KiB
Rust
//! Different token definitions.
|
|
//! Loosely based on token.h from CPython source:
|
|
use num_bigint::BigInt;
|
|
use std::fmt::{self, Write};
|
|
use crate::ast;
|
|
|
|
/// Python source code can be tokenized in a sequence of these tokens.
|
|
#[derive(Clone, Debug, PartialEq)]
|
|
pub enum Tok {
|
|
Name { name: ast::StrRef },
|
|
Int { value: BigInt },
|
|
Float { value: f64 },
|
|
Complex { real: f64, imag: f64 },
|
|
String { value: String, is_fstring: bool },
|
|
Bytes { value: Vec<u8> },
|
|
Newline,
|
|
Indent,
|
|
Dedent,
|
|
StartModule,
|
|
StartInteractive,
|
|
StartExpression,
|
|
EndOfFile,
|
|
Lpar,
|
|
Rpar,
|
|
Lsqb,
|
|
Rsqb,
|
|
Colon,
|
|
Comma,
|
|
Semi,
|
|
Plus,
|
|
Minus,
|
|
Star,
|
|
Slash,
|
|
Vbar, // '|'
|
|
Amper, // '&'
|
|
Less,
|
|
Greater,
|
|
Equal,
|
|
Dot,
|
|
Percent,
|
|
Lbrace,
|
|
Rbrace,
|
|
EqEqual,
|
|
NotEqual,
|
|
LessEqual,
|
|
GreaterEqual,
|
|
Tilde,
|
|
CircumFlex,
|
|
LeftShift,
|
|
RightShift,
|
|
DoubleStar,
|
|
DoubleStarEqual, // '**='
|
|
PlusEqual,
|
|
MinusEqual,
|
|
StarEqual,
|
|
SlashEqual,
|
|
PercentEqual,
|
|
AmperEqual, // '&='
|
|
VbarEqual,
|
|
CircumflexEqual, // '^='
|
|
LeftShiftEqual,
|
|
RightShiftEqual,
|
|
DoubleSlash, // '//'
|
|
DoubleSlashEqual,
|
|
ColonEqual,
|
|
At,
|
|
AtEqual,
|
|
Rarrow,
|
|
Ellipsis,
|
|
|
|
// Keywords (alphabetically):
|
|
False,
|
|
None,
|
|
True,
|
|
|
|
And,
|
|
As,
|
|
Assert,
|
|
Async,
|
|
Await,
|
|
Break,
|
|
Class,
|
|
Continue,
|
|
Def,
|
|
Del,
|
|
Elif,
|
|
Else,
|
|
Except,
|
|
Finally,
|
|
For,
|
|
From,
|
|
Global,
|
|
If,
|
|
Import,
|
|
In,
|
|
Is,
|
|
Lambda,
|
|
Nonlocal,
|
|
Not,
|
|
Or,
|
|
Pass,
|
|
Raise,
|
|
Return,
|
|
Try,
|
|
While,
|
|
With,
|
|
Yield,
|
|
}
|
|
|
|
impl fmt::Display for Tok {
|
|
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
|
|
use Tok::*;
|
|
match self {
|
|
Name { name } => write!(f, "'{}'", ast::get_str_from_ref(&ast::get_str_ref_lock(), *name)),
|
|
Int { value } => write!(f, "'{}'", value),
|
|
Float { value } => write!(f, "'{}'", value),
|
|
Complex { real, imag } => write!(f, "{}j{}", real, imag),
|
|
String { value, is_fstring } => {
|
|
if *is_fstring {
|
|
write!(f, "f")?
|
|
}
|
|
write!(f, "{:?}", value)
|
|
}
|
|
Bytes { value } => {
|
|
write!(f, "b\"")?;
|
|
for i in value {
|
|
match i {
|
|
9 => f.write_str("\\t")?,
|
|
10 => f.write_str("\\n")?,
|
|
13 => f.write_str("\\r")?,
|
|
32..=126 => f.write_char(*i as char)?,
|
|
_ => write!(f, "\\x{:02x}", i)?,
|
|
}
|
|
}
|
|
f.write_str("\"")
|
|
}
|
|
Newline => f.write_str("Newline"),
|
|
Indent => f.write_str("Indent"),
|
|
Dedent => f.write_str("Dedent"),
|
|
StartModule => f.write_str("StartProgram"),
|
|
StartInteractive => f.write_str("StartInteractive"),
|
|
StartExpression => f.write_str("StartExpression"),
|
|
EndOfFile => f.write_str("EOF"),
|
|
Lpar => f.write_str("'('"),
|
|
Rpar => f.write_str("')'"),
|
|
Lsqb => f.write_str("'['"),
|
|
Rsqb => f.write_str("']'"),
|
|
Colon => f.write_str("':'"),
|
|
Comma => f.write_str("','"),
|
|
Semi => f.write_str("';'"),
|
|
Plus => f.write_str("'+'"),
|
|
Minus => f.write_str("'-'"),
|
|
Star => f.write_str("'*'"),
|
|
Slash => f.write_str("'/'"),
|
|
Vbar => f.write_str("'|'"),
|
|
Amper => f.write_str("'&'"),
|
|
Less => f.write_str("'<'"),
|
|
Greater => f.write_str("'>'"),
|
|
Equal => f.write_str("'='"),
|
|
Dot => f.write_str("'.'"),
|
|
Percent => f.write_str("'%'"),
|
|
Lbrace => f.write_str("'{'"),
|
|
Rbrace => f.write_str("'}'"),
|
|
EqEqual => f.write_str("'=='"),
|
|
NotEqual => f.write_str("'!='"),
|
|
LessEqual => f.write_str("'<='"),
|
|
GreaterEqual => f.write_str("'>='"),
|
|
Tilde => f.write_str("'~'"),
|
|
CircumFlex => f.write_str("'^'"),
|
|
LeftShift => f.write_str("'<<'"),
|
|
RightShift => f.write_str("'>>'"),
|
|
DoubleStar => f.write_str("'**'"),
|
|
DoubleStarEqual => f.write_str("'**='"),
|
|
PlusEqual => f.write_str("'+='"),
|
|
MinusEqual => f.write_str("'-='"),
|
|
StarEqual => f.write_str("'*='"),
|
|
SlashEqual => f.write_str("'/='"),
|
|
PercentEqual => f.write_str("'%='"),
|
|
AmperEqual => f.write_str("'&='"),
|
|
VbarEqual => f.write_str("'|='"),
|
|
CircumflexEqual => f.write_str("'^='"),
|
|
LeftShiftEqual => f.write_str("'<<='"),
|
|
RightShiftEqual => f.write_str("'>>='"),
|
|
DoubleSlash => f.write_str("'//'"),
|
|
DoubleSlashEqual => f.write_str("'//='"),
|
|
At => f.write_str("'@'"),
|
|
AtEqual => f.write_str("'@='"),
|
|
Rarrow => f.write_str("'->'"),
|
|
Ellipsis => f.write_str("'...'"),
|
|
False => f.write_str("'False'"),
|
|
None => f.write_str("'None'"),
|
|
True => f.write_str("'True'"),
|
|
And => f.write_str("'and'"),
|
|
As => f.write_str("'as'"),
|
|
Assert => f.write_str("'assert'"),
|
|
Async => f.write_str("'async'"),
|
|
Await => f.write_str("'await'"),
|
|
Break => f.write_str("'break'"),
|
|
Class => f.write_str("'class'"),
|
|
Continue => f.write_str("'continue'"),
|
|
Def => f.write_str("'def'"),
|
|
Del => f.write_str("'del'"),
|
|
Elif => f.write_str("'elif'"),
|
|
Else => f.write_str("'else'"),
|
|
Except => f.write_str("'except'"),
|
|
Finally => f.write_str("'finally'"),
|
|
For => f.write_str("'for'"),
|
|
From => f.write_str("'from'"),
|
|
Global => f.write_str("'global'"),
|
|
If => f.write_str("'if'"),
|
|
Import => f.write_str("'import'"),
|
|
In => f.write_str("'in'"),
|
|
Is => f.write_str("'is'"),
|
|
Lambda => f.write_str("'lambda'"),
|
|
Nonlocal => f.write_str("'nonlocal'"),
|
|
Not => f.write_str("'not'"),
|
|
Or => f.write_str("'or'"),
|
|
Pass => f.write_str("'pass'"),
|
|
Raise => f.write_str("'raise'"),
|
|
Return => f.write_str("'return'"),
|
|
Try => f.write_str("'try'"),
|
|
While => f.write_str("'while'"),
|
|
With => f.write_str("'with'"),
|
|
Yield => f.write_str("'yield'"),
|
|
ColonEqual => f.write_str("':='"),
|
|
}
|
|
}
|
|
}
|