From c1bcda09ecdf3dac8c2d4109224f94231734a4c0 Mon Sep 17 00:00:00 2001 From: LunaStev Date: Mon, 5 Jan 2026 17:31:42 +0900 Subject: [PATCH] refactor: decompose lexer into functional submodules Break down the lexer implementation into logical components to improve code organization and readability. Changes: - **New Module Structure**: - `core.rs`: `Lexer` and `Token` struct definitions and entry points. - `cursor.rs`: Low-level source navigation (`advance`, `peek`, `match_next`). - `scan.rs`: Main token dispatch logic (`next_token`). - `ident.rs`: Identifier scanning and keyword mapping. - `literals.rs`: String and character literal parsing. - `trivia.rs`: Whitespace and comment skipping. - `common.rs`: Internal shared imports. - **Integration**: - Updated `front/lexer/src/lib.rs` and `mod.rs` to expose the new structure. - Updated imports in `front/parser` to align with the refactored lexer API (explicit `use lexer::token::TokenType` where necessary). This modularization separates concerns, making the lexer easier to maintain and extend. Signed-off-by: LunaStev --- front/lexer/src/lexer/common.rs | 2 + front/lexer/src/lexer/core.rs | 51 ++ front/lexer/src/lexer/cursor.rs | 52 ++ front/lexer/src/lexer/ident.rs | 303 +++++++++ front/lexer/src/lexer/lexer.rs | 961 --------------------------- front/lexer/src/lexer/literals.rs | 60 ++ front/lexer/src/lexer/mod.rs | 13 +- front/lexer/src/lexer/scan.rs | 417 ++++++++++++ front/lexer/src/lexer/trivia.rs | 40 ++ front/lexer/src/lib.rs | 1 + front/lexer/src/{lexer => }/token.rs | 0 front/parser/src/format.rs | 3 +- front/parser/src/parser/asm.rs | 3 +- front/parser/src/parser/control.rs | 3 +- front/parser/src/parser/decl.rs | 3 +- front/parser/src/parser/expr.rs | 3 +- front/parser/src/parser/functions.rs | 3 +- front/parser/src/parser/io.rs | 3 +- front/parser/src/parser/items.rs | 3 +- front/parser/src/parser/parse.rs | 3 +- front/parser/src/parser/stmt.rs | 3 +- front/parser/src/parser/types.rs | 3 +- front/parser/src/type_system.rs | 3 +- 23 files changed, 959 insertions(+), 977 deletions(-) create mode 100644 front/lexer/src/lexer/common.rs create mode 100644 front/lexer/src/lexer/core.rs create mode 100644 front/lexer/src/lexer/cursor.rs create mode 100644 front/lexer/src/lexer/ident.rs delete mode 100644 front/lexer/src/lexer/lexer.rs create mode 100644 front/lexer/src/lexer/literals.rs create mode 100644 front/lexer/src/lexer/scan.rs create mode 100644 front/lexer/src/lexer/trivia.rs rename front/lexer/src/{lexer => }/token.rs (100%) diff --git a/front/lexer/src/lexer/common.rs b/front/lexer/src/lexer/common.rs new file mode 100644 index 00000000..4e26bf66 --- /dev/null +++ b/front/lexer/src/lexer/common.rs @@ -0,0 +1,2 @@ +use crate::*; +use std::str::FromStr; \ No newline at end of file diff --git a/front/lexer/src/lexer/core.rs b/front/lexer/src/lexer/core.rs new file mode 100644 index 00000000..4506283a --- /dev/null +++ b/front/lexer/src/lexer/core.rs @@ -0,0 +1,51 @@ +use crate::token::TokenType; +use super::common::*; + +#[derive(Debug, Clone)] +pub struct Token { + pub token_type: TokenType, + pub lexeme: String, + pub line: usize, +} + +impl Token { + pub fn new(token_type: TokenType, lexeme: String, line: usize) -> Self { + Token { token_type, lexeme, line } + } +} + +impl Default for Token { + fn default() -> Self { + Token { + token_type: TokenType::Eof, + lexeme: String::new(), + line: 0, + } + } +} + +#[derive(Debug)] +pub struct Lexer<'a> { + pub source: &'a str, + pub current: usize, + pub line: usize, +} + +impl<'a> Lexer<'a> { + pub fn new(source: &'a str) -> Lexer<'a> { + Lexer { source, current: 0, line: 1 } + } + + pub fn tokenize(&mut self) -> Vec { + let mut tokens = Vec::new(); + loop { + let token = self.next_token(); // scan.rs에 구현 + if token.token_type == TokenType::Eof { + tokens.push(token); + break; + } + tokens.push(token); + } + tokens + } +} diff --git a/front/lexer/src/lexer/cursor.rs b/front/lexer/src/lexer/cursor.rs new file mode 100644 index 00000000..c6912615 --- /dev/null +++ b/front/lexer/src/lexer/cursor.rs @@ -0,0 +1,52 @@ +use super::{Lexer}; + +impl<'a> Lexer<'a> { + pub(crate) fn is_at_end(&self) -> bool { + self.current >= self.source.len() + } + + pub(crate) fn advance(&mut self) -> char { + if self.is_at_end() { + return '\0'; + } + + let rest = &self.source[self.current..]; + let (ch, size) = match std::str::from_utf8(rest.as_ref()) { + Ok(s) => { + let mut chars = s.chars(); + if let Some(c) = chars.next() { (c, c.len_utf8()) } else { ('\0', 1) } + } + Err(_) => ('\0', 1), + }; + + self.current += size; + ch + } + + pub(crate) fn peek(&self) -> char { + if self.is_at_end() { + '\0' + } else { + let rest = &self.source[self.current..]; + match std::str::from_utf8(rest.as_ref()) { + Ok(s) => s.chars().next().unwrap_or('\0'), + Err(_) => '\0', + } + } + } + + pub(crate) fn peek_next(&self) -> char { + if self.current + 1 >= self.source.len() { + '\0' + } else { + self.source.chars().nth(self.current + 1).unwrap_or('\0') + } + } + + pub(crate) fn match_next(&mut self, expected: char) -> bool { + if self.is_at_end() { return false; } + if self.peek() != expected { return false; } + self.advance(); + true + } +} diff --git a/front/lexer/src/lexer/ident.rs b/front/lexer/src/lexer/ident.rs new file mode 100644 index 00000000..1eff7f1a --- /dev/null +++ b/front/lexer/src/lexer/ident.rs @@ -0,0 +1,303 @@ +use crate::token::*; +use super::common::*; +use super::{Lexer, Token}; + +impl<'a> Lexer<'a> { + pub(crate) fn identifier(&mut self) -> String { + let start = if self.current > 0 { self.current - 1 } else { 0 }; + + while !self.is_at_end() { + let c = self.peek(); + if c.is_alphabetic() || c.is_numeric() || c == '_' { + self.advance(); + } else { + break; + } + } + + self.source[start..self.current].to_string() + } + + pub(crate) fn keyword_or_ident_token(&self, ident: String) -> Token { + // 여기엔 기존 match identifier.as_str() { ... } 통째로 옮겨와 + // 단, Token { token_type: ..., lexeme: ..., line: self.line } 형태 그대로 유지 + // 마지막 default는 Identifier(ident.clone()) + match ident.as_str() { + "fun" => Token { + token_type: TokenType::Fun, + lexeme: "fun".to_string(), + line: self.line, + }, + "var" => Token { + token_type: TokenType::Var, + lexeme: "var".to_string(), + line: self.line, + }, + "deref" => Token { + token_type: TokenType::Deref, + lexeme: "deref".to_string(), + line: self.line, + }, + "let" => Token { + token_type: TokenType::Let, + lexeme: "let".to_string(), + line: self.line, + }, + "mut" => Token { + token_type: TokenType::Mut, + lexeme: "mut".to_string(), + line: self.line, + }, + "const" => Token { + token_type: TokenType::Const, + lexeme: "const".to_string(), + line: self.line, + }, + "if" => Token { + token_type: TokenType::If, + lexeme: "if".to_string(), + line: self.line, + }, + "else" => Token { + token_type: TokenType::Else, + lexeme: "else".to_string(), + line: self.line, + }, + "proto" => Token { + token_type: TokenType::Proto, + lexeme: "proto".to_string(), + line: self.line, + }, + "struct" => Token { + token_type: TokenType::Struct, + lexeme: "struct".to_string(), + line: self.line, + }, + "while" => Token { + token_type: TokenType::While, + lexeme: "while".to_string(), + line: self.line, + }, + "for" => Token { + token_type: TokenType::For, + lexeme: "for".to_string(), + line: self.line, + }, + "module" => Token { + token_type: TokenType::Module, + lexeme: "module".to_string(), + line: self.line, + }, + "class" => Token { + token_type: TokenType::Class, + lexeme: "class".to_string(), + line: self.line, + }, + "in" => Token { + token_type: TokenType::In, + lexeme: "in".to_string(), + line: self.line, + }, + "out" => Token { + token_type: TokenType::Out, + lexeme: "out".to_string(), + line: self.line, + }, + "is" => Token { + token_type: TokenType::Is, + lexeme: "is".to_string(), + line: self.line, + }, + "asm" => Token { + token_type: TokenType::Asm, + lexeme: "asm".to_string(), + line: self.line, + }, + "xnand" => Token { + token_type: TokenType::Xnand, + lexeme: "xnand".to_string(), + line: self.line, + }, + "import" => Token { + token_type: TokenType::Import, + lexeme: "import".to_string(), + line: self.line, + }, + "return" => Token { + token_type: TokenType::Return, + lexeme: "return".to_string(), + line: self.line, + }, + "continue" => Token { + token_type: TokenType::Continue, + lexeme: "continue".to_string(), + line: self.line, + }, + "print" => Token { + token_type: TokenType::Print, + lexeme: "print".to_string(), + line: self.line, + }, + "input" => Token { + token_type: TokenType::Input, + lexeme: "input".to_string(), + line: self.line, + }, + "println" => Token { + token_type: TokenType::Println, + lexeme: "println".to_string(), + line: self.line, + }, + "match" => Token { + token_type: TokenType::Match, + lexeme: "match".to_string(), + line: self.line, + }, + "char" => Token { + token_type: TokenType::TypeChar, + lexeme: "char".to_string(), + line: self.line, + }, + "byte" => Token { + token_type: TokenType::TypeByte, + lexeme: "byte".to_string(), + line: self.line, + }, + "ptr" => Token { + token_type: TokenType::Identifier("ptr".to_string()), + lexeme: "ptr".to_string(), + line: self.line, + }, + "array" => Token { + token_type: TokenType::Identifier("array".to_string()), + lexeme: "array".to_string(), + line: self.line, + }, + "isz" => Token { + token_type: TokenType::TokenTypeInt(IntegerType::ISZ), + lexeme: "isz".to_string(), + line: self.line, + }, + "i8" => Token { + token_type: TokenType::TokenTypeInt(IntegerType::I8), + lexeme: "i8".to_string(), + line: self.line, + }, + "i16" => Token { + token_type: TokenType::TokenTypeInt(IntegerType::I16), + lexeme: "i16".to_string(), + line: self.line, + }, + "i32" => Token { + token_type: TokenType::TokenTypeInt(IntegerType::I32), + lexeme: "i32".to_string(), + line: self.line, + }, + "i64" => Token { + token_type: TokenType::TokenTypeInt(IntegerType::I64), + lexeme: "i64".to_string(), + line: self.line, + }, + "i128" => Token { + token_type: TokenType::TokenTypeInt(IntegerType::I128), + lexeme: "i128".to_string(), + line: self.line, + }, + "i256" => Token { + token_type: TokenType::TokenTypeInt(IntegerType::I256), + lexeme: "i256".to_string(), + line: self.line, + }, + "i512" => Token { + token_type: TokenType::TokenTypeInt(IntegerType::I512), + lexeme: "i512".to_string(), + line: self.line, + }, + "i1024" => Token { + token_type: TokenType::TokenTypeInt(IntegerType::I1024), + lexeme: "i1024".to_string(), + line: self.line, + }, + "usz" => Token { + token_type: TokenType::TokenTypeUint(UnsignedIntegerType::USZ), + lexeme: "usz".to_string(), + line: self.line, + }, + "u8" => Token { + token_type: TokenType::TokenTypeUint(UnsignedIntegerType::U8), + lexeme: "u8".to_string(), + line: self.line, + }, + "u16" => Token { + token_type: TokenType::TokenTypeUint(UnsignedIntegerType::U16), + lexeme: "u16".to_string(), + line: self.line, + }, + "u32" => Token { + token_type: TokenType::TokenTypeUint(UnsignedIntegerType::U32), + lexeme: "u32".to_string(), + line: self.line, + }, + "u64" => Token { + token_type: TokenType::TokenTypeUint(UnsignedIntegerType::U64), + lexeme: "u64".to_string(), + line: self.line, + }, + "u128" => Token { + token_type: TokenType::TokenTypeUint(UnsignedIntegerType::U128), + lexeme: "u128".to_string(), + line: self.line, + }, + "u256" => Token { + token_type: TokenType::TokenTypeUint(UnsignedIntegerType::U256), + lexeme: "u256".to_string(), + line: self.line, + }, + "u512" => Token { + token_type: TokenType::TokenTypeUint(UnsignedIntegerType::U512), + lexeme: "u512".to_string(), + line: self.line, + }, + "u1024" => Token { + token_type: TokenType::TokenTypeUint(UnsignedIntegerType::U1024), + lexeme: "u1024".to_string(), + line: self.line, + }, + "f32" => Token { + token_type: TokenType::TokenTypeFloat(FloatType::F32), + lexeme: "f32".to_string(), + line: self.line, + }, + "f64" => Token { + token_type: TokenType::TokenTypeFloat(FloatType::F64), + lexeme: "f64".to_string(), + line: self.line, + }, + "str" => Token { + token_type: TokenType::TypeString, + lexeme: "str".to_string(), + line: self.line, + }, + "break" => Token { + token_type: TokenType::Break, + lexeme: "break".to_string(), + line: self.line, + }, + "true" => Token { + token_type: TokenType::BoolLiteral(true), + lexeme: "true".to_string(), + line: self.line, + }, + "false" => Token { + token_type: TokenType::BoolLiteral(false), + lexeme: "false".to_string(), + line: self.line, + }, + _ => Token { + token_type: TokenType::Identifier(ident.clone()), + lexeme: ident, + line: self.line, + }, + } + } +} diff --git a/front/lexer/src/lexer/lexer.rs b/front/lexer/src/lexer/lexer.rs deleted file mode 100644 index c201809e..00000000 --- a/front/lexer/src/lexer/lexer.rs +++ /dev/null @@ -1,961 +0,0 @@ -use crate::*; -use std::str::FromStr; - -#[derive(Debug, Clone)] -pub struct Token { - pub token_type: TokenType, - pub lexeme: String, - pub line: usize, -} - -impl Token { - pub fn new(token_type: TokenType, lexeme: String, line: usize) -> Self { - Token { - token_type, - lexeme, - line, - } - } -} - -impl Default for Token { - fn default() -> Self { - Token { - token_type: TokenType::Eof, // Set default token type to EOF - lexeme: String::new(), // The default lexeme is an empty string - line: 0, // Default line number is 0 - } - } -} - -#[derive(Debug)] -pub struct Lexer<'a> { - pub source: &'a str, - pub current: usize, - pub line: usize, -} - -impl<'a> Lexer<'a> { - pub fn new(source: &'a str) -> Lexer<'a> { - Lexer { - source, - current: 0, - line: 1, - } - } - - fn is_at_end(&self) -> bool { - self.current >= self.source.len() - } - - fn advance(&mut self) -> char { - if self.is_at_end() { - return '\0'; - } - - let rest = &self.source[self.current..]; - let (ch, size) = match std::str::from_utf8(rest.as_ref()) { - Ok(s) => { - let mut chars = s.chars(); - if let Some(c) = chars.next() { - (c, c.len_utf8()) - } else { - ('\0', 1) - } - } - Err(_) => ('\0', 1), - }; - - self.current += size; - ch - } - - fn skip_whitespace(&mut self) { - while !self.is_at_end() { - let c = self.peek(); - match c { - ' ' | '\r' | '\t' => { - self.advance(); - } - '\n' => { - self.line += 1; - self.advance(); - } - _ => break, - } - } - } - - fn peek(&self) -> char { - if self.is_at_end() { - '\0' - } else { - let rest = &self.source[self.current..]; - match std::str::from_utf8(rest.as_ref()) { - Ok(s) => s.chars().next().unwrap_or('\0'), - Err(_) => '\0', - } - } - } - - fn match_next(&mut self, expected: char) -> bool { - if self.is_at_end() { - return false; - } - if self.peek() != expected { - return false; - } - self.advance(); - true - } - - pub fn tokenize(&mut self) -> Vec { - let mut tokens = Vec::new(); - - loop { - let token = self.next_token(); - - if token.token_type == TokenType::Eof { - tokens.push(token); - break; - } - tokens.push(token); - } - tokens - } - - fn skip_comment(&mut self) { - while !self.is_at_end() && self.peek() != '\n' { - self.advance(); - } - } - - fn skip_multiline_comment(&mut self) { - while !self.is_at_end() { - if self.peek() == '*' && self.peek_next() == '/' { - self.advance(); - self.advance(); - break; - } - - if self.peek() == '\n' { - self.line += 1; - } - - self.advance(); - } - - if self.is_at_end() { - panic!("Unterminated block comment"); - } - } - - fn peek_next(&self) -> char { - if self.current + 1 >= self.source.len() { - '\0' - } else { - self.source.chars().nth(self.current + 1).unwrap_or('\0') - } - } - - /* - pub fn consume(&mut self) { - if let Some(current_char) = self.source.chars().nth(self.current) { - if current_char == '\n' { - self.line += 1; - } - println!("Consuming character: {}, at position: {}", current_char, self.current); - self.current += 1; - } - } - - pub fn consume_n(&mut self, n: usize) { - for _ in 0..n { - self.consume(); - } - println!("Consumed {} characters, current position: {}", n, self.current); - } - */ - - pub fn next_token(&mut self) -> Token { - self.skip_whitespace(); - - if self.is_at_end() { - return Token { - token_type: TokenType::Eof, - lexeme: String::new(), - line: self.line, - }; - } - - let c = self.advance(); - - match c { - '+' => { - if self.match_next('+') { - Token { - token_type: TokenType::Increment, - lexeme: "++".to_string(), - line: self.line, - } - } else if self.match_next('=') { - Token { - token_type: TokenType::PlusEq, - lexeme: "+=".to_string(), - line: self.line, - } - } else { - Token { - token_type: TokenType::Plus, - lexeme: "+".to_string(), - line: self.line, - } - } - } - '-' => { - if self.match_next('-') { - Token { - token_type: TokenType::Decrement, - lexeme: "--".to_string(), - line: self.line, - } - } else if self.match_next('>') { - Token { - token_type: TokenType::Arrow, - lexeme: "->".to_string(), - line: self.line, - } - } else if self.match_next('=') { - Token { - token_type: TokenType::MinusEq, - lexeme: "-=".to_string(), - line: self.line, - } - } else { - Token { - token_type: TokenType::Minus, - lexeme: "-".to_string(), - line: self.line, - } - } - } - '*' => { - if self.match_next('=') { - Token { - token_type: TokenType::StarEq, - lexeme: "*=".to_string(), - line: self.line, - } - } else { - Token { - token_type: TokenType::Star, - lexeme: "*".to_string(), - line: self.line, - } - } - } - '.' => Token { - token_type: TokenType::Dot, - lexeme: ".".to_string(), - line: self.line, - }, - '/' => { - if self.match_next('/') { - self.skip_comment(); - self.next_token() - } else if self.match_next('*') { - self.skip_multiline_comment(); - self.next_token() - } else if self.match_next('=') { - Token { - token_type: TokenType::DivEq, - lexeme: "/=".to_string(), - line: self.line, - } - } else { - Token { - token_type: TokenType::Div, - lexeme: "/".to_string(), - line: self.line, - } - } - } - '%' => { - if self.match_next('=') { - Token { - token_type: TokenType::RemainderEq, - lexeme: "%=".to_string(), - line: self.line, - } - } else { - Token { - token_type: TokenType::Remainder, - lexeme: "%".to_string(), - line: self.line, - } - } - } - ';' => Token { - token_type: TokenType::SemiColon, - lexeme: ";".to_string(), - line: self.line, - }, - ':' => Token { - token_type: TokenType::Colon, - lexeme: ":".to_string(), - line: self.line, - }, - '<' => { - if self.match_next('<') { - Token { - token_type: TokenType::Rol, - lexeme: "<<".to_string(), - line: self.line, - } - } else if self.match_next('=') { - Token { - token_type: TokenType::LchevrEq, - lexeme: "<=".to_string(), - line: self.line, - } - } else { - Token { - token_type: TokenType::Lchevr, - lexeme: "<".to_string(), - line: self.line, - } - } - } - '>' => { - if self.match_next('>') { - Token { - token_type: TokenType::Ror, - lexeme: ">>".to_string(), - line: self.line, - } - } else if self.match_next('=') { - Token { - token_type: TokenType::RchevrEq, - lexeme: ">=".to_string(), - line: self.line, - } - } else { - Token { - token_type: TokenType::Rchevr, - lexeme: ">".to_string(), - line: self.line, - } - } - } - '(' => Token { - token_type: TokenType::Lparen, - lexeme: "(".to_string(), - line: self.line, - }, - ')' => Token { - token_type: TokenType::Rparen, - lexeme: ")".to_string(), - line: self.line, - }, - '{' => Token { - token_type: TokenType::Lbrace, - lexeme: "{".to_string(), - line: self.line, - }, - '}' => Token { - token_type: TokenType::Rbrace, - lexeme: "}".to_string(), - line: self.line, - }, - '[' => Token { - token_type: TokenType::Lbrack, - lexeme: "[".to_string(), - line: self.line, - }, - ']' => Token { - token_type: TokenType::Rbrack, - lexeme: "]".to_string(), - line: self.line, - }, - '=' => { - if self.match_next('=') { - Token { - token_type: TokenType::EqualTwo, - lexeme: "==".to_string(), - line: self.line, - } - } else { - Token { - token_type: TokenType::Equal, - lexeme: "=".to_string(), - line: self.line, - } - } - } - '&' => { - if self.match_next('&') { - Token { - token_type: TokenType::LogicalAnd, - lexeme: "&&".to_string(), - line: self.line, - } - } else { - Token { - token_type: TokenType::AddressOf, - lexeme: "&".to_string(), - line: self.line, - } - } - } - '|' => { - if self.match_next('|') { - Token { - token_type: TokenType::LogicalOr, - lexeme: "||".to_string(), - line: self.line, - } - } else { - Token { - token_type: TokenType::BitwiseOr, - lexeme: "|".to_string(), - line: self.line, - } - } - } - '!' => { - if self.match_next('=') { - Token { - token_type: TokenType::NotEqual, - lexeme: "!=".to_string(), - line: self.line, - } - } else if self.match_next('&') { - Token { - token_type: TokenType::Nand, - lexeme: "!&".to_string(), - line: self.line, - } - } else if self.match_next('|') { - Token { - token_type: TokenType::Nor, - lexeme: "!|".to_string(), - line: self.line, - } - } else { - Token { - token_type: TokenType::Not, - lexeme: "!".to_string(), - line: self.line, - } - } - } - '^' => Token { - token_type: TokenType::Xor, - lexeme: "^".to_string(), - line: self.line, - }, - '~' => { - if self.match_next('^') { - Token { - token_type: TokenType::Xnor, - lexeme: "~^".to_string(), - line: self.line, - } - } else { - Token { - token_type: TokenType::BitwiseNot, - lexeme: "~".to_string(), - line: self.line, - } - } - } - '?' => { - if self.match_next('?') { - Token { - token_type: TokenType::NullCoalesce, - lexeme: "??".to_string(), - line: self.line, - } - } else { - Token { - token_type: TokenType::Condition, - lexeme: "?".to_string(), - line: self.line, - } - } - } - ',' => Token { - token_type: TokenType::Comma, - lexeme: ",".to_string(), - line: self.line, - }, - '\'' => { - let value = self.char_literal(); - Token { - token_type: TokenType::CharLiteral(value), - lexeme: format!("'{}'", value), - line: self.line, - } - }, - '"' => { - let string_value = self.string(); - Token { - token_type: TokenType::String(string_value.clone()), - lexeme: format!("\"{}\"", string_value), - line: self.line, - } - } - 'a'..='z' | 'A'..='Z' | '_' => { - let identifier = self.identifier(); - match identifier.as_str() { - "fun" => Token { - token_type: TokenType::Fun, - lexeme: "fun".to_string(), - line: self.line, - }, - "var" => Token { - token_type: TokenType::Var, - lexeme: "var".to_string(), - line: self.line, - }, - "deref" => Token { - token_type: TokenType::Deref, - lexeme: "deref".to_string(), - line: self.line, - }, - "let" => Token { - token_type: TokenType::Let, - lexeme: "let".to_string(), - line: self.line, - }, - "mut" => Token { - token_type: TokenType::Mut, - lexeme: "mut".to_string(), - line: self.line, - }, - "const" => Token { - token_type: TokenType::Const, - lexeme: "const".to_string(), - line: self.line, - }, - "if" => Token { - token_type: TokenType::If, - lexeme: "if".to_string(), - line: self.line, - }, - "else" => Token { - token_type: TokenType::Else, - lexeme: "else".to_string(), - line: self.line, - }, - "proto" => Token { - token_type: TokenType::Proto, - lexeme: "proto".to_string(), - line: self.line, - }, - "struct" => Token { - token_type: TokenType::Struct, - lexeme: "struct".to_string(), - line: self.line, - }, - "while" => Token { - token_type: TokenType::While, - lexeme: "while".to_string(), - line: self.line, - }, - "for" => Token { - token_type: TokenType::For, - lexeme: "for".to_string(), - line: self.line, - }, - "module" => Token { - token_type: TokenType::Module, - lexeme: "module".to_string(), - line: self.line, - }, - "class" => Token { - token_type: TokenType::Class, - lexeme: "class".to_string(), - line: self.line, - }, - "in" => Token { - token_type: TokenType::In, - lexeme: "in".to_string(), - line: self.line, - }, - "out" => Token { - token_type: TokenType::Out, - lexeme: "out".to_string(), - line: self.line, - }, - "is" => Token { - token_type: TokenType::Is, - lexeme: "is".to_string(), - line: self.line, - }, - "asm" => Token { - token_type: TokenType::Asm, - lexeme: "asm".to_string(), - line: self.line, - }, - "xnand" => Token { - token_type: TokenType::Xnand, - lexeme: "xnand".to_string(), - line: self.line, - }, - "import" => Token { - token_type: TokenType::Import, - lexeme: "import".to_string(), - line: self.line, - }, - "return" => Token { - token_type: TokenType::Return, - lexeme: "return".to_string(), - line: self.line, - }, - "continue" => Token { - token_type: TokenType::Continue, - lexeme: "continue".to_string(), - line: self.line, - }, - "print" => Token { - token_type: TokenType::Print, - lexeme: "print".to_string(), - line: self.line, - }, - "input" => Token { - token_type: TokenType::Input, - lexeme: "input".to_string(), - line: self.line, - }, - "println" => Token { - token_type: TokenType::Println, - lexeme: "println".to_string(), - line: self.line, - }, - "match" => Token { - token_type: TokenType::Match, - lexeme: "match".to_string(), - line: self.line, - }, - "char" => Token { - token_type: TokenType::TypeChar, - lexeme: "char".to_string(), - line: self.line, - }, - "byte" => Token { - token_type: TokenType::TypeByte, - lexeme: "byte".to_string(), - line: self.line, - }, - "ptr" => Token { - token_type: TokenType::Identifier("ptr".to_string()), - lexeme: "ptr".to_string(), - line: self.line, - }, - "array" => Token { - token_type: TokenType::Identifier("array".to_string()), - lexeme: "array".to_string(), - line: self.line, - }, - "isz" => Token { - token_type: TokenType::TokenTypeInt(IntegerType::ISZ), - lexeme: "isz".to_string(), - line: self.line, - }, - "i8" => Token { - token_type: TokenType::TokenTypeInt(IntegerType::I8), - lexeme: "i8".to_string(), - line: self.line, - }, - "i16" => Token { - token_type: TokenType::TokenTypeInt(IntegerType::I16), - lexeme: "i16".to_string(), - line: self.line, - }, - "i32" => Token { - token_type: TokenType::TokenTypeInt(IntegerType::I32), - lexeme: "i32".to_string(), - line: self.line, - }, - "i64" => Token { - token_type: TokenType::TokenTypeInt(IntegerType::I64), - lexeme: "i64".to_string(), - line: self.line, - }, - "i128" => Token { - token_type: TokenType::TokenTypeInt(IntegerType::I128), - lexeme: "i128".to_string(), - line: self.line, - }, - "i256" => Token { - token_type: TokenType::TokenTypeInt(IntegerType::I256), - lexeme: "i256".to_string(), - line: self.line, - }, - "i512" => Token { - token_type: TokenType::TokenTypeInt(IntegerType::I512), - lexeme: "i512".to_string(), - line: self.line, - }, - "i1024" => Token { - token_type: TokenType::TokenTypeInt(IntegerType::I1024), - lexeme: "i1024".to_string(), - line: self.line, - }, - "usz" => Token { - token_type: TokenType::TokenTypeUint(UnsignedIntegerType::USZ), - lexeme: "usz".to_string(), - line: self.line, - }, - "u8" => Token { - token_type: TokenType::TokenTypeUint(UnsignedIntegerType::U8), - lexeme: "u8".to_string(), - line: self.line, - }, - "u16" => Token { - token_type: TokenType::TokenTypeUint(UnsignedIntegerType::U16), - lexeme: "u16".to_string(), - line: self.line, - }, - "u32" => Token { - token_type: TokenType::TokenTypeUint(UnsignedIntegerType::U32), - lexeme: "u32".to_string(), - line: self.line, - }, - "u64" => Token { - token_type: TokenType::TokenTypeUint(UnsignedIntegerType::U64), - lexeme: "u64".to_string(), - line: self.line, - }, - "u128" => Token { - token_type: TokenType::TokenTypeUint(UnsignedIntegerType::U128), - lexeme: "u128".to_string(), - line: self.line, - }, - "u256" => Token { - token_type: TokenType::TokenTypeUint(UnsignedIntegerType::U256), - lexeme: "u256".to_string(), - line: self.line, - }, - "u512" => Token { - token_type: TokenType::TokenTypeUint(UnsignedIntegerType::U512), - lexeme: "u512".to_string(), - line: self.line, - }, - "u1024" => Token { - token_type: TokenType::TokenTypeUint(UnsignedIntegerType::U1024), - lexeme: "u1024".to_string(), - line: self.line, - }, - "f32" => Token { - token_type: TokenType::TokenTypeFloat(FloatType::F32), - lexeme: "f32".to_string(), - line: self.line, - }, - "f64" => Token { - token_type: TokenType::TokenTypeFloat(FloatType::F64), - lexeme: "f64".to_string(), - line: self.line, - }, - "str" => Token { - token_type: TokenType::TypeString, - lexeme: "str".to_string(), - line: self.line, - }, - "break" => Token { - token_type: TokenType::Break, - lexeme: "break".to_string(), - line: self.line, - }, - "true" => Token { - token_type: TokenType::BoolLiteral(true), - lexeme: "true".to_string(), - line: self.line, - }, - "false" => Token { - token_type: TokenType::BoolLiteral(false), - lexeme: "false".to_string(), - line: self.line, - }, - _ => Token { - token_type: TokenType::Identifier(identifier.clone()), - lexeme: identifier, - line: self.line, - }, - } - } - '0'..='9' => { - if c == '0' && (self.peek() == 'b' || self.peek() == 'B') { - self.advance(); // consume 'b' or 'B' - - let mut bin_str = String::new(); - while self.peek() == '0' || self.peek() == '1' { - bin_str.push(self.advance()); - } - - let value = i64::from_str_radix(&bin_str, 2).unwrap_or(0); - - return Token { - token_type: TokenType::Number(value), - lexeme: format!("0b{}", bin_str), - line: self.line, - } - } - - if c == '0' && (self.peek() == 'x' || self.peek() == 'X') { - self.advance(); // consume 'x' or 'X' - - let mut hex_str = String::new(); - while self.peek().is_ascii_hexdigit() { - hex_str.push(self.advance()); - } - - let value = i64::from_str_radix(&hex_str, 16).unwrap_or(0); - - return Token { - token_type: TokenType::Number(value), - lexeme: format!("0x{}", hex_str), - line: self.line, - }; - } - - let mut num_str = c.to_string(); - while self.peek().is_ascii_digit() { - num_str.push(self.advance()); - } - - let is_float = if self.peek() == '.' { - num_str.push('.'); - self.advance(); - while self.peek().is_ascii_digit() { - num_str.push(self.advance()); - } - true - } else { - false - }; - - let token_type = if is_float { - num_str.parse::().map(TokenType::Float).unwrap() - } else { - num_str.parse::().map(TokenType::Number).unwrap() - }; - - Token { - token_type, - lexeme: num_str, - line: self.line, - } - } - _ => { - if c == '\0' { - eprintln!("[eprintln] Null character encountered — likely unintended"); - panic!("[panic] Null character (`\\0`) is not allowed in source"); - } else if c == '\\' { - eprintln!("[eprintln] Unexpected backslash outside of string"); - panic!("[panic] Unexpected character: '\\' outside of string"); - } else { - eprintln!( - "[eprintln] Unexpected character: {:?} (code: {})", - c, c as u32 - ); - panic!("[panic] Unexpected character: {:?}", c); - } - } - } - } - - // Add string literal processing function - fn string(&mut self) -> String { - if self.peek() == '"' { - self.advance(); - } - - let mut string_literal = String::new(); - - while !self.is_at_end() && self.peek() != '"' { - let c = self.advance(); - - if c == '\\' { - let next = self.advance(); - match next { - 'n' => string_literal.push('\n'), - 't' => string_literal.push('\t'), - 'r' => string_literal.push('\r'), - '\\' => string_literal.push('\\'), - '"' => string_literal.push('"'), - _ => { - string_literal.push('\\'); - string_literal.push(next); - } - } - } else { - string_literal.push(c); - } - } - - if self.is_at_end() { - panic!("Unterminated string."); - } - - self.advance(); // closing quote - - string_literal - } - - fn char_literal(&mut self) -> char { - let c = if self.peek() == '\\' { - self.advance(); // consume '\' - let escaped = self.advance(); - match escaped { - 'n' => '\n', - 't' => '\t', - 'r' => '\r', - '\\' => '\\', - '\'' => '\'', - '"' => '"', - _ => panic!("Invalid escape sequence in char literal"), - } - } else { - self.advance() - }; - - if self.peek() != '\'' { - panic!("Unterminated or invalid char literal"); - } - - self.advance(); // closing ' - - c - } - - fn identifier(&mut self) -> String { - let start = if self.current > 0 { - self.current - 1 - } else { - 0 - }; - - while !self.is_at_end() { - let c = self.peek(); - if c.is_alphabetic() || c.is_numeric() || c == '_' { - self.advance(); - } else { - break; - } - } - - self.source[start..self.current].to_string() - } - - fn number(&mut self) -> i64 { - let start = self.current - 1; - while !self.is_at_end() && self.peek().is_numeric() { - self.advance(); - } - - let number_str = &self.source[start..self.current]; - i64::from_str(number_str).unwrap_or(0) - } -} diff --git a/front/lexer/src/lexer/literals.rs b/front/lexer/src/lexer/literals.rs new file mode 100644 index 00000000..e969203d --- /dev/null +++ b/front/lexer/src/lexer/literals.rs @@ -0,0 +1,60 @@ +use super::Lexer; + +impl<'a> Lexer<'a> { + pub(crate) fn string(&mut self) -> String { + if self.peek() == '"' { self.advance(); } + + let mut string_literal = String::new(); + while !self.is_at_end() && self.peek() != '"' { + let c = self.advance(); + + if c == '\\' { + let next = self.advance(); + match next { + 'n' => string_literal.push('\n'), + 't' => string_literal.push('\t'), + 'r' => string_literal.push('\r'), + '\\' => string_literal.push('\\'), + '"' => string_literal.push('"'), + _ => { + string_literal.push('\\'); + string_literal.push(next); + } + } + } else { + string_literal.push(c); + } + } + + if self.is_at_end() { + panic!("Unterminated string."); + } + + self.advance(); // closing quote + string_literal + } + + pub(crate) fn char_literal(&mut self) -> char { + let c = if self.peek() == '\\' { + self.advance(); + let escaped = self.advance(); + match escaped { + 'n' => '\n', + 't' => '\t', + 'r' => '\r', + '\\' => '\\', + '\'' => '\'', + '"' => '"', + _ => panic!("Invalid escape sequence in char literal"), + } + } else { + self.advance() + }; + + if self.peek() != '\'' { + panic!("Unterminated or invalid char literal"); + } + self.advance(); // closing ' + c + } +} diff --git a/front/lexer/src/lexer/mod.rs b/front/lexer/src/lexer/mod.rs index a0653226..fa6b54ab 100644 --- a/front/lexer/src/lexer/mod.rs +++ b/front/lexer/src/lexer/mod.rs @@ -17,8 +17,13 @@ clippy::new_without_default )] -pub mod lexer; -pub mod token; +mod common; +mod core; +mod cursor; +mod trivia; +mod literals; +mod ident; +mod scan; + +pub use core::{Lexer, Token}; -pub use lexer::*; -pub use token::*; diff --git a/front/lexer/src/lexer/scan.rs b/front/lexer/src/lexer/scan.rs new file mode 100644 index 00000000..9e20b36f --- /dev/null +++ b/front/lexer/src/lexer/scan.rs @@ -0,0 +1,417 @@ +use crate::token::*; +use super::common::*; +use super::{Lexer, Token}; + +impl<'a> Lexer<'a> { + pub fn next_token(&mut self) -> Token { + self.skip_whitespace(); + + if self.is_at_end() { + return Token { token_type: TokenType::Eof, lexeme: String::new(), line: self.line }; + } + + let c = self.advance(); + + match c { + '+' => { + if self.match_next('+') { + Token { + token_type: TokenType::Increment, + lexeme: "++".to_string(), + line: self.line, + } + } else if self.match_next('=') { + Token { + token_type: TokenType::PlusEq, + lexeme: "+=".to_string(), + line: self.line, + } + } else { + Token { + token_type: TokenType::Plus, + lexeme: "+".to_string(), + line: self.line, + } + } + } + '-' => { + if self.match_next('-') { + Token { + token_type: TokenType::Decrement, + lexeme: "--".to_string(), + line: self.line, + } + } else if self.match_next('>') { + Token { + token_type: TokenType::Arrow, + lexeme: "->".to_string(), + line: self.line, + } + } else if self.match_next('=') { + Token { + token_type: TokenType::MinusEq, + lexeme: "-=".to_string(), + line: self.line, + } + } else { + Token { + token_type: TokenType::Minus, + lexeme: "-".to_string(), + line: self.line, + } + } + } + '*' => { + if self.match_next('=') { + Token { + token_type: TokenType::StarEq, + lexeme: "*=".to_string(), + line: self.line, + } + } else { + Token { + token_type: TokenType::Star, + lexeme: "*".to_string(), + line: self.line, + } + } + } + '.' => Token { + token_type: TokenType::Dot, + lexeme: ".".to_string(), + line: self.line, + }, + '/' => { + if self.match_next('/') { + self.skip_comment(); + self.next_token() + } else if self.match_next('*') { + self.skip_multiline_comment(); + self.next_token() + } else if self.match_next('=') { + Token { + token_type: TokenType::DivEq, + lexeme: "/=".to_string(), + line: self.line, + } + } else { + Token { + token_type: TokenType::Div, + lexeme: "/".to_string(), + line: self.line, + } + } + } + '%' => { + if self.match_next('=') { + Token { + token_type: TokenType::RemainderEq, + lexeme: "%=".to_string(), + line: self.line, + } + } else { + Token { + token_type: TokenType::Remainder, + lexeme: "%".to_string(), + line: self.line, + } + } + } + ';' => Token { + token_type: TokenType::SemiColon, + lexeme: ";".to_string(), + line: self.line, + }, + ':' => Token { + token_type: TokenType::Colon, + lexeme: ":".to_string(), + line: self.line, + }, + '<' => { + if self.match_next('<') { + Token { + token_type: TokenType::Rol, + lexeme: "<<".to_string(), + line: self.line, + } + } else if self.match_next('=') { + Token { + token_type: TokenType::LchevrEq, + lexeme: "<=".to_string(), + line: self.line, + } + } else { + Token { + token_type: TokenType::Lchevr, + lexeme: "<".to_string(), + line: self.line, + } + } + } + '>' => { + if self.match_next('>') { + Token { + token_type: TokenType::Ror, + lexeme: ">>".to_string(), + line: self.line, + } + } else if self.match_next('=') { + Token { + token_type: TokenType::RchevrEq, + lexeme: ">=".to_string(), + line: self.line, + } + } else { + Token { + token_type: TokenType::Rchevr, + lexeme: ">".to_string(), + line: self.line, + } + } + } + '(' => Token { + token_type: TokenType::Lparen, + lexeme: "(".to_string(), + line: self.line, + }, + ')' => Token { + token_type: TokenType::Rparen, + lexeme: ")".to_string(), + line: self.line, + }, + '{' => Token { + token_type: TokenType::Lbrace, + lexeme: "{".to_string(), + line: self.line, + }, + '}' => Token { + token_type: TokenType::Rbrace, + lexeme: "}".to_string(), + line: self.line, + }, + '[' => Token { + token_type: TokenType::Lbrack, + lexeme: "[".to_string(), + line: self.line, + }, + ']' => Token { + token_type: TokenType::Rbrack, + lexeme: "]".to_string(), + line: self.line, + }, + '=' => { + if self.match_next('=') { + Token { + token_type: TokenType::EqualTwo, + lexeme: "==".to_string(), + line: self.line, + } + } else { + Token { + token_type: TokenType::Equal, + lexeme: "=".to_string(), + line: self.line, + } + } + } + '&' => { + if self.match_next('&') { + Token { + token_type: TokenType::LogicalAnd, + lexeme: "&&".to_string(), + line: self.line, + } + } else { + Token { + token_type: TokenType::AddressOf, + lexeme: "&".to_string(), + line: self.line, + } + } + } + '|' => { + if self.match_next('|') { + Token { + token_type: TokenType::LogicalOr, + lexeme: "||".to_string(), + line: self.line, + } + } else { + Token { + token_type: TokenType::BitwiseOr, + lexeme: "|".to_string(), + line: self.line, + } + } + } + '!' => { + if self.match_next('=') { + Token { + token_type: TokenType::NotEqual, + lexeme: "!=".to_string(), + line: self.line, + } + } else if self.match_next('&') { + Token { + token_type: TokenType::Nand, + lexeme: "!&".to_string(), + line: self.line, + } + } else if self.match_next('|') { + Token { + token_type: TokenType::Nor, + lexeme: "!|".to_string(), + line: self.line, + } + } else { + Token { + token_type: TokenType::Not, + lexeme: "!".to_string(), + line: self.line, + } + } + } + '^' => Token { + token_type: TokenType::Xor, + lexeme: "^".to_string(), + line: self.line, + }, + '~' => { + if self.match_next('^') { + Token { + token_type: TokenType::Xnor, + lexeme: "~^".to_string(), + line: self.line, + } + } else { + Token { + token_type: TokenType::BitwiseNot, + lexeme: "~".to_string(), + line: self.line, + } + } + } + '?' => { + if self.match_next('?') { + Token { + token_type: TokenType::NullCoalesce, + lexeme: "??".to_string(), + line: self.line, + } + } else { + Token { + token_type: TokenType::Condition, + lexeme: "?".to_string(), + line: self.line, + } + } + } + ',' => Token { + token_type: TokenType::Comma, + lexeme: ",".to_string(), + line: self.line, + }, + '\'' => { + let value = self.char_literal(); + Token { + token_type: TokenType::CharLiteral(value), + lexeme: format!("'{}'", value), + line: self.line, + } + }, + '"' => { + let string_value = self.string(); + Token { + token_type: TokenType::String(string_value.clone()), + lexeme: format!("\"{}\"", string_value), + line: self.line, + } + } + + 'a'..='z' | 'A'..='Z' | '_' => { + let ident = self.identifier(); + self.keyword_or_ident_token(ident) + } + + '0'..='9' => { + if c == '0' && (self.peek() == 'b' || self.peek() == 'B') { + self.advance(); // consume 'b' or 'B' + + let mut bin_str = String::new(); + while self.peek() == '0' || self.peek() == '1' { + bin_str.push(self.advance()); + } + + let value = i64::from_str_radix(&bin_str, 2).unwrap_or(0); + + return Token { + token_type: TokenType::Number(value), + lexeme: format!("0b{}", bin_str), + line: self.line, + } + } + + if c == '0' && (self.peek() == 'x' || self.peek() == 'X') { + self.advance(); // consume 'x' or 'X' + + let mut hex_str = String::new(); + while self.peek().is_ascii_hexdigit() { + hex_str.push(self.advance()); + } + + let value = i64::from_str_radix(&hex_str, 16).unwrap_or(0); + + return Token { + token_type: TokenType::Number(value), + lexeme: format!("0x{}", hex_str), + line: self.line, + }; + } + + let mut num_str = c.to_string(); + while self.peek().is_ascii_digit() { + num_str.push(self.advance()); + } + + let is_float = if self.peek() == '.' { + num_str.push('.'); + self.advance(); + while self.peek().is_ascii_digit() { + num_str.push(self.advance()); + } + true + } else { + false + }; + + let token_type = if is_float { + num_str.parse::().map(TokenType::Float).unwrap() + } else { + num_str.parse::().map(TokenType::Number).unwrap() + }; + + Token { + token_type, + lexeme: num_str, + line: self.line, + } + } + + _ => { + if c == '\0' { + eprintln!("[eprintln] Null character encountered — likely unintended"); + panic!("[panic] Null character (`\\0`) is not allowed in source"); + } else if c == '\\' { + eprintln!("[eprintln] Unexpected backslash outside of string"); + panic!("[panic] Unexpected character: '\\' outside of string"); + } else { + eprintln!( + "[eprintln] Unexpected character: {:?} (code: {})", + c, c as u32 + ); + panic!("[panic] Unexpected character: {:?}", c); + } + } + } + } +} diff --git a/front/lexer/src/lexer/trivia.rs b/front/lexer/src/lexer/trivia.rs new file mode 100644 index 00000000..8622d197 --- /dev/null +++ b/front/lexer/src/lexer/trivia.rs @@ -0,0 +1,40 @@ +use super::Lexer; + +impl<'a> Lexer<'a> { + pub(crate) fn skip_whitespace(&mut self) { + while !self.is_at_end() { + let c = self.peek(); + match c { + ' ' | '\r' | '\t' => { self.advance(); } + '\n' => { self.line += 1; self.advance(); } + _ => break, + } + } + } + + pub(crate) fn skip_comment(&mut self) { + while !self.is_at_end() && self.peek() != '\n' { + self.advance(); + } + } + + pub(crate) fn skip_multiline_comment(&mut self) { + while !self.is_at_end() { + if self.peek() == '*' && self.peek_next() == '/' { + self.advance(); + self.advance(); + break; + } + + if self.peek() == '\n' { + self.line += 1; + } + + self.advance(); + } + + if self.is_at_end() { + panic!("Unterminated block comment"); + } + } +} diff --git a/front/lexer/src/lib.rs b/front/lexer/src/lib.rs index 0f94cf3d..b17be8a6 100644 --- a/front/lexer/src/lib.rs +++ b/front/lexer/src/lib.rs @@ -1,3 +1,4 @@ pub mod lexer; +pub mod token; pub use lexer::*; diff --git a/front/lexer/src/lexer/token.rs b/front/lexer/src/token.rs similarity index 100% rename from front/lexer/src/lexer/token.rs rename to front/lexer/src/token.rs diff --git a/front/parser/src/format.rs b/front/parser/src/format.rs index 238eb3e1..6a1e96f5 100644 --- a/front/parser/src/format.rs +++ b/front/parser/src/format.rs @@ -1,6 +1,7 @@ use crate::ast::Expression::Variable; use crate::ast::{AssignOperator, Expression, FormatPart, IncDecKind, Literal, Operator}; -use lexer::{Token, TokenType}; +use lexer::Token; +use lexer::token::TokenType; use std::iter::Peekable; use std::slice::Iter; diff --git a/front/parser/src/parser/asm.rs b/front/parser/src/parser/asm.rs index 1ceeca55..8e20c7e5 100644 --- a/front/parser/src/parser/asm.rs +++ b/front/parser/src/parser/asm.rs @@ -1,6 +1,7 @@ use std::iter::Peekable; use std::slice::Iter; -use lexer::{Token, TokenType}; +use lexer::Token; +use lexer::token::TokenType; use crate::ast::{ASTNode, StatementNode}; pub fn parse_asm_block(tokens: &mut Peekable>) -> Option { diff --git a/front/parser/src/parser/control.rs b/front/parser/src/parser/control.rs index baccadcc..5e6128a1 100644 --- a/front/parser/src/parser/control.rs +++ b/front/parser/src/parser/control.rs @@ -1,6 +1,7 @@ use std::iter::Peekable; use std::slice::Iter; -use lexer::{Token, TokenType}; +use lexer::Token; +use lexer::token::TokenType; use crate::ast::{ASTNode, Expression, StatementNode}; use crate::format::parse_expression; use crate::parser::stmt::parse_block; diff --git a/front/parser/src/parser/decl.rs b/front/parser/src/parser/decl.rs index eb06997b..dfead370 100644 --- a/front/parser/src/parser/decl.rs +++ b/front/parser/src/parser/decl.rs @@ -1,6 +1,7 @@ use std::iter::Peekable; use std::slice::Iter; -use lexer::{Token, TokenType}; +use lexer::Token; +use lexer::token::TokenType; use crate::ast::{ASTNode, Expression, Mutability, VariableNode, WaveType}; use crate::format::parse_expression; use crate::parser::types::{parse_type, token_type_to_wave_type}; diff --git a/front/parser/src/parser/expr.rs b/front/parser/src/parser/expr.rs index 779ada55..fa7cb1ca 100644 --- a/front/parser/src/parser/expr.rs +++ b/front/parser/src/parser/expr.rs @@ -1,6 +1,7 @@ use std::iter::Peekable; use std::slice::Iter; -use lexer::{Token, TokenType}; +use lexer::Token; +use lexer::token::TokenType; use crate::ast::Expression; use crate::format::parse_expression; diff --git a/front/parser/src/parser/functions.rs b/front/parser/src/parser/functions.rs index bcc426f7..94deb814 100644 --- a/front/parser/src/parser/functions.rs +++ b/front/parser/src/parser/functions.rs @@ -1,7 +1,8 @@ use std::collections::HashSet; use std::iter::Peekable; use std::slice::Iter; -use lexer::{Token, TokenType}; +use lexer::Token; +use lexer::token::TokenType; use crate::ast::{ASTNode, FunctionNode, ParameterNode, StatementNode, Value}; use crate::format::parse_expression; use crate::parser::asm::*; diff --git a/front/parser/src/parser/io.rs b/front/parser/src/parser/io.rs index a34b3eeb..02f90f0e 100644 --- a/front/parser/src/parser/io.rs +++ b/front/parser/src/parser/io.rs @@ -1,7 +1,8 @@ use std::iter::Peekable; use std::slice::Iter; use regex::Regex; -use lexer::{Token, TokenType}; +use lexer::Token; +use lexer::token::TokenType; use crate::ast::{ASTNode, StatementNode}; use crate::format::parse_expression; diff --git a/front/parser/src/parser/items.rs b/front/parser/src/parser/items.rs index a6be69a1..a55da390 100644 --- a/front/parser/src/parser/items.rs +++ b/front/parser/src/parser/items.rs @@ -1,6 +1,7 @@ use std::iter::Peekable; use std::slice::Iter; -use lexer::{Token, TokenType}; +use lexer::Token; +use lexer::token::TokenType; use crate::ast::{ASTNode, ProtoImplNode, StatementNode, StructNode, WaveType}; use crate::parser::functions::parse_function; use crate::parser::types::parse_type_from_token; diff --git a/front/parser/src/parser/parse.rs b/front/parser/src/parser/parse.rs index 4ce846ed..39119369 100644 --- a/front/parser/src/parser/parse.rs +++ b/front/parser/src/parser/parse.rs @@ -1,4 +1,5 @@ -use lexer::{Token, TokenType}; +use lexer::Token; +use lexer::token::TokenType; use crate::ast::ASTNode; use crate::parser::decl::*; use crate::parser::functions::parse_function; diff --git a/front/parser/src/parser/stmt.rs b/front/parser/src/parser/stmt.rs index e201e174..e5dd49cd 100644 --- a/front/parser/src/parser/stmt.rs +++ b/front/parser/src/parser/stmt.rs @@ -1,6 +1,7 @@ use std::iter::Peekable; use std::slice::Iter; -use lexer::{Token, TokenType}; +use lexer::Token; +use lexer::token::TokenType; use crate::ast::{ASTNode, AssignOperator, Expression, Operator, StatementNode}; use crate::format::{parse_expression, parse_expression_from_token}; use crate::parser::control::{parse_for, parse_if, parse_while}; diff --git a/front/parser/src/parser/types.rs b/front/parser/src/parser/types.rs index 082a218b..b3df2c30 100644 --- a/front/parser/src/parser/types.rs +++ b/front/parser/src/parser/types.rs @@ -1,6 +1,7 @@ use std::iter::Peekable; use std::slice::Iter; -use lexer::{FloatType, IntegerType, Token, TokenType, UnsignedIntegerType}; +use lexer::Token; +use lexer::token::*; use crate::ast::WaveType; pub fn token_type_to_wave_type(token_type: &TokenType) -> Option { diff --git a/front/parser/src/type_system.rs b/front/parser/src/type_system.rs index 3e3d73f7..344ec13e 100644 --- a/front/parser/src/type_system.rs +++ b/front/parser/src/type_system.rs @@ -1,6 +1,7 @@ use crate::ast::WaveType; use crate::*; -use lexer::{Token, TokenType}; +use lexer::Token; +use lexer::token::TokenType; use std::iter::Peekable; use std::slice::Iter;