refactor: decompose lexer into functional submodules

front/lexer/src/lexer/common.rs

@@ -0,0 +1,2 @@
+use crate::*;
+use std::str::FromStr;

front/lexer/src/lexer/core.rs

@@ -0,0 +1,51 @@
+use crate::token::TokenType;
+use super::common::*;
+
+#[derive(Debug, Clone)]
+pub struct Token {
+    pub token_type: TokenType,
+    pub lexeme: String,
+    pub line: usize,
+}
+
+impl Token {
+    pub fn new(token_type: TokenType, lexeme: String, line: usize) -> Self {
+        Token { token_type, lexeme, line }
+    }
+}
+
+impl Default for Token {
+    fn default() -> Self {
+        Token {
+            token_type: TokenType::Eof,
+            lexeme: String::new(),
+            line: 0,
+        }
+    }
+}
+
+#[derive(Debug)]
+pub struct Lexer<'a> {
+    pub source: &'a str,
+    pub current: usize,
+    pub line: usize,
+}
+
+impl<'a> Lexer<'a> {
+    pub fn new(source: &'a str) -> Lexer<'a> {
+        Lexer { source, current: 0, line: 1 }
+    }
+
+    pub fn tokenize(&mut self) -> Vec<Token> {
+        let mut tokens = Vec::new();
+        loop {
+            let token = self.next_token(); // scan.rs에 구현
+            if token.token_type == TokenType::Eof {
+                tokens.push(token);
+                break;
+            }
+            tokens.push(token);
+        }
+        tokens
+    }
+}

front/lexer/src/lexer/cursor.rs

@@ -0,0 +1,52 @@
+use super::{Lexer};
+
+impl<'a> Lexer<'a> {
+    pub(crate) fn is_at_end(&self) -> bool {
+        self.current >= self.source.len()
+    }
+
+    pub(crate) fn advance(&mut self) -> char {
+        if self.is_at_end() {
+            return '\0';
+        }
+
+        let rest = &self.source[self.current..];
+        let (ch, size) = match std::str::from_utf8(rest.as_ref()) {
+            Ok(s) => {
+                let mut chars = s.chars();
+                if let Some(c) = chars.next() { (c, c.len_utf8()) } else { ('\0', 1) }
+            }
+            Err(_) => ('\0', 1),
+        };
+
+        self.current += size;
+        ch
+    }
+
+    pub(crate) fn peek(&self) -> char {
+        if self.is_at_end() {
+            '\0'
+        } else {
+            let rest = &self.source[self.current..];
+            match std::str::from_utf8(rest.as_ref()) {
+                Ok(s) => s.chars().next().unwrap_or('\0'),
+                Err(_) => '\0',
+            }
+        }
+    }
+
+    pub(crate) fn peek_next(&self) -> char {
+        if self.current + 1 >= self.source.len() {
+            '\0'
+        } else {
+            self.source.chars().nth(self.current + 1).unwrap_or('\0')
+        }
+    }
+
+    pub(crate) fn match_next(&mut self, expected: char) -> bool {
+        if self.is_at_end() { return false; }
+        if self.peek() != expected { return false; }
+        self.advance();
+        true
+    }
+}