From cf19fd7db4c650a524664491616f2ffacc5aeee4 Mon Sep 17 00:00:00 2001 From: Alexis <> Date: Tue, 21 Jul 2026 21:47:04 +0200 Subject: [PATCH 01/10] add(source): tokenise --- .gitignore | 2 ++ Cargo.lock | 39 +++++++++++++++++++++++++++++++++++++++ Cargo.toml | 1 + src/ast.rs | 1 + src/lexer.rs | 34 ++++++++++++++++++++++++++++++++++ src/lib.rs | 4 ++++ src/main.rs | 25 ++++++++++++++++++++++--- src/source.rs | 9 ++++++++- 8 files changed, 111 insertions(+), 4 deletions(-) create mode 100644 src/ast.rs create mode 100644 src/lexer.rs diff --git a/.gitignore b/.gitignore index 3faceb9..f81a14c 100644 --- a/.gitignore +++ b/.gitignore @@ -15,3 +15,5 @@ Cargo.lock # Generated by nix result + +.skribi diff --git a/Cargo.lock b/Cargo.lock index 1e2da8b..0c02339 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -228,6 +228,12 @@ dependencies = [ "windows-sys", ] +[[package]] +name = "fnv" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" + [[package]] name = "gimli" version = "0.32.3" @@ -306,6 +312,38 @@ version = "0.4.33" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0ceec5bc11778974d1bcb055b18002eba7f4b3518b6a0081b3af5f21666da9ad" +[[package]] +name = "logos" +version = "0.16.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eb2c55a318a87600ea870ff8c2012148b44bf18b74fad48d0f835c38c7d07c5f" +dependencies = [ + "logos-derive", +] + +[[package]] +name = "logos-codegen" +version = "0.16.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "58b3ffaa284e1350d017a57d04ada118c4583cf260c8fb01e0fe28a2e9cf8970" +dependencies = [ + "fnv", + "proc-macro2", + "quote", + "regex-automata", + "regex-syntax", + "syn 2.0.119", +] + +[[package]] +name = "logos-derive" +version = "0.16.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52d3a9855747c17eaf4383823f135220716ab49bea5fbea7dd42cc9a92f8aa31" +dependencies = [ + "logos-codegen", +] + [[package]] name = "memchr" version = "2.8.3" @@ -480,6 +518,7 @@ dependencies = [ "clap", "env_logger", "log", + "logos", "miette", ] diff --git a/Cargo.toml b/Cargo.toml index f272dd8..2bcb3c6 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -7,4 +7,5 @@ edition = "2024" clap = { version = "4.6.2", features = ["derive"] } env_logger = "0.11.11" log = "0.4.33" +logos = "0.16.1" miette = { version = "7.6.0", features = ["fancy"] } diff --git a/src/ast.rs b/src/ast.rs new file mode 100644 index 0000000..8b13789 --- /dev/null +++ b/src/ast.rs @@ -0,0 +1 @@ + diff --git a/src/lexer.rs b/src/lexer.rs new file mode 100644 index 0000000..3ef66e1 --- /dev/null +++ b/src/lexer.rs @@ -0,0 +1,34 @@ +use logos::{Logos, SpannedIter}; + +#[derive(Logos, Clone, PartialEq)] +pub enum Tokens<'src> { + /// Names: variables, functions, ... + #[regex(r#"[a-zA-Z][a-zA-Z0-9_]*"#)] + Identifier(&'src str), + /// Deprecated keyword to detect native calls, + /// still there to test compatibility + #[token("skr_app")] + NativeCall, + + /// Just a ( + #[token("(")] + LeftParenthesis, + /// Just a ) + #[token(")")] + RightParenthesis, + + /// Anything that is skipped + #[regex(r"[ \t\n]+", logos::skip)] + Ignore, + + /// Any character not used by other tokens, + /// mainly used when parsing bloc title + #[regex(".", priority = 0)] + Error(&'src str), +} + +/// Split a file content into tokens +pub fn tokenise<'src>(arg: &'src str) -> SpannedIter<'src, Tokens<'src>> { + // Inspired from the logos example + Tokens::lexer(arg).spanned() +} diff --git a/src/lib.rs b/src/lib.rs index 8c488a7..c03bb21 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1,4 +1,8 @@ +/// This module is used to store ast structs +pub(crate) mod ast; /// This module handles reading from inputs pub mod file; +/// Used to lex the files +pub(crate) mod lexer; /// This module handles multi sources pub mod source; diff --git a/src/main.rs b/src/main.rs index 3461e5a..20a9f31 100644 --- a/src/main.rs +++ b/src/main.rs @@ -4,10 +4,12 @@ // Skribi's shell // //////////////////// +use std::fs::create_dir_all; + use clap::Parser; -use log::trace; -use miette::{Context, Result}; +use log::{info, trace}; +use miette::{Context, IntoDiagnostic, Result}; use skribi::{file::File, source::SourceManager}; @@ -18,6 +20,10 @@ struct Arguments { /// The source file to use. Defaults to STDIN. /// STDIN is currently not supported. source: Option, + /// Sets the path of the compilation folder. + /// Defaults to `.skribi`. + #[arg(short, long, default_value = ".skribi")] + compile_path: String, /// Log more information, set the level to INFO. /// For fine-grained control over log levels, use the RUST_LOG variable. #[arg(short, long)] @@ -31,6 +37,17 @@ struct Arguments { run: bool, } +/// Creates a folder to store everything +fn create_skribi_directory(path: &str) -> Result<()> { + trace!("About to create hidden directory `{}`", path); + create_dir_all(path).into_diagnostic().context(format!( + "While creating hidden `{}` directory to store compiled files", + path + ))?; + info!("Hidden directory `{}` created for compiled files", path); + Ok(()) +} + /// Launch the interpreter fn main() -> Result<()> { let args = Arguments::parse(); @@ -41,10 +58,12 @@ fn main() -> Result<()> { } else if args.very_verbose { logger.filter_level(log::LevelFilter::Trace); } - logger.init(); + logger.init(); trace!("Logger initialised, entenring main"); + create_skribi_directory(&args.compile_path)?; + if let Some(path) = args.source { let file = File::from_file(&path).context("While reading file passed as argument")?; let mut manager = SourceManager::empty(); diff --git a/src/source.rs b/src/source.rs index 8b5a5d9..b24954d 100644 --- a/src/source.rs +++ b/src/source.rs @@ -3,7 +3,7 @@ use std::collections::HashMap; use log::{info, trace}; use miette::{Context, LabeledSpan, Result, Severity, miette}; -use crate::file::File; +use crate::{file::File, lexer::tokenise}; pub struct Source<'file> { file: File<'file>, @@ -11,6 +11,13 @@ pub struct Source<'file> { impl Source<'_> { pub fn new<'file>(file: File<'file>) -> Source<'file> { + trace!("Entenring source creation for `{}`", file.name); + let tokens = tokenise(&file.content); + let size = tokens.size_hint(); + info!( + "File `{}` splitted into at least {} tokens", + file.name, size.0, + ); Source { file } } From 161646ef9891e7f15399f816aa3f7d5540749841 Mon Sep 17 00:00:00 2001 From: Alexis <> Date: Tue, 11 Aug 2026 16:44:23 +0200 Subject: [PATCH 02/10] add(source): log tokens --- src/lexer.rs | 17 ++++++++++++++++- src/source.rs | 6 ++++++ 2 files changed, 22 insertions(+), 1 deletion(-) diff --git a/src/lexer.rs b/src/lexer.rs index 3ef66e1..930e3b3 100644 --- a/src/lexer.rs +++ b/src/lexer.rs @@ -1,6 +1,8 @@ +use std::fmt::{Display, Formatter}; + use logos::{Logos, SpannedIter}; -#[derive(Logos, Clone, PartialEq)] +#[derive(Logos, Clone, PartialEq, Debug)] pub enum Tokens<'src> { /// Names: variables, functions, ... #[regex(r#"[a-zA-Z][a-zA-Z0-9_]*"#)] @@ -27,6 +29,19 @@ pub enum Tokens<'src> { Error(&'src str), } +impl Display for Tokens<'_> { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + f.write_str(match self { + Self::Identifier(str) => str, + Self::LeftParenthesis => "(", + Self::RightParenthesis => ")", + Self::Ignore => " ", + Self::NativeCall => "skr_app", + Self::Error(err) => err, + }) + } +} + /// Split a file content into tokens pub fn tokenise<'src>(arg: &'src str) -> SpannedIter<'src, Tokens<'src>> { // Inspired from the logos example diff --git a/src/source.rs b/src/source.rs index 163ccda..dfdea58 100644 --- a/src/source.rs +++ b/src/source.rs @@ -14,10 +14,16 @@ impl Source<'_> { trace!("Entenring source creation for `{}`", file.name); let tokens = tokenise(&file.content); let size = tokens.size_hint(); + // Not used for anything else right now + // Will be directly used in parser in next PR info!( + // In general, 0 is detected as we have an indefinite size + // The tokens are parsed on demand I suppose "File `{}` splitted into at least {} tokens", file.name, size.0, ); + // Added to see something + trace!("Tokens: {:?}", tokens.map(|(r, _)| r).collect::>()); Source { file } } From 071ab53e79cac239dec69db33227a27484f6f6be Mon Sep 17 00:00:00 2001 From: Alexis <> Date: Tue, 11 Aug 2026 16:58:42 +0200 Subject: [PATCH 03/10] fix(miette): render correctly panics and warnings --- src/main.rs | 6 +++++- src/source.rs | 5 +++-- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/src/main.rs b/src/main.rs index 519b201..b9f25ab 100644 --- a/src/main.rs +++ b/src/main.rs @@ -17,7 +17,7 @@ use clap::Parser; use env_logger::{Builder, Env}; use log::trace; -use miette::Result; +use miette::{Result, set_panic_hook}; use cli::Arguments; @@ -31,6 +31,10 @@ fn main() -> Result<()> { .write_style("SKRIBI_C_LOG_STYLE"), ); + // Allows to render panics using miette + // Allows an uniform representation of errors + set_panic_hook(); + // To ignore the env variable in production: // #[cfg(not (debug_assertions))] // logger.filter_level(LevelFilter::Warn); diff --git a/src/source.rs b/src/source.rs index dfdea58..0361e9f 100644 --- a/src/source.rs +++ b/src/source.rs @@ -1,6 +1,6 @@ use std::collections::HashMap; -use log::{debug, info, trace}; +use log::{debug, info, trace, warn}; use miette::{Context, LabeledSpan, Result, Severity, miette}; use crate::{file::File, lexer::tokenise}; @@ -38,7 +38,8 @@ impl Source<'_> { "Found deprecated skr_app" ) .with_source_code(self.file.create_source()); - return Err(error); + + warn!("Warning: {:?}", error); } todo!("Finish execution (not the point for now)") } From 08d6974f1357c2867f3718a79572413654f52777 Mon Sep 17 00:00:00 2001 From: Alexis <> Date: Tue, 11 Aug 2026 19:15:36 +0200 Subject: [PATCH 04/10] fix(create_dir): not hidden if no . --- src/cli.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/cli.rs b/src/cli.rs index 71769ae..ef4aaf2 100644 --- a/src/cli.rs +++ b/src/cli.rs @@ -21,12 +21,12 @@ pub(crate) struct Build { /// Creates a folder to store everything fn create_skribi_directory(path: &str) -> Result<()> { - trace!("About to create hidden directory `{}`", path); + trace!("About to create directory `{}`", path); create_dir_all(path).into_diagnostic().context(format!( - "While creating hidden `{}` directory to store compiled files", + "While creating `{}` directory to store compiled files", path ))?; - info!("Hidden directory `{}` created for compiled files", path); + info!("Directory `{}` created for compiled files", path); Ok(()) } From 76dfc5c2c01d1c29830e9f16cc4a669dc28d2fbf Mon Sep 17 00:00:00 2001 From: Alexis <> Date: Tue, 11 Aug 2026 19:20:44 +0200 Subject: [PATCH 05/10] doc(lexer): explain the logos priority --- src/lexer.rs | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/src/lexer.rs b/src/lexer.rs index 930e3b3..4276128 100644 --- a/src/lexer.rs +++ b/src/lexer.rs @@ -2,6 +2,11 @@ use std::fmt::{Display, Formatter}; use logos::{Logos, SpannedIter}; +// NOTE: logos is smart: like CSS, it calculates a priority score based on the +// specificity of the rule. "token" has the priority over anything else. Then, +// regex, with complicated rules. Sometimes, the priority argument can be used +// to avoid confusions. + #[derive(Logos, Clone, PartialEq, Debug)] pub enum Tokens<'src> { /// Names: variables, functions, ... @@ -12,14 +17,12 @@ pub enum Tokens<'src> { #[token("skr_app")] NativeCall, - /// Just a ( #[token("(")] LeftParenthesis, - /// Just a ) #[token(")")] RightParenthesis, - /// Anything that is skipped + /// Note: no need of them in parsing #[regex(r"[ \t\n]+", logos::skip)] Ignore, From 13d5c6e3dc997f50573fc13166e9ecbba109bd9e Mon Sep 17 00:00:00 2001 From: Alexis <> Date: Tue, 11 Aug 2026 19:22:28 +0200 Subject: [PATCH 06/10] add(lexer): _ at start of name --- src/lexer.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/lexer.rs b/src/lexer.rs index 4276128..6baed93 100644 --- a/src/lexer.rs +++ b/src/lexer.rs @@ -10,7 +10,7 @@ use logos::{Logos, SpannedIter}; #[derive(Logos, Clone, PartialEq, Debug)] pub enum Tokens<'src> { /// Names: variables, functions, ... - #[regex(r#"[a-zA-Z][a-zA-Z0-9_]*"#)] + #[regex(r#"[a-zA-Z_][a-zA-Z0-9_]*"#)] Identifier(&'src str), /// Deprecated keyword to detect native calls, /// still there to test compatibility From bf5bf0296f4375699829e27498f22e0eedd25916 Mon Sep 17 00:00:00 2001 From: Alexis <> Date: Wed, 12 Aug 2026 22:45:07 +0200 Subject: [PATCH 07/10] add(cargo): string interner utility --- Cargo.lock | 35 +++++++++++++++++++++++++++++++++++ Cargo.toml | 1 + 2 files changed, 36 insertions(+) diff --git a/Cargo.lock b/Cargo.lock index 0c02339..6bc93fd 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -234,12 +234,27 @@ version = "1.0.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" +[[package]] +name = "foldhash" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77ce24cb58228fbb8aa041425bb1050850ac19177686ea6e0f41a70416f56fdb" + [[package]] name = "gimli" version = "0.32.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e629b9b98ef3dd8afe6ca2bd0f89306cec16d43d907889945bc5d6687f2f13c7" +[[package]] +name = "hashbrown" +version = "0.16.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "841d1cc9bed7f9236f321df977030373f4a4163ae1a7dbfe1a51a2c1a51d9100" +dependencies = [ + "foldhash", +] + [[package]] name = "heck" version = "0.5.0" @@ -491,6 +506,15 @@ dependencies = [ "windows-sys", ] +[[package]] +name = "serde" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4148590afebada386688f18773da617792bf2ef03ffc1e4cbd2b1d45b023e0ba" +dependencies = [ + "serde_core", +] + [[package]] name = "serde_core" version = "1.0.229" @@ -520,6 +544,17 @@ dependencies = [ "log", "logos", "miette", + "string-interner", +] + +[[package]] +name = "string-interner" +version = "0.20.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ad3df9b59e2eded8d825c7c4363ad339a20fb6bc0b9a4778560f518f59910b15" +dependencies = [ + "hashbrown", + "serde", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index 2bcb3c6..5de61ef 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -9,3 +9,4 @@ env_logger = "0.11.11" log = "0.4.33" logos = "0.16.1" miette = { version = "7.6.0", features = ["fancy"] } +string-interner = "0.20.0" From 32a0c8c4db0532ecd65a19d2d9608c74a0881c10 Mon Sep 17 00:00:00 2001 From: Alexis <> Date: Wed, 12 Aug 2026 22:54:36 +0200 Subject: [PATCH 08/10] rm(treewide): lifetime simplification --- src/cli.rs | 6 +++--- src/file.rs | 23 ++++++++++++++--------- src/source.rs | 20 ++++++++++---------- 3 files changed, 27 insertions(+), 22 deletions(-) diff --git a/src/cli.rs b/src/cli.rs index ef4aaf2..b447705 100644 --- a/src/cli.rs +++ b/src/cli.rs @@ -1,4 +1,4 @@ -use std::fs::create_dir_all; +use std::{fs::create_dir_all, sync::Arc}; use log::{LevelFilter, info, trace}; @@ -12,7 +12,7 @@ use miette::{Context, IntoDiagnostic, Result}; pub(crate) struct Build { /// The source file to use. Defaults to STDIN. /// STDIN is currently not supported. - pub(crate) source: Option, + pub(crate) source: Option>, /// Sets the path of the compilation folder. /// Defaults to `.skribi`. #[arg(short, long, default_value = ".skribi")] @@ -36,7 +36,7 @@ impl Build { create_skribi_directory(&self.compile_path)?; if let Some(path) = self.source { - let file = File::from_file(&path).context("While reading file passed as argument")?; + let file = File::from_file(path).context("While reading file passed as argument")?; let mut manager = SourceManager::empty(); manager.add_file(file); diff --git a/src/file.rs b/src/file.rs index 03ee085..5ff409d 100644 --- a/src/file.rs +++ b/src/file.rs @@ -1,27 +1,32 @@ +use std::sync::Arc; + use log::{trace, warn}; use miette::{Context, IntoDiagnostic, NamedSource, Result}; -pub struct File<'name> { - pub(crate) name: &'name str, - pub(crate) content: String, +/// Usage of arc as copies of strings have a big footprint. +/// Used in many cases, even in this file. +/// Avoids lifetime and allows acceptable file cloning. +pub struct File { + pub(crate) name: Arc, + pub(crate) content: Arc, } -impl File<'_> { - pub fn from_file<'name>(path: &'name str) -> Result> { +impl File { + pub fn from_file<'name>(path: Arc) -> Result { trace!("Reading file `{}`", path); if !path.ends_with(".skrb") { warn!("File `{}` does not end in .skrb", path); } - let content = std::fs::read_to_string(path) + let content = std::fs::read_to_string(path.as_ref()) .into_diagnostic() .context(format!("While reading file `{}`", path))?; Ok(File { name: path, - content, + content: content.into(), }) } - pub fn create_source(&self) -> NamedSource { - NamedSource::new(self.name, self.content.clone()) + pub fn create_source(&self) -> NamedSource> { + NamedSource::new(self.name.as_ref(), self.content.clone()) } } diff --git a/src/source.rs b/src/source.rs index 0361e9f..0f4a51c 100644 --- a/src/source.rs +++ b/src/source.rs @@ -1,16 +1,16 @@ -use std::collections::HashMap; +use std::{collections::HashMap, sync::Arc}; use log::{debug, info, trace, warn}; use miette::{Context, LabeledSpan, Result, Severity, miette}; use crate::{file::File, lexer::tokenise}; -pub struct Source<'file> { - file: File<'file>, +pub struct Source { + file: File, } -impl Source<'_> { - pub fn new<'file>(file: File<'file>) -> Source<'file> { +impl Source { + pub fn new(file: File) -> Source { trace!("Entenring source creation for `{}`", file.name); let tokens = tokenise(&file.content); let size = tokens.size_hint(); @@ -45,20 +45,20 @@ impl Source<'_> { } } -pub struct SourceManager<'sources> { - files: HashMap<&'sources str, Source<'sources>>, +pub struct SourceManager { + files: HashMap, Source>, } -impl<'manager> SourceManager<'manager> { +impl<'manager> SourceManager { pub fn empty() -> Self { SourceManager { files: HashMap::new(), } } - pub fn add_file<'file: 'manager>(&mut self, file: File<'file>) { + pub fn add_file<'file: 'manager>(&mut self, file: File) { debug!("Adding file {} into source files", file.name); - self.files.insert(file.name, Source::new(file)); + self.files.insert(file.name.clone(), Source::new(file)); } pub fn compile(&self) -> Result<()> { From 776256eb0641048b2fcebdce33013dd73e694ed0 Mon Sep 17 00:00:00 2001 From: Alexis <> Date: Wed, 12 Aug 2026 23:23:32 +0200 Subject: [PATCH 09/10] add(lexer): interning strings --- src/lexer.rs | 51 ++++++++++++++++++++++++++++++++++----------------- src/source.rs | 9 ++++++--- 2 files changed, 40 insertions(+), 20 deletions(-) diff --git a/src/lexer.rs b/src/lexer.rs index 6baed93..9cb45bc 100644 --- a/src/lexer.rs +++ b/src/lexer.rs @@ -1,17 +1,22 @@ +use std::fmt::{Debug}; use std::fmt::{Display, Formatter}; use logos::{Logos, SpannedIter}; +use string_interner::DefaultSymbol; +use string_interner::DefaultStringInterner; +use string_interner::Symbol; // NOTE: logos is smart: like CSS, it calculates a priority score based on the // specificity of the rule. "token" has the priority over anything else. Then, // regex, with complicated rules. Sometimes, the priority argument can be used // to avoid confusions. -#[derive(Logos, Clone, PartialEq, Debug)] -pub enum Tokens<'src> { +#[derive(Logos, Clone, PartialEq)] +#[logos(extras = &'s mut DefaultStringInterner)] +pub enum Tokens { /// Names: variables, functions, ... - #[regex(r#"[a-zA-Z_][a-zA-Z0-9_]*"#)] - Identifier(&'src str), + #[regex(r#"[a-zA-Z_][a-zA-Z0-9_]*"#, |lex| lex.extras.get_or_intern(lex.slice()))] + Identifier(DefaultSymbol), /// Deprecated keyword to detect native calls, /// still there to test compatibility #[token("skr_app")] @@ -28,25 +33,37 @@ pub enum Tokens<'src> { /// Any character not used by other tokens, /// mainly used when parsing bloc title - #[regex(".", priority = 0)] - Error(&'src str), + #[regex(".", |lex| lex.extras.get_or_intern(lex.slice()), priority = 0)] + Error(DefaultSymbol), } -impl Display for Tokens<'_> { +impl Display for Tokens { fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { - f.write_str(match self { - Self::Identifier(str) => str, - Self::LeftParenthesis => "(", - Self::RightParenthesis => ")", - Self::Ignore => " ", - Self::NativeCall => "skr_app", - Self::Error(err) => err, - }) + if let Self::Identifier(str) = self { + write!(f, "{}", str.to_usize()) + } else if let Self::Error(err) = self { + write!(f, "{}", err.to_usize()) + } else { + write!(f, "{}", match self { + Self::LeftParenthesis => "(", + Self::RightParenthesis => ")", + Self::Ignore => " ", + Self::NativeCall => "skr_app", + // WARNING: when adding tokens, always check the above list + _ => unreachable!(), + }) + } + } +} + +impl Debug for Tokens { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + write!(f, "<{}>", self) } } /// Split a file content into tokens -pub fn tokenise<'src>(arg: &'src str) -> SpannedIter<'src, Tokens<'src>> { +pub fn tokenise<'a>(arg: &'a str, interner: &'a mut DefaultStringInterner) -> SpannedIter<'a, Tokens> { // Inspired from the logos example - Tokens::lexer(arg).spanned() + Tokens::lexer_with_extras(arg, interner).spanned() } diff --git a/src/source.rs b/src/source.rs index 0f4a51c..8387ad1 100644 --- a/src/source.rs +++ b/src/source.rs @@ -2,6 +2,7 @@ use std::{collections::HashMap, sync::Arc}; use log::{debug, info, trace, warn}; use miette::{Context, LabeledSpan, Result, Severity, miette}; +use string_interner::{DefaultStringInterner}; use crate::{file::File, lexer::tokenise}; @@ -10,9 +11,9 @@ pub struct Source { } impl Source { - pub fn new(file: File) -> Source { + pub fn new(file: File, interner: &mut DefaultStringInterner) -> Source { trace!("Entenring source creation for `{}`", file.name); - let tokens = tokenise(&file.content); + let tokens = tokenise(&file.content, interner); let size = tokens.size_hint(); // Not used for anything else right now // Will be directly used in parser in next PR @@ -46,19 +47,21 @@ impl Source { } pub struct SourceManager { + interner: DefaultStringInterner, files: HashMap, Source>, } impl<'manager> SourceManager { pub fn empty() -> Self { SourceManager { + interner: DefaultStringInterner::default(), files: HashMap::new(), } } pub fn add_file<'file: 'manager>(&mut self, file: File) { debug!("Adding file {} into source files", file.name); - self.files.insert(file.name.clone(), Source::new(file)); + self.files.insert(file.name.clone(), Source::new(file, &mut self.interner)); } pub fn compile(&self) -> Result<()> { From 4b43073ef71b3a1111f336b0249a6a696202ee3a Mon Sep 17 00:00:00 2001 From: Alexis <> Date: Sat, 15 Aug 2026 23:00:28 +0200 Subject: [PATCH 10/10] fmt+clippy --- src/file.rs | 2 +- src/lexer.rs | 29 ++++++++++++++++++----------- src/source.rs | 9 +++++---- 3 files changed, 24 insertions(+), 16 deletions(-) diff --git a/src/file.rs b/src/file.rs index 5ff409d..a175f03 100644 --- a/src/file.rs +++ b/src/file.rs @@ -12,7 +12,7 @@ pub struct File { } impl File { - pub fn from_file<'name>(path: Arc) -> Result { + pub fn from_file(path: Arc) -> Result { trace!("Reading file `{}`", path); if !path.ends_with(".skrb") { warn!("File `{}` does not end in .skrb", path); diff --git a/src/lexer.rs b/src/lexer.rs index 9cb45bc..c4545cb 100644 --- a/src/lexer.rs +++ b/src/lexer.rs @@ -1,9 +1,9 @@ -use std::fmt::{Debug}; +use std::fmt::Debug; use std::fmt::{Display, Formatter}; use logos::{Logos, SpannedIter}; -use string_interner::DefaultSymbol; use string_interner::DefaultStringInterner; +use string_interner::DefaultSymbol; use string_interner::Symbol; // NOTE: logos is smart: like CSS, it calculates a priority score based on the @@ -44,14 +44,18 @@ impl Display for Tokens { } else if let Self::Error(err) = self { write!(f, "{}", err.to_usize()) } else { - write!(f, "{}", match self { - Self::LeftParenthesis => "(", - Self::RightParenthesis => ")", - Self::Ignore => " ", - Self::NativeCall => "skr_app", - // WARNING: when adding tokens, always check the above list - _ => unreachable!(), - }) + write!( + f, + "{}", + match self { + Self::LeftParenthesis => "(", + Self::RightParenthesis => ")", + Self::Ignore => " ", + Self::NativeCall => "skr_app", + // WARNING: when adding tokens, always check the above list + _ => unreachable!(), + } + ) } } } @@ -63,7 +67,10 @@ impl Debug for Tokens { } /// Split a file content into tokens -pub fn tokenise<'a>(arg: &'a str, interner: &'a mut DefaultStringInterner) -> SpannedIter<'a, Tokens> { +pub fn tokenise<'a>( + arg: &'a str, + interner: &'a mut DefaultStringInterner, +) -> SpannedIter<'a, Tokens> { // Inspired from the logos example Tokens::lexer_with_extras(arg, interner).spanned() } diff --git a/src/source.rs b/src/source.rs index 8387ad1..01d462d 100644 --- a/src/source.rs +++ b/src/source.rs @@ -2,7 +2,7 @@ use std::{collections::HashMap, sync::Arc}; use log::{debug, info, trace, warn}; use miette::{Context, LabeledSpan, Result, Severity, miette}; -use string_interner::{DefaultStringInterner}; +use string_interner::DefaultStringInterner; use crate::{file::File, lexer::tokenise}; @@ -51,7 +51,7 @@ pub struct SourceManager { files: HashMap, Source>, } -impl<'manager> SourceManager { +impl SourceManager { pub fn empty() -> Self { SourceManager { interner: DefaultStringInterner::default(), @@ -59,9 +59,10 @@ impl<'manager> SourceManager { } } - pub fn add_file<'file: 'manager>(&mut self, file: File) { + pub fn add_file(&mut self, file: File) { debug!("Adding file {} into source files", file.name); - self.files.insert(file.name.clone(), Source::new(file, &mut self.interner)); + self.files + .insert(file.name.clone(), Source::new(file, &mut self.interner)); } pub fn compile(&self) -> Result<()> {