String interning for lexer and removed lifetime using Arc

.gitignore

@@ -15,3 +15,5 @@ Cargo.lock
 
 # Generated by nix
 result
+
+.skribi

Cargo.toml

@@ -7,4 +7,6 @@ edition = "2024"
 clap = { version = "4.6.2", features = ["derive"] }
 env_logger = "0.11.11"
 log = "0.4.33"
+logos = "0.16.1"
 miette = { version = "7.6.0", features = ["fancy"] }
+string-interner = "0.20.0"

src/cli.rs

@@ -1,23 +1,42 @@
-use log::LevelFilter;
+use std::{fs::create_dir_all, sync::Arc};
+
+use log::{LevelFilter, info, trace};
 
 use crate::file::File;
 use crate::source::SourceManager;
 
 use clap::Parser;
-use miette::{Context, Result};
+use miette::{Context, IntoDiagnostic, Result};
 
 #[derive(Parser, Debug)]
 pub(crate) struct Build {
     /// The source file to use. Defaults to STDIN.
     /// STDIN is currently not supported.
-    pub(crate) source: Option<String>,
+    pub(crate) source: Option<Arc<str>>,
+    /// Sets the path of the compilation folder.
+    /// Defaults to `.skribi`.
+    #[arg(short, long, default_value = ".skribi")]
+    compile_path: String,
+}
+
+/// Creates a folder to store everything
+fn create_skribi_directory(path: &str) -> Result<()> {
+    trace!("About to create directory `{}`", path);
+    create_dir_all(path).into_diagnostic().context(format!(
+        "While creating `{}` directory to store compiled files",
+        path
+    ))?;
+    info!("Directory `{}` created for compiled files", path);
+    Ok(())
 }
 
 impl Build {
     /// Compile the source code
     pub(crate) fn execute(self) -> Result<()> {
+        create_skribi_directory(&self.compile_path)?;
+
         if let Some(path) = self.source {
-            let file = File::from_file(&path).context("While reading file passed as argument")?;
+            let file = File::from_file(path).context("While reading file passed as argument")?;
             let mut manager = SourceManager::empty();
             manager.add_file(file);
 

src/file.rs

@@ -1,27 +1,32 @@
+use std::sync::Arc;
+
 use log::{trace, warn};
 use miette::{Context, IntoDiagnostic, NamedSource, Result};
 
-pub struct File<'name> {
-    pub(crate) name: &'name str,
-    pub(crate) content: String,
+/// Usage of arc as copies of strings have a big footprint.
+/// Used in many cases, even in this file.
+/// Avoids lifetime and allows acceptable file cloning.
+pub struct File {
+    pub(crate) name: Arc<str>,
+    pub(crate) content: Arc<str>,
 }
 
-impl File<'_> {
-    pub fn from_file<'name>(path: &'name str) -> Result<File<'name>> {
+impl File {
+    pub fn from_file(path: Arc<str>) -> Result<File> {
         trace!("Reading file `{}`", path);
         if !path.ends_with(".skrb") {
             warn!("File `{}` does not end in .skrb", path);
         }
-        let content = std::fs::read_to_string(path)
+        let content = std::fs::read_to_string(path.as_ref())
             .into_diagnostic()
             .context(format!("While reading file `{}`", path))?;
         Ok(File {
             name: path,
-            content,
+            content: content.into(),
         })
     }
 
-    pub fn create_source(&self) -> NamedSource<String> {
-        NamedSource::new(self.name, self.content.clone())
+    pub fn create_source(&self) -> NamedSource<Arc<str>> {
+        NamedSource::new(self.name.as_ref(), self.content.clone())
     }
 }

src/lexer.rs

@@ -0,0 +1,76 @@
+use std::fmt::Debug;
+use std::fmt::{Display, Formatter};
+
+use logos::{Logos, SpannedIter};
+use string_interner::DefaultStringInterner;
+use string_interner::DefaultSymbol;
+use string_interner::Symbol;
+
+// NOTE: logos is smart: like CSS, it calculates a priority score based on the
+// specificity of the rule. "token" has the priority over anything else. Then,
+// regex, with complicated rules. Sometimes, the priority argument can be used
+// to avoid confusions.
+
+#[derive(Logos, Clone, PartialEq)]
+#[logos(extras = &'s mut DefaultStringInterner)]
+pub enum Tokens {
+    /// Names: variables, functions, ...
+    #[regex(r#"[a-zA-Z_][a-zA-Z0-9_]*"#, |lex| lex.extras.get_or_intern(lex.slice()))]
+    Identifier(DefaultSymbol),
+    /// Deprecated keyword to detect native calls,
+    /// still there to test compatibility
+    #[token("skr_app")]
+    NativeCall,
+
+    #[token("(")]
+    LeftParenthesis,
+    #[token(")")]
+    RightParenthesis,
+
+    /// Note: no need of them in parsing
+    #[regex(r"[ \t\n]+", logos::skip)]
+    Ignore,
+
+    /// Any character not used by other tokens,
+    /// mainly used when parsing bloc title
+    #[regex(".", |lex| lex.extras.get_or_intern(lex.slice()), priority = 0)]
+    Error(DefaultSymbol),
+}
+
+impl Display for Tokens {
+    fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
+        if let Self::Identifier(str) = self {
+            write!(f, "{}", str.to_usize())
+        } else if let Self::Error(err) = self {
+            write!(f, "{}", err.to_usize())
+        } else {
+            write!(
+                f,
+                "{}",
+                match self {
+                    Self::LeftParenthesis => "(",
+                    Self::RightParenthesis => ")",
+                    Self::Ignore => " ",
+                    Self::NativeCall => "skr_app",
+                    // WARNING: when adding tokens, always check the above list
+                    _ => unreachable!(),
+                }
+            )
+        }
+    }
+}
+
+impl Debug for Tokens {
+    fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
+        write!(f, "<{}>", self)
+    }
+}
+
+/// Split a file content into tokens
+pub fn tokenise<'a>(
+    arg: &'a str,
+    interner: &'a mut DefaultStringInterner,
+) -> SpannedIter<'a, Tokens> {
+    // Inspired from the logos example
+    Tokens::lexer_with_extras(arg, interner).spanned()
+}

src/main.rs

@@ -8,14 +8,16 @@
 mod cli;
 /// This module handles reading from inputs
 mod file;
+/// Used to lex the files
+mod lexer;
 /// This module handles multi sources
 mod source;
 
 use clap::Parser;
 
 use env_logger::{Builder, Env};
 use log::trace;
-use miette::Result;
+use miette::{Result, set_panic_hook};
 
 use cli::Arguments;
 
@@ -29,6 +31,10 @@ fn main() -> Result<()> {
             .write_style("SKRIBI_C_LOG_STYLE"),
     );
 
+    // Allows to render panics using miette
+    // Allows an uniform representation of errors
+    set_panic_hook();
+
     // To ignore the env variable in production:
     // #[cfg(not (debug_assertions))]
     // logger.filter_level(LevelFilter::Warn);
@@ -41,7 +47,6 @@ fn main() -> Result<()> {
     }
 
     logger.init();
-
     trace!("Logger initialised, entenring main");
 
     args.cmd.execute()

src/source.rs

@@ -1,16 +1,30 @@
-use std::collections::HashMap;
+use std::{collections::HashMap, sync::Arc};
 
-use log::{debug, trace};
+use log::{debug, info, trace, warn};
 use miette::{Context, LabeledSpan, Result, Severity, miette};
+use string_interner::DefaultStringInterner;
 
-use crate::file::File;
+use crate::{file::File, lexer::tokenise};
 
-pub struct Source<'file> {
-    file: File<'file>,
+pub struct Source {
+    file: File,
 }
 
-impl Source<'_> {
-    pub fn new<'file>(file: File<'file>) -> Source<'file> {
+impl Source {
+    pub fn new(file: File, interner: &mut DefaultStringInterner) -> Source {
+        trace!("Entenring source creation for `{}`", file.name);
+        let tokens = tokenise(&file.content, interner);
+        let size = tokens.size_hint();
+        // Not used for anything else right now
+        // Will be directly used in parser in next PR
+        info!(
+            // In general, 0 is detected as we have an indefinite size
+            // The tokens are parsed on demand I suppose
+            "File `{}` splitted into at least {} tokens",
+            file.name, size.0,
+        );
+        // Added to see something
+        trace!("Tokens: {:?}", tokens.map(|(r, _)| r).collect::<Vec<_>>());
         Source { file }
     }
 
@@ -25,26 +39,30 @@ impl Source<'_> {
                 "Found deprecated skr_app"
             )
             .with_source_code(self.file.create_source());
-            return Err(error);
+
+            warn!("Warning: {:?}", error);
         }
         todo!("Finish execution (not the point for now)")
     }
 }
 
-pub struct SourceManager<'sources> {
-    files: HashMap<&'sources str, Source<'sources>>,
+pub struct SourceManager {
+    interner: DefaultStringInterner,
+    files: HashMap<Arc<str>, Source>,
 }
 
-impl<'manager> SourceManager<'manager> {
+impl SourceManager {
     pub fn empty() -> Self {
         SourceManager {
+            interner: DefaultStringInterner::default(),
             files: HashMap::new(),
         }
     }
 
-    pub fn add_file<'file: 'manager>(&mut self, file: File<'file>) {
+    pub fn add_file(&mut self, file: File) {
         debug!("Adding file {} into source files", file.name);
-        self.files.insert(file.name, Source::new(file));
+        self.files
+            .insert(file.name.clone(), Source::new(file, &mut self.interner));
     }
 
     pub fn compile(&self) -> Result<()> {