diff --git a/.gitignore b/.gitignore index 3faceb9..f81a14c 100644 --- a/.gitignore +++ b/.gitignore @@ -15,3 +15,5 @@ Cargo.lock # Generated by nix result + +.skribi diff --git a/Cargo.lock b/Cargo.lock index 1e2da8b..6bc93fd 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -228,12 +228,33 @@ dependencies = [ "windows-sys", ] +[[package]] +name = "fnv" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" + +[[package]] +name = "foldhash" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77ce24cb58228fbb8aa041425bb1050850ac19177686ea6e0f41a70416f56fdb" + [[package]] name = "gimli" version = "0.32.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e629b9b98ef3dd8afe6ca2bd0f89306cec16d43d907889945bc5d6687f2f13c7" +[[package]] +name = "hashbrown" +version = "0.16.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "841d1cc9bed7f9236f321df977030373f4a4163ae1a7dbfe1a51a2c1a51d9100" +dependencies = [ + "foldhash", +] + [[package]] name = "heck" version = "0.5.0" @@ -306,6 +327,38 @@ version = "0.4.33" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0ceec5bc11778974d1bcb055b18002eba7f4b3518b6a0081b3af5f21666da9ad" +[[package]] +name = "logos" +version = "0.16.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eb2c55a318a87600ea870ff8c2012148b44bf18b74fad48d0f835c38c7d07c5f" +dependencies = [ + "logos-derive", +] + +[[package]] +name = "logos-codegen" +version = "0.16.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "58b3ffaa284e1350d017a57d04ada118c4583cf260c8fb01e0fe28a2e9cf8970" +dependencies = [ + "fnv", + "proc-macro2", + "quote", + "regex-automata", + "regex-syntax", + "syn 2.0.119", +] + +[[package]] +name = "logos-derive" +version = "0.16.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52d3a9855747c17eaf4383823f135220716ab49bea5fbea7dd42cc9a92f8aa31" +dependencies = [ + "logos-codegen", +] + [[package]] name = "memchr" version = "2.8.3" @@ -453,6 +506,15 @@ dependencies = [ "windows-sys", ] +[[package]] +name = "serde" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4148590afebada386688f18773da617792bf2ef03ffc1e4cbd2b1d45b023e0ba" +dependencies = [ + "serde_core", +] + [[package]] name = "serde_core" version = "1.0.229" @@ -480,7 +542,19 @@ dependencies = [ "clap", "env_logger", "log", + "logos", "miette", + "string-interner", +] + +[[package]] +name = "string-interner" +version = "0.20.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ad3df9b59e2eded8d825c7c4363ad339a20fb6bc0b9a4778560f518f59910b15" +dependencies = [ + "hashbrown", + "serde", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index f272dd8..5de61ef 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -7,4 +7,6 @@ edition = "2024" clap = { version = "4.6.2", features = ["derive"] } env_logger = "0.11.11" log = "0.4.33" +logos = "0.16.1" miette = { version = "7.6.0", features = ["fancy"] } +string-interner = "0.20.0" diff --git a/src/cli.rs b/src/cli.rs index cbece8d..b447705 100644 --- a/src/cli.rs +++ b/src/cli.rs @@ -1,23 +1,42 @@ -use log::LevelFilter; +use std::{fs::create_dir_all, sync::Arc}; + +use log::{LevelFilter, info, trace}; use crate::file::File; use crate::source::SourceManager; use clap::Parser; -use miette::{Context, Result}; +use miette::{Context, IntoDiagnostic, Result}; #[derive(Parser, Debug)] pub(crate) struct Build { /// The source file to use. Defaults to STDIN. /// STDIN is currently not supported. - pub(crate) source: Option, + pub(crate) source: Option>, + /// Sets the path of the compilation folder. + /// Defaults to `.skribi`. + #[arg(short, long, default_value = ".skribi")] + compile_path: String, +} + +/// Creates a folder to store everything +fn create_skribi_directory(path: &str) -> Result<()> { + trace!("About to create directory `{}`", path); + create_dir_all(path).into_diagnostic().context(format!( + "While creating `{}` directory to store compiled files", + path + ))?; + info!("Directory `{}` created for compiled files", path); + Ok(()) } impl Build { /// Compile the source code pub(crate) fn execute(self) -> Result<()> { + create_skribi_directory(&self.compile_path)?; + if let Some(path) = self.source { - let file = File::from_file(&path).context("While reading file passed as argument")?; + let file = File::from_file(path).context("While reading file passed as argument")?; let mut manager = SourceManager::empty(); manager.add_file(file); diff --git a/src/file.rs b/src/file.rs index 03ee085..a175f03 100644 --- a/src/file.rs +++ b/src/file.rs @@ -1,27 +1,32 @@ +use std::sync::Arc; + use log::{trace, warn}; use miette::{Context, IntoDiagnostic, NamedSource, Result}; -pub struct File<'name> { - pub(crate) name: &'name str, - pub(crate) content: String, +/// Usage of arc as copies of strings have a big footprint. +/// Used in many cases, even in this file. +/// Avoids lifetime and allows acceptable file cloning. +pub struct File { + pub(crate) name: Arc, + pub(crate) content: Arc, } -impl File<'_> { - pub fn from_file<'name>(path: &'name str) -> Result> { +impl File { + pub fn from_file(path: Arc) -> Result { trace!("Reading file `{}`", path); if !path.ends_with(".skrb") { warn!("File `{}` does not end in .skrb", path); } - let content = std::fs::read_to_string(path) + let content = std::fs::read_to_string(path.as_ref()) .into_diagnostic() .context(format!("While reading file `{}`", path))?; Ok(File { name: path, - content, + content: content.into(), }) } - pub fn create_source(&self) -> NamedSource { - NamedSource::new(self.name, self.content.clone()) + pub fn create_source(&self) -> NamedSource> { + NamedSource::new(self.name.as_ref(), self.content.clone()) } } diff --git a/src/lexer.rs b/src/lexer.rs new file mode 100644 index 0000000..c4545cb --- /dev/null +++ b/src/lexer.rs @@ -0,0 +1,76 @@ +use std::fmt::Debug; +use std::fmt::{Display, Formatter}; + +use logos::{Logos, SpannedIter}; +use string_interner::DefaultStringInterner; +use string_interner::DefaultSymbol; +use string_interner::Symbol; + +// NOTE: logos is smart: like CSS, it calculates a priority score based on the +// specificity of the rule. "token" has the priority over anything else. Then, +// regex, with complicated rules. Sometimes, the priority argument can be used +// to avoid confusions. + +#[derive(Logos, Clone, PartialEq)] +#[logos(extras = &'s mut DefaultStringInterner)] +pub enum Tokens { + /// Names: variables, functions, ... + #[regex(r#"[a-zA-Z_][a-zA-Z0-9_]*"#, |lex| lex.extras.get_or_intern(lex.slice()))] + Identifier(DefaultSymbol), + /// Deprecated keyword to detect native calls, + /// still there to test compatibility + #[token("skr_app")] + NativeCall, + + #[token("(")] + LeftParenthesis, + #[token(")")] + RightParenthesis, + + /// Note: no need of them in parsing + #[regex(r"[ \t\n]+", logos::skip)] + Ignore, + + /// Any character not used by other tokens, + /// mainly used when parsing bloc title + #[regex(".", |lex| lex.extras.get_or_intern(lex.slice()), priority = 0)] + Error(DefaultSymbol), +} + +impl Display for Tokens { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + if let Self::Identifier(str) = self { + write!(f, "{}", str.to_usize()) + } else if let Self::Error(err) = self { + write!(f, "{}", err.to_usize()) + } else { + write!( + f, + "{}", + match self { + Self::LeftParenthesis => "(", + Self::RightParenthesis => ")", + Self::Ignore => " ", + Self::NativeCall => "skr_app", + // WARNING: when adding tokens, always check the above list + _ => unreachable!(), + } + ) + } + } +} + +impl Debug for Tokens { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + write!(f, "<{}>", self) + } +} + +/// Split a file content into tokens +pub fn tokenise<'a>( + arg: &'a str, + interner: &'a mut DefaultStringInterner, +) -> SpannedIter<'a, Tokens> { + // Inspired from the logos example + Tokens::lexer_with_extras(arg, interner).spanned() +} diff --git a/src/main.rs b/src/main.rs index 6e95ca3..b9f25ab 100644 --- a/src/main.rs +++ b/src/main.rs @@ -8,6 +8,8 @@ mod cli; /// This module handles reading from inputs mod file; +/// Used to lex the files +mod lexer; /// This module handles multi sources mod source; @@ -15,7 +17,7 @@ use clap::Parser; use env_logger::{Builder, Env}; use log::trace; -use miette::Result; +use miette::{Result, set_panic_hook}; use cli::Arguments; @@ -29,6 +31,10 @@ fn main() -> Result<()> { .write_style("SKRIBI_C_LOG_STYLE"), ); + // Allows to render panics using miette + // Allows an uniform representation of errors + set_panic_hook(); + // To ignore the env variable in production: // #[cfg(not (debug_assertions))] // logger.filter_level(LevelFilter::Warn); @@ -41,7 +47,6 @@ fn main() -> Result<()> { } logger.init(); - trace!("Logger initialised, entenring main"); args.cmd.execute() diff --git a/src/source.rs b/src/source.rs index 76b90f8..01d462d 100644 --- a/src/source.rs +++ b/src/source.rs @@ -1,16 +1,30 @@ -use std::collections::HashMap; +use std::{collections::HashMap, sync::Arc}; -use log::{debug, trace}; +use log::{debug, info, trace, warn}; use miette::{Context, LabeledSpan, Result, Severity, miette}; +use string_interner::DefaultStringInterner; -use crate::file::File; +use crate::{file::File, lexer::tokenise}; -pub struct Source<'file> { - file: File<'file>, +pub struct Source { + file: File, } -impl Source<'_> { - pub fn new<'file>(file: File<'file>) -> Source<'file> { +impl Source { + pub fn new(file: File, interner: &mut DefaultStringInterner) -> Source { + trace!("Entenring source creation for `{}`", file.name); + let tokens = tokenise(&file.content, interner); + let size = tokens.size_hint(); + // Not used for anything else right now + // Will be directly used in parser in next PR + info!( + // In general, 0 is detected as we have an indefinite size + // The tokens are parsed on demand I suppose + "File `{}` splitted into at least {} tokens", + file.name, size.0, + ); + // Added to see something + trace!("Tokens: {:?}", tokens.map(|(r, _)| r).collect::>()); Source { file } } @@ -25,26 +39,30 @@ impl Source<'_> { "Found deprecated skr_app" ) .with_source_code(self.file.create_source()); - return Err(error); + + warn!("Warning: {:?}", error); } todo!("Finish execution (not the point for now)") } } -pub struct SourceManager<'sources> { - files: HashMap<&'sources str, Source<'sources>>, +pub struct SourceManager { + interner: DefaultStringInterner, + files: HashMap, Source>, } -impl<'manager> SourceManager<'manager> { +impl SourceManager { pub fn empty() -> Self { SourceManager { + interner: DefaultStringInterner::default(), files: HashMap::new(), } } - pub fn add_file<'file: 'manager>(&mut self, file: File<'file>) { + pub fn add_file(&mut self, file: File) { debug!("Adding file {} into source files", file.name); - self.files.insert(file.name, Source::new(file)); + self.files + .insert(file.name.clone(), Source::new(file, &mut self.interner)); } pub fn compile(&self) -> Result<()> {