From 7539a8750221af1c6d089f8e7ee1c144b8d8d310 Mon Sep 17 00:00:00 2001 From: Mayur Kale Date: Thu, 12 Nov 2026 22:04:22 +0000 Subject: [PATCH] feat(compiler): load a project and order its models Project::load reads a project laid out as RFC 0001 (revised in #101) describes: infostash.toml, the sources in sources/*.toml and the models in clean/ and ready/, where subdirectories only organise files. Each model's header is read with the RFC's grammar, and the tables it reads come from its SQL as DataFusion parses it, so CTE names and table functions are not mistaken for tables. Every name must carry its layer, reads must be defined and may not reach a later layer, and a cycle is named in full. Errors give the file and, where there is one, the line. Models are returned in run order, ties broken by table name. The old string-splitting directives() is replaced by the Directives type. Planning each model is left for a follow-up. Refs #7 Signed-off-by: Mayur Kale --- Cargo.lock | 61 ++++++ crates/compiler/Cargo.toml | 7 + crates/compiler/src/config.rs | 68 +++++++ crates/compiler/src/directives.rs | 215 +++++++++++++++++++++ crates/compiler/src/graph.rs | 81 ++++++++ crates/compiler/src/lib.rs | 246 ++++++++++++++++++++++-- crates/compiler/src/reads.rs | 192 +++++++++++++++++++ crates/compiler/src/tests.rs | 304 ++++++++++++++++++++++++++++++ 8 files changed, 1155 insertions(+), 19 deletions(-) create mode 100644 crates/compiler/src/config.rs create mode 100644 crates/compiler/src/directives.rs create mode 100644 crates/compiler/src/graph.rs create mode 100644 crates/compiler/src/reads.rs create mode 100644 crates/compiler/src/tests.rs diff --git a/Cargo.lock b/Cargo.lock index ac79f65..a721b7f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2599,6 +2599,13 @@ dependencies = [ [[package]] name = "infostash-compiler" version = "0.1.0" +dependencies = [ + "datafusion", + "infostash-model", + "serde", + "tempfile", + "toml", +] [[package]] name = "infostash-connectors" @@ -3891,6 +3898,15 @@ dependencies = [ "syn 3.0.9", ] +[[package]] +name = "serde_spanned" +version = "1.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c187e0a369471c550af028cba56d4f4fa2a489ad28151ae95a4d340a2a8b782" +dependencies = [ + "serde_core", +] + [[package]] name = "serde_urlencoded" version = "0.7.1" @@ -4537,6 +4553,45 @@ dependencies = [ "tokio", ] +[[package]] +name = "toml" +version = "1.2.1+spec-1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d5755c51863cdac711f1956baf8eb9588e060755f7d3c4c979e70ea1e1f32cd7" +dependencies = [ + "indexmap 2.14.2", + "serde_core", + "serde_spanned", + "toml_datetime", + "toml_parser", + "toml_writer", + "winnow", +] + +[[package]] +name = "toml_datetime" +version = "1.1.3+spec-1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "173f25c883cd2176a2d8212155638c0a60d484a9d84546d270866f40f56b6e49" +dependencies = [ + "serde_core", +] + +[[package]] +name = "toml_parser" +version = "1.1.5+spec-1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "baa693a8032d7e1cada7d0041e96126df243179ff061456783ac7f12bda4744c" +dependencies = [ + "winnow", +] + +[[package]] +name = "toml_writer" +version = "1.2.0+spec-1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "009dbb7700f11c55ef83aebbe4c0e9cf1f1ac0844bd8d6f00dc38f302026e13a" + [[package]] name = "tower" version = "0.5.3" @@ -5121,6 +5176,12 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" +[[package]] +name = "winnow" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8c3b93199df0ab874cdf0c6a94f0a0dc3423fe41f1cafb62b8bd7b1deb277aef" + [[package]] name = "wit-bindgen" version = "0.57.1" diff --git a/crates/compiler/Cargo.toml b/crates/compiler/Cargo.toml index 653ec60..deff8b2 100644 --- a/crates/compiler/Cargo.toml +++ b/crates/compiler/Cargo.toml @@ -7,6 +7,13 @@ license.workspace = true repository.workspace = true [dependencies] +datafusion.workspace = true +infostash-model.workspace = true +serde = { version = "1.0.229", features = ["derive"] } +toml = "1.2.1" + +[dev-dependencies] +tempfile = "3.27.0" [lints] workspace = true diff --git a/crates/compiler/src/config.rs b/crates/compiler/src/config.rs new file mode 100644 index 0000000..7002326 --- /dev/null +++ b/crates/compiler/src/config.rs @@ -0,0 +1,68 @@ +//! The TOML files of a project: `infostash.toml` and `sources/*.toml`. + +use std::path::PathBuf; + +use infostash_model::{Error, Result}; +use serde::Deserialize; + +/// The project format this version reads. +const FORMAT: u32 = 1; + +/// `infostash.toml`, as written. +#[derive(Deserialize)] +#[serde(deny_unknown_fields)] +pub(crate) struct ProjectFile { + pub project: ProjectSection, + pub warehouse: WarehouseSection, +} + +#[derive(Deserialize)] +#[serde(deny_unknown_fields)] +pub(crate) struct ProjectSection { + pub name: String, + pub format: u32, +} + +#[derive(Deserialize)] +#[serde(deny_unknown_fields)] +pub(crate) struct WarehouseSection { + pub path: PathBuf, +} + +/// Read `infostash.toml`. Unknown keys and any format but this one are +/// errors. +pub(crate) fn project(file: &str, text: &str) -> Result { + let parsed: ProjectFile = toml::from_str(text).map_err(|e| toml_error(file, text, &e))?; + if parsed.project.format != FORMAT { + return Err(Error::Parse(format!( + "{file}: format {} is not supported; this version reads format {FORMAT}", + parsed.project.format + ))); + } + Ok(parsed) +} + +/// Read a source file into its connector and the keys that belong to it. +pub(crate) fn source(file: &str, text: &str) -> Result<(String, toml::Table)> { + let mut options: toml::Table = toml::from_str(text).map_err(|e| toml_error(file, text, &e))?; + match options.remove("connector") { + Some(toml::Value::String(connector)) => Ok((connector, options)), + Some(_) => Err(Error::Parse(format!( + "{file}: `connector` must be a string" + ))), + None => Err(Error::Parse(format!("{file}: `connector` is missing"))), + } +} + +/// A TOML error as `file:line: message`, or `file: message` when the +/// parser gives no position. +fn toml_error(file: &str, text: &str, e: &toml::de::Error) -> Error { + let line = e + .span() + .and_then(|span| text.get(..span.start)) + .map(|before| before.matches('\n').count() + 1); + match line { + Some(line) => Error::Parse(format!("{file}:{line}: {}", e.message())), + None => Error::Parse(format!("{file}: {}", e.message())), + } +} diff --git a/crates/compiler/src/directives.rs b/crates/compiler/src/directives.rs new file mode 100644 index 0000000..7b4d7b6 --- /dev/null +++ b/crates/compiler/src/directives.rs @@ -0,0 +1,215 @@ +//! The header grammar of RFC 0001: `-- @key: value` comments above the SQL. + +use infostash_model::{Error, Result}; + +/// A model's settings, read from the `-- @key: value` comments in its +/// header. Which of them a run accepts is decided elsewhere. +#[derive(Debug, Clone, Default, PartialEq, Eq)] +pub struct Directives { + /// `@unique_key`: the key columns, in the order written. + pub unique_key: Option>, + /// `@check`: every check, in file order. + pub checks: Vec, + /// `@full_refresh: true`. + pub full_refresh: bool, + /// `@description`. + pub description: Option, +} + +/// One `@check` directive. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Check { + /// The name written between `@check` and the colon, if any. + pub name: Option, + /// The SQL boolean expression, as written. + pub expr: String, + /// The line of the model file it is on, counting from 1. + pub line: usize, +} + +/// Read the directives in the header of a model file. The header is the run +/// of blank and `--` lines at the top; a directive starts at column one with +/// `-- @`, and anything after the header is SQL, not a directive. +pub(crate) fn parse(file: &str, sql: &str) -> Result { + let mut directives = Directives::default(); + let mut once = Vec::new(); + for (index, line) in sql.lines().enumerate() { + let trimmed = line.trim(); + if !trimmed.is_empty() && !trimmed.starts_with("--") { + break; + } + let Some(rest) = line.strip_prefix("-- @") else { + continue; + }; + let err = |message: String| Error::Parse(format!("{file}:{}: {message}", index + 1)); + let (head, value) = rest + .split_once(':') + .ok_or_else(|| err(format!("`-- @{}` needs a `:` and a value", rest.trim())))?; + let value = value.trim(); + let mut words = head.split_whitespace(); + let key = words.next().unwrap_or_default(); + let name = words.next(); + if words.next().is_some() { + return Err(err(format!( + "`@{}`: expected `@key` or `@check name`", + head.trim() + ))); + } + if !matches!(key, "unique_key" | "check" | "full_refresh" | "description") { + return Err(err(format!("unknown directive `@{key}`"))); + } + if value.is_empty() { + return Err(err(format!("`@{key}` needs a value"))); + } + if key == "check" { + directives + .checks + .push(check(name, value, index + 1).map_err(err)?); + continue; + } + if name.is_some() { + return Err(err(format!("only `@check` takes a name, not `@{key}`"))); + } + if once.contains(&key) { + return Err(err(format!("`@{key}` may appear only once"))); + } + once.push(key); + match key { + "unique_key" => directives.unique_key = Some(columns(value).map_err(err)?), + "full_refresh" if value == "true" => directives.full_refresh = true, + "full_refresh" => { + return Err(err(format!( + "`@full_refresh` takes only `true`, not `{value}`" + ))); + } + _ => directives.description = Some(value.to_string()), + } + } + check_unique_names(&directives.checks) + .map_err(|(line, message)| Error::Parse(format!("{file}:{line}: {message}")))?; + Ok(directives) +} + +fn check(name: Option<&str>, expr: &str, line: usize) -> std::result::Result { + if let Some(name) = name.filter(|n| !crate::is_name(n)) { + return Err(format!( + "check name `{name}` must use lower case letters, digits and `_`, and start with a letter" + )); + } + Ok(Check { + name: name.map(str::to_string), + expr: expr.to_string(), + line, + }) +} + +fn columns(value: &str) -> std::result::Result, String> { + let columns: Vec = value.split(',').map(|c| c.trim().to_string()).collect(); + if columns.iter().any(String::is_empty) { + return Err(format!("`@unique_key: {value}` has an empty column")); + } + Ok(columns) +} + +fn check_unique_names(checks: &[Check]) -> std::result::Result<(), (usize, String)> { + for (i, check) in checks.iter().enumerate() { + let Some(name) = &check.name else { continue }; + if checks[..i].iter().any(|c| c.name.as_ref() == Some(name)) { + return Err((check.line, format!("two checks are named `{name}`"))); + } + } + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn parse_ok(sql: &str) -> Directives { + parse("m.sql", sql).unwrap() + } + + fn parse_err(sql: &str) -> String { + match parse("m.sql", sql) { + Err(Error::Parse(message)) => message, + other => panic!("expected a parse error, got {other:?}"), + } + } + + #[test] + fn values_may_hold_colons_and_are_trimmed() { + let d = parse_ok("-- \n-- @check: amount::decimal > 0 \nselect 1"); + assert_eq!(d.checks[0].expr, "amount::decimal > 0"); + assert_eq!(d.checks[0].line, 2); + } + + #[test] + fn splits_a_compound_unique_key() { + let d = parse_ok("-- @unique_key: order_id, line_no\nselect 1"); + assert_eq!( + d.unique_key, + Some(vec!["order_id".into(), "line_no".into()]) + ); + } + + #[test] + fn other_comment_shapes_are_ordinary_comments() { + let d = parse_ok("--@unique_key: a\n -- @unique_key: b\n-- plain\n\nselect 1"); + assert_eq!(d, Directives::default()); + } + + #[test] + fn a_directive_after_the_sql_starts_is_a_comment() { + let d = parse_ok("select 1\n-- @nonsense: here\n-- @check: x"); + assert_eq!(d, Directives::default()); + } + + #[test] + fn rejects_malformed_directives() { + let cases = [ + ("-- @unique_key", "`-- @unique_key` needs a `:` and a value"), + ( + "-- @check a b: x", + "`@check a b`: expected `@key` or `@check name`", + ), + ("-- @Unique_key: a", "unknown directive `@Unique_key`"), + ("-- @description: ", "`@description` needs a value"), + ( + "-- @unique_key k: a", + "only `@check` takes a name, not `@unique_key`", + ), + ( + "-- @unique_key: a,,b", + "`@unique_key: a,,b` has an empty column", + ), + ( + "-- @full_refresh: yes", + "`@full_refresh` takes only `true`, not `yes`", + ), + ( + "-- @check Bad: x", + "check name `Bad` must use lower case letters, digits and `_`, and start with a letter", + ), + ]; + for (sql, message) in cases { + assert_eq!(parse_err(sql), format!("m.sql:1: {message}"), "{sql}"); + } + } + + #[test] + fn rejects_a_repeated_directive() { + let sql = "-- @description: a\n-- @description: b"; + assert_eq!( + parse_err(sql), + "m.sql:2: `@description` may appear only once" + ); + } + + #[test] + fn checks_may_repeat_but_names_may_not() { + let d = parse_ok("-- @check: a\n-- @check: b\n-- @check x: c"); + assert_eq!(d.checks.len(), 3); + let sql = "-- @check x: a\n-- @check: b\n-- @check x: c"; + assert_eq!(parse_err(sql), "m.sql:3: two checks are named `x`"); + } +} diff --git a/crates/compiler/src/graph.rs b/crates/compiler/src/graph.rs new file mode 100644 index 0000000..23673d5 --- /dev/null +++ b/crates/compiler/src/graph.rs @@ -0,0 +1,81 @@ +//! Checks the tables each model reads and puts the models in run order. + +use std::collections::{BTreeMap, BTreeSet}; + +use infostash_model::{Error, Result, TableRef}; + +/// A model as loaded, before its reads are checked. +pub(crate) struct Node { + pub table: TableRef, + /// The model file, for error messages. + pub file: String, + /// Each table read, with the first line that names it. + pub reads: BTreeMap, +} + +/// Check every read against the sources and models, then order the models +/// so that each comes after the models it reads. Ties go in table order, so +/// the order is the same on every run. +pub(crate) fn run_order(sources: &BTreeSet, nodes: Vec) -> Result> { + let mut pending: BTreeMap = + nodes.into_iter().map(|n| (n.table.clone(), n)).collect(); + for node in pending.values() { + for (table, line) in &node.reads { + let at = format!("{}:{line}", node.file); + if !sources.contains(table) && !pending.contains_key(table) { + return Err(Error::Parse(format!( + "{at}: no source or model defines `{table}`" + ))); + } + if table.layer > node.table.layer { + return Err(Error::Parse(format!( + "{at}: `{}` can't read `{table}`, which is in a later layer", + node.table + ))); + } + } + } + let mut ordered = Vec::with_capacity(pending.len()); + while let Some(next) = pending + .values() + .find(|n| n.reads.keys().all(|t| !pending.contains_key(t))) + .map(|n| n.table.clone()) + { + if let Some(node) = pending.remove(&next) { + ordered.push(node); + } + } + match pending.first_key_value() { + None => Ok(ordered), + Some((start, _)) => Err(cycle(&pending, start)), + } +} + +/// Name a cycle among models that can't be ordered. Each of them reads +/// another of them, so following those reads must come back round. +fn cycle(pending: &BTreeMap, start: &TableRef) -> Error { + let mut path: Vec<&TableRef> = vec![start]; + let mut current = start; + loop { + // Every model left reads another model left, or it would have run. + let next = pending + .get(current) + .and_then(|n| n.reads.keys().find(|t| pending.contains_key(*t))) + .unwrap_or(start); + if let Some(at) = path.iter().position(|t| *t == next) { + let first = path[at]; + let mut names: Vec = path[at..].iter().map(|t| t.to_string()).collect(); + names.push(next.to_string()); + let place = pending + .get(first) + .and_then(|n| { + let to = path.get(at + 1).copied().unwrap_or(next); + n.reads.get(to).map(|line| format!("{}:{line}", n.file)) + }) + .unwrap_or_default(); + return Error::Parse(format!("{place}: dependency cycle: {}", names.join(" -> "))); + } + path.push(next); + current = next; + } +} diff --git a/crates/compiler/src/lib.rs b/crates/compiler/src/lib.rs index 5b803ff..aa13466 100644 --- a/crates/compiler/src/lib.rs +++ b/crates/compiler/src/lib.rs @@ -2,29 +2,237 @@ //! incremental plans. Dependencies come from the table names each query //! reads; settings come from `-- @key: value` comments, so there is no //! templating language to learn. +//! +//! [`Project::load`] reads a project laid out as RFC 0001 describes, checks +//! every name, directive and table read, and returns the models in the order +//! they run. -use std::collections::BTreeMap; +use std::collections::{BTreeMap, BTreeSet}; +use std::fs; +use std::io; +use std::path::{Path, PathBuf}; -/// Settings parsed from `-- @key: value` comment lines. -pub fn directives(sql: &str) -> BTreeMap { - sql.lines() - .filter_map(|l| l.trim().strip_prefix("--")) - .filter_map(|l| l.trim().strip_prefix('@')) - .filter_map(|l| l.split_once(':')) - .map(|(k, v)| (k.trim().to_string(), v.trim().to_string())) - .collect() +use infostash_model::{Error, Layer, Result, TableRef}; + +mod config; +mod directives; +mod graph; +mod reads; + +pub use directives::{Check, Directives}; + +/// The project file at the root of every project. +const PROJECT_FILE: &str = "infostash.toml"; + +/// A loaded project: its sources, and its models in run order. +#[derive(Debug)] +pub struct Project { + name: String, + warehouse: PathBuf, + sources: Vec, + models: Vec, } -#[cfg(test)] -mod tests { - use super::*; +/// A Raw table, loaded by a connector. +#[derive(Debug, Clone, PartialEq)] +pub struct Source { + pub table: TableRef, + /// The source file, relative to the project root. + pub file: PathBuf, + pub connector: String, + /// Every key but `connector`. The connector checks them. + pub options: toml::Table, +} + +/// A Clean or Ready table, written by one SQL query. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Model { + pub table: TableRef, + /// The model file, relative to the project root. + pub file: PathBuf, + pub sql: String, + pub directives: Directives, + /// The tables the query reads. + pub inputs: BTreeSet, +} + +impl Project { + /// Load the project at `root`. Any error names the file, and the line + /// when there is one. + pub fn load(root: &Path) -> Result { + let text = match fs::read_to_string(root.join(PROJECT_FILE)) { + Ok(text) => text, + Err(e) if e.kind() == io::ErrorKind::NotFound => { + return Err(Error::NotFound(format!( + "no {PROJECT_FILE} in {}", + root.display() + ))); + } + Err(e) => return Err(io_error(&root.join(PROJECT_FILE), &e)), + }; + let file = config::project(PROJECT_FILE, &text)?; + let sources = load_sources(root)?; + let mut nodes = Vec::new(); + let mut models = BTreeMap::new(); + for layer in [Layer::Clean, Layer::Ready] { + for (table, path) in tables_in(root, layer.alias(), "sql", true)? { + let file = display(&path); + let sql = read(root, &path)?; + let directives = directives::parse(&file, &sql)?; + let reads = reads::tables_read(&file, &sql)?; + let table = TableRef::new(layer, table); + nodes.push(graph::Node { + table: table.clone(), + file, + reads: reads.clone(), + }); + models.insert( + table.clone(), + Model { + table, + file: path, + sql, + directives, + inputs: reads.into_keys().collect(), + }, + ); + } + } + let defined = sources.iter().map(|s| s.table.clone()).collect(); + let models = graph::run_order(&defined, nodes)? + .into_iter() + .filter_map(|node| models.remove(&node.table)) + .collect(); + Ok(Self { + name: file.project.name, + warehouse: root.join(file.warehouse.path), + sources, + models, + }) + } + + pub fn name(&self) -> &str { + &self.name + } + + /// The warehouse directory, resolved against the project root. + pub fn warehouse(&self) -> &Path { + &self.warehouse + } - #[test] - fn reads_directives() { - let sql = "-- @unique_key: order_id\n-- plain comment\n-- @partition_by: day\nselect 1"; - let d = directives(sql); - assert_eq!(d["unique_key"], "order_id"); - assert_eq!(d["partition_by"], "day"); - assert_eq!(d.len(), 2); + /// The sources, sorted by table. + pub fn sources(&self) -> &[Source] { + &self.sources } + + /// The models in run order: each comes after every model it reads. + pub fn models(&self) -> &[Model] { + &self.models + } +} + +/// Whether `name` is a valid table or check name: lower case letters, +/// digits and `_`, starting with a letter. +pub(crate) fn is_name(name: &str) -> bool { + name.starts_with(|c: char| c.is_ascii_lowercase()) + && name + .chars() + .all(|c| c.is_ascii_lowercase() || c.is_ascii_digit() || c == '_') } + +fn load_sources(root: &Path) -> Result> { + let mut sources = Vec::new(); + for (table, path) in tables_in(root, "sources", "toml", false)? { + let (connector, options) = config::source(&display(&path), &read(root, &path)?)?; + sources.push(Source { + table: TableRef::new(Layer::Raw, table), + file: path, + connector, + options, + }); + } + Ok(sources) +} + +/// The tables defined by the files with extension `ext` in `dir`, by name, +/// with each file's path relative to the root. A missing directory defines +/// none. Two files with one name are an error. +fn tables_in( + root: &Path, + dir: &str, + ext: &str, + recurse: bool, +) -> Result> { + let mut files = Vec::new(); + collect(root, Path::new(dir), ext, recurse, &mut files)?; + files.sort(); + let mut tables: BTreeMap = BTreeMap::new(); + for path in files { + let name = path + .file_stem() + .and_then(|s| s.to_str()) + .unwrap_or_default() + .to_string(); + if !is_name(&name) || name.ends_with("__quarantine") { + return Err(Error::Parse(format!( + "{}: a table name uses lower case letters, digits and `_`, starts with a letter \ + and does not end in `__quarantine`", + display(&path) + ))); + } + if let Some(first) = tables.get(&name) { + return Err(Error::Parse(format!( + "{} and {} both define `{name}`", + display(first), + display(&path) + ))); + } + tables.insert(name, path); + } + Ok(tables) +} + +/// Add the files under `dir` with extension `ext` to `files`. +fn collect( + root: &Path, + dir: &Path, + ext: &str, + recurse: bool, + files: &mut Vec, +) -> Result<()> { + let entries = match fs::read_dir(root.join(dir)) { + Ok(entries) => entries, + Err(e) if e.kind() == io::ErrorKind::NotFound => return Ok(()), + Err(e) => return Err(io_error(&root.join(dir), &e)), + }; + for entry in entries { + let entry = entry.map_err(|e| io_error(&root.join(dir), &e))?; + let path = dir.join(entry.file_name()); + let kind = fs::metadata(root.join(&path)).map_err(|e| io_error(&path, &e))?; + if kind.is_dir() && recurse { + collect(root, &path, ext, recurse, files)?; + } else if kind.is_file() && path.extension().is_some_and(|e| e == ext) { + files.push(path); + } + } + Ok(()) +} + +fn read(root: &Path, path: &Path) -> Result { + fs::read_to_string(root.join(path)).map_err(|e| io_error(path, &e)) +} + +/// A path as shown in messages, with `/` between parts on every platform. +fn display(path: &Path) -> String { + path.iter() + .map(|part| part.to_string_lossy()) + .collect::>() + .join("/") +} + +fn io_error(path: &Path, e: &io::Error) -> Error { + Error::Storage(format!("{}: {e}", path.display())) +} + +#[cfg(test)] +mod tests; diff --git a/crates/compiler/src/reads.rs b/crates/compiler/src/reads.rs new file mode 100644 index 0000000..98d0b4d --- /dev/null +++ b/crates/compiler/src/reads.rs @@ -0,0 +1,192 @@ +//! The tables a model's query reads, found in its parsed SQL. + +use std::collections::{BTreeMap, HashSet}; +use std::ops::ControlFlow; + +use datafusion::sql::parser::{DFParser, Statement}; +use datafusion::sql::sqlparser::ast::{ + Ident, ObjectName, Query, Statement as SqlStatement, TableFactor, Visit, Visitor, +}; +use infostash_model::{Error, Layer, Result, TableRef}; + +/// The tables a model's SQL reads, each with the first line that names it. +/// The SQL must be one query; names of its CTEs are not tables. +pub(crate) fn tables_read(file: &str, sql: &str) -> Result> { + let statements = DFParser::parse_sql(sql).map_err(|e| Error::Parse(format!("{file}: {e}")))?; + let query = match Vec::from(statements).as_slice() { + [Statement::Statement(statement)] => match statement.as_ref() { + SqlStatement::Query(query) => query.clone(), + _ => return Err(not_one_query(file)), + }, + _ => return Err(not_one_query(file)), + }; + let mut names = Names::default(); + let _ = query.visit(&mut names); + let mut tables = BTreeMap::new(); + for name in &names.tables { + if let Some((table, line)) = resolve(file, name, &names.ctes)? { + tables.entry(table).or_insert(line); + } + } + Ok(tables) +} + +fn not_one_query(file: &str) -> Error { + Error::Parse(format!("{file}: a model is one `select` statement")) +} + +/// The CTE names and table names in a query, collected in one walk. +#[derive(Default)] +struct Names { + ctes: HashSet, + tables: Vec, +} + +impl Visitor for Names { + type Break = (); + + fn pre_visit_query(&mut self, query: &Query) -> ControlFlow<()> { + if let Some(with) = &query.with { + self.ctes + .extend(with.cte_tables.iter().map(|cte| normalise(&cte.alias.name))); + } + ControlFlow::Continue(()) + } + + fn pre_visit_table_factor(&mut self, factor: &TableFactor) -> ControlFlow<()> { + // A table function such as `generate_series(1, 3)` reads no table. + if let TableFactor::Table { + name, args: None, .. + } = factor + { + self.tables.push(name.clone()); + } + ControlFlow::Continue(()) + } +} + +/// The table a name refers to and its line, or `None` for a CTE. +fn resolve( + file: &str, + name: &ObjectName, + ctes: &HashSet, +) -> Result> { + let parts: Vec<&Ident> = name.0.iter().filter_map(|part| part.as_ident()).collect(); + let line = parts.first().map_or(0, |ident| { + usize::try_from(ident.span.start.line).unwrap_or(0) + }); + let err = |message: String| Error::Parse(format!("{file}:{line}: {message}")); + if parts.len() != name.0.len() { + return Err(err(format!("`{name}` is not a table name"))); + } + match parts.as_slice() { + [table] if ctes.contains(&normalise(table)) => Ok(None), + [table] => { + let table = normalise(table); + Err(err(format!( + "`{table}` needs its layer, as in `raw.{table}`, `clean.{table}` or `ready.{table}`" + ))) + } + [layer, table] => match Layer::from_namespace(&normalise(layer)) { + Some(layer) => Ok(Some((TableRef::new(layer, normalise(table)), line))), + None => Err(err(format!( + "`{name}`: unknown layer `{}`; use raw, clean or ready", + normalise(layer) + ))), + }, + _ => Err(err(format!("`{name}`: name a table as `.`"))), + } +} + +/// An identifier as DataFusion resolves it: unquoted names are lower case. +fn normalise(ident: &Ident) -> String { + match ident.quote_style { + Some(_) => ident.value.clone(), + None => ident.value.to_lowercase(), + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn read(sql: &str) -> Vec<(String, usize)> { + tables_read("m.sql", sql) + .unwrap() + .into_iter() + .map(|(table, line)| (table.to_string(), line)) + .collect() + } + + fn read_err(sql: &str) -> String { + match tables_read("m.sql", sql) { + Err(Error::Parse(message)) => message, + other => panic!("expected a parse error, got {other:?}"), + } + } + + #[test] + fn finds_tables_in_joins_and_subqueries_with_their_first_line() { + let sql = "select *\nfrom raw.a\njoin (select * from clean.b) b on true\n\ + where exists (select 1 from raw.a)\nunion all select * from gold.c;"; + assert_eq!( + read(sql), + [ + ("bronze.a".to_string(), 2), + ("silver.b".to_string(), 3), + ("gold.c".to_string(), 5) + ] + ); + } + + #[test] + fn cte_names_are_not_tables() { + let sql = "with recent as (select * from raw.orders)\nselect * from recent"; + assert_eq!(read(sql), [("bronze.orders".to_string(), 1)]); + } + + #[test] + fn table_functions_read_no_table() { + assert!(read("select * from generate_series(1, 3)").is_empty()); + } + + #[test] + fn unquoted_names_are_lower_cased() { + assert_eq!( + read("select * from RAW.Orders"), + [("bronze.orders".to_string(), 1)] + ); + assert_eq!( + read("select * from raw.\"Orders\""), + [("bronze.Orders".to_string(), 1)] + ); + } + + #[test] + fn rejects_names_without_a_known_layer() { + assert_eq!( + read_err("select *\nfrom orders"), + "m.sql:2: `orders` needs its layer, as in `raw.orders`, `clean.orders` or `ready.orders`" + ); + assert_eq!( + read_err("select * from staging.orders"), + "m.sql:1: `staging.orders`: unknown layer `staging`; use raw, clean or ready" + ); + assert_eq!( + read_err("select * from cat.raw.orders"), + "m.sql:1: `cat.raw.orders`: name a table as `.
`" + ); + } + + #[test] + fn a_model_is_one_query() { + for sql in ["select 1; select 2", "create table x (a int)", ""] { + assert_eq!( + read_err(sql), + "m.sql: a model is one `select` statement", + "{sql}" + ); + } + assert!(read_err("select (").starts_with("m.sql: ")); + } +} diff --git a/crates/compiler/src/tests.rs b/crates/compiler/src/tests.rs new file mode 100644 index 0000000..325d2dc --- /dev/null +++ b/crates/compiler/src/tests.rs @@ -0,0 +1,304 @@ +//! Tests that load whole projects from temporary directories. + +use std::fs; +use std::path::Path; + +use infostash_model::{Error, Layer, TableRef}; +use tempfile::TempDir; + +use crate::{Check, Project}; + +const PROJECT: &str = + "[project]\nname = \"shop\"\nformat = 1\n\n[warehouse]\npath = \".infostash/warehouse\"\n"; +const ORDERS: &str = "connector = \"csv\"\npath = \"data/orders/*.csv\"\n"; + +/// A project directory holding `files`, each as (path, contents). +fn project(files: &[(&str, &str)]) -> TempDir { + let dir = TempDir::new().unwrap(); + for (path, text) in files { + let path = dir.path().join(path); + fs::create_dir_all(path.parent().unwrap()).unwrap(); + fs::write(path, text).unwrap(); + } + dir +} + +/// A shop project with the orders source and the given models. +fn shop(models: &[(&str, &str)]) -> TempDir { + let mut files = vec![("infostash.toml", PROJECT), ("sources/orders.toml", ORDERS)]; + files.extend_from_slice(models); + project(&files) +} + +fn parse_error(dir: &TempDir) -> String { + match Project::load(dir.path()) { + Err(Error::Parse(message)) => message, + other => panic!("expected a parse error, got {other:?}"), + } +} + +fn silver(name: &str) -> TableRef { + TableRef::new(Layer::Clean, name) +} + +fn gold(name: &str) -> TableRef { + TableRef::new(Layer::Ready, name) +} + +#[test] +fn loads_the_rfc_example() { + let dir = shop(&[ + ( + "clean/orders.sql", + "-- clean/orders.sql\n\ + -- @unique_key: order_id\n\ + -- @check not_negative: amount >= 0\n\ + -- @check: customer_id is not null\n\ + -- @description: One row per order, latest version wins.\n\ + select try_cast(amount as decimal(12, 2)) as amount from raw.orders\n", + ), + ( + "ready/daily_revenue.sql", + "-- @full_refresh: true\nselect sum(amount) as revenue from clean.orders\n", + ), + ]); + let project = Project::load(dir.path()).unwrap(); + assert_eq!(project.name(), "shop"); + assert_eq!(project.warehouse(), dir.path().join(".infostash/warehouse")); + let source = &project.sources()[0]; + assert_eq!(source.table, TableRef::new(Layer::Raw, "orders")); + assert_eq!(source.connector, "csv"); + assert_eq!(source.options["path"].as_str(), Some("data/orders/*.csv")); + assert!(!source.options.contains_key("connector")); + + let [orders, revenue] = project.models() else { + panic!("expected two models"); + }; + assert_eq!(orders.table, silver("orders")); + assert_eq!(orders.file, Path::new("clean/orders.sql")); + assert_eq!(orders.directives.unique_key, Some(vec!["order_id".into()])); + assert_eq!( + orders.directives.checks, + vec![ + Check { + name: Some("not_negative".into()), + expr: "amount >= 0".into(), + line: 3 + }, + Check { + name: None, + expr: "customer_id is not null".into(), + line: 4 + }, + ] + ); + assert_eq!( + orders.directives.description.as_deref(), + Some("One row per order, latest version wins.") + ); + assert!(!orders.directives.full_refresh); + assert_eq!(orders.inputs, [TableRef::new(Layer::Raw, "orders")].into()); + assert_eq!(revenue.table, gold("daily_revenue")); + assert!(revenue.directives.full_refresh); + assert_eq!(revenue.inputs, [silver("orders")].into()); +} + +#[test] +fn orders_models_after_what_they_read() { + // In name order `a` would run first, but it reads `b`, which reads `c`. + let dir = shop(&[ + ("ready/a.sql", "select * from gold.b"), + ("ready/b.sql", "select * from ready.c join clean.d on true"), + ("ready/c.sql", "select * from silver.d"), + ("clean/d.sql", "select * from bronze.orders"), + ("clean/e.sql", "select * from raw.orders"), + ]); + let project = Project::load(dir.path()).unwrap(); + let order: Vec = project + .models() + .iter() + .map(|m| m.table.to_string()) + .collect(); + assert_eq!( + order, + ["silver.d", "silver.e", "gold.c", "gold.b", "gold.a"] + ); +} + +#[test] +fn subdirectories_do_not_change_the_name() { + let dir = shop(&[("clean/sales/eu/orders.sql", "select * from raw.orders")]); + let project = Project::load(dir.path()).unwrap(); + assert_eq!(project.models()[0].table, silver("orders")); + assert_eq!( + project.models()[0].file, + Path::new("clean/sales/eu/orders.sql") + ); +} + +#[test] +fn ignores_other_files() { + let dir = shop(&[ + ("README.md", "# shop"), + ("clean/notes.txt", "not a model"), + ("sources/nested/extra.toml", "not = \"a source\""), + ]); + let project = Project::load(dir.path()).unwrap(); + assert_eq!(project.sources().len(), 1); + assert!(project.models().is_empty()); +} + +#[test] +fn a_missing_project_file_is_not_found() { + let dir = project(&[]); + assert!(matches!(Project::load(dir.path()), Err(Error::NotFound(_)))); +} + +#[test] +fn rejects_unknown_project_keys() { + let dir = project(&[("infostash.toml", &format!("{PROJECT}\n[extra]\nkey = 1\n"))]); + let message = parse_error(&dir); + assert!(message.starts_with("infostash.toml:8: "), "{message}"); + assert!(message.contains("extra"), "{message}"); +} + +#[test] +fn rejects_another_format() { + let dir = project(&[( + "infostash.toml", + &PROJECT.replace("format = 1", "format = 2"), + )]); + assert!(parse_error(&dir).contains("format 2 is not supported")); +} + +#[test] +fn a_source_needs_a_string_connector() { + let missing = shop(&[("sources/items.toml", "path = \"x.csv\"")]); + assert_eq!( + parse_error(&missing), + "sources/items.toml: `connector` is missing" + ); + let number = shop(&[("sources/items.toml", "connector = 1")]); + assert_eq!( + parse_error(&number), + "sources/items.toml: `connector` must be a string" + ); +} + +#[test] +fn rejects_bad_table_names() { + for name in ["Orders", "1orders", "orders__quarantine", "my-orders"] { + let dir = shop(&[(&format!("clean/{name}.sql"), "select 1")]); + let message = parse_error(&dir); + assert!( + message.starts_with(&format!("clean/{name}.sql: ")), + "{message}" + ); + } +} + +#[test] +fn rejects_two_files_with_one_name_in_a_layer() { + let dir = shop(&[ + ("clean/a/orders.sql", "select * from raw.orders"), + ("clean/b/orders.sql", "select * from raw.orders"), + ]); + assert_eq!( + parse_error(&dir), + "clean/a/orders.sql and clean/b/orders.sql both define `orders`" + ); +} + +#[test] +fn one_name_may_appear_in_each_layer() { + let dir = shop(&[ + ("clean/orders.sql", "select * from raw.orders"), + ("ready/orders.sql", "select * from clean.orders"), + ]); + assert_eq!(Project::load(dir.path()).unwrap().models().len(), 2); +} + +#[test] +fn rejects_a_read_of_an_undefined_table() { + let dir = shop(&[("clean/orders.sql", "-- note\nselect *\nfrom raw.items")]); + assert_eq!( + parse_error(&dir), + "clean/orders.sql:3: no source or model defines `bronze.items`" + ); +} + +#[test] +fn rejects_a_read_of_a_later_layer() { + let dir = shop(&[ + ("clean/orders.sql", "select * from ready.revenue"), + ("ready/revenue.sql", "select * from raw.orders"), + ]); + assert_eq!( + parse_error(&dir), + "clean/orders.sql:1: `silver.orders` can't read `gold.revenue`, which is in a later layer" + ); +} + +#[test] +fn ready_may_read_raw() { + let dir = shop(&[("ready/orders.sql", "select * from raw.orders")]); + assert!(Project::load(dir.path()).is_ok()); +} + +#[test] +fn names_a_cycle() { + let dir = shop(&[ + ("clean/a.sql", "select * from clean.b"), + ("clean/b.sql", "select * from\nclean.c"), + ("clean/c.sql", "select * from clean.b"), + ]); + assert_eq!( + parse_error(&dir), + "clean/b.sql:2: dependency cycle: silver.b -> silver.c -> silver.b" + ); +} + +#[test] +fn a_model_reading_itself_is_a_cycle() { + let dir = shop(&[("clean/a.sql", "select * from clean.a")]); + assert_eq!( + parse_error(&dir), + "clean/a.sql:1: dependency cycle: silver.a -> silver.a" + ); +} + +#[test] +fn rejects_a_bad_directive_with_its_line() { + let dir = shop(&[( + "clean/orders.sql", + "-- top\n-- @partition_by: day\nselect * from raw.orders", + )]); + assert_eq!( + parse_error(&dir), + "clean/orders.sql:2: unknown directive `@partition_by`" + ); +} + +#[test] +fn rejects_sql_that_is_not_one_query() { + let dir = shop(&[("clean/orders.sql", "select 1; select 2")]); + assert_eq!( + parse_error(&dir), + "clean/orders.sql: a model is one `select` statement" + ); +} + +#[test] +fn reports_a_project_file_that_cannot_be_read() { + let dir = project(&[("infostash.toml/x", "")]); + assert!(matches!(Project::load(dir.path()), Err(Error::Storage(_)))); +} + +#[test] +fn reports_a_model_directory_that_cannot_be_read() { + let dir = project(&[ + ("infostash.toml", PROJECT), + ("clean", "a file, not a directory"), + ]); + assert!(matches!(Project::load(dir.path()), Err(Error::Storage(_)))); +}