diff --git a/Cargo.lock b/Cargo.lock index ac79f65..a721b7f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2599,6 +2599,13 @@ dependencies = [ [[package]] name = "infostash-compiler" version = "0.1.0" +dependencies = [ + "datafusion", + "infostash-model", + "serde", + "tempfile", + "toml", +] [[package]] name = "infostash-connectors" @@ -3891,6 +3898,15 @@ dependencies = [ "syn 3.0.9", ] +[[package]] +name = "serde_spanned" +version = "1.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c187e0a369471c550af028cba56d4f4fa2a489ad28151ae95a4d340a2a8b782" +dependencies = [ + "serde_core", +] + [[package]] name = "serde_urlencoded" version = "0.7.1" @@ -4537,6 +4553,45 @@ dependencies = [ "tokio", ] +[[package]] +name = "toml" +version = "1.2.1+spec-1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d5755c51863cdac711f1956baf8eb9588e060755f7d3c4c979e70ea1e1f32cd7" +dependencies = [ + "indexmap 2.14.2", + "serde_core", + "serde_spanned", + "toml_datetime", + "toml_parser", + "toml_writer", + "winnow", +] + +[[package]] +name = "toml_datetime" +version = "1.1.3+spec-1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "173f25c883cd2176a2d8212155638c0a60d484a9d84546d270866f40f56b6e49" +dependencies = [ + "serde_core", +] + +[[package]] +name = "toml_parser" +version = "1.1.5+spec-1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "baa693a8032d7e1cada7d0041e96126df243179ff061456783ac7f12bda4744c" +dependencies = [ + "winnow", +] + +[[package]] +name = "toml_writer" +version = "1.2.0+spec-1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "009dbb7700f11c55ef83aebbe4c0e9cf1f1ac0844bd8d6f00dc38f302026e13a" + [[package]] name = "tower" version = "0.5.3" @@ -5121,6 +5176,12 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" +[[package]] +name = "winnow" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8c3b93199df0ab874cdf0c6a94f0a0dc3423fe41f1cafb62b8bd7b1deb277aef" + [[package]] name = "wit-bindgen" version = "0.57.1" diff --git a/crates/compiler/Cargo.toml b/crates/compiler/Cargo.toml index 653ec60..deff8b2 100644 --- a/crates/compiler/Cargo.toml +++ b/crates/compiler/Cargo.toml @@ -7,6 +7,13 @@ license.workspace = true repository.workspace = true [dependencies] +datafusion.workspace = true +infostash-model.workspace = true +serde = { version = "1.0.229", features = ["derive"] } +toml = "1.2.1" + +[dev-dependencies] +tempfile = "3.27.0" [lints] workspace = true diff --git a/crates/compiler/src/config.rs b/crates/compiler/src/config.rs new file mode 100644 index 0000000..7002326 --- /dev/null +++ b/crates/compiler/src/config.rs @@ -0,0 +1,68 @@ +//! The TOML files of a project: `infostash.toml` and `sources/*.toml`. + +use std::path::PathBuf; + +use infostash_model::{Error, Result}; +use serde::Deserialize; + +/// The project format this version reads. +const FORMAT: u32 = 1; + +/// `infostash.toml`, as written. +#[derive(Deserialize)] +#[serde(deny_unknown_fields)] +pub(crate) struct ProjectFile { + pub project: ProjectSection, + pub warehouse: WarehouseSection, +} + +#[derive(Deserialize)] +#[serde(deny_unknown_fields)] +pub(crate) struct ProjectSection { + pub name: String, + pub format: u32, +} + +#[derive(Deserialize)] +#[serde(deny_unknown_fields)] +pub(crate) struct WarehouseSection { + pub path: PathBuf, +} + +/// Read `infostash.toml`. Unknown keys and any format but this one are +/// errors. +pub(crate) fn project(file: &str, text: &str) -> Result { + let parsed: ProjectFile = toml::from_str(text).map_err(|e| toml_error(file, text, &e))?; + if parsed.project.format != FORMAT { + return Err(Error::Parse(format!( + "{file}: format {} is not supported; this version reads format {FORMAT}", + parsed.project.format + ))); + } + Ok(parsed) +} + +/// Read a source file into its connector and the keys that belong to it. +pub(crate) fn source(file: &str, text: &str) -> Result<(String, toml::Table)> { + let mut options: toml::Table = toml::from_str(text).map_err(|e| toml_error(file, text, &e))?; + match options.remove("connector") { + Some(toml::Value::String(connector)) => Ok((connector, options)), + Some(_) => Err(Error::Parse(format!( + "{file}: `connector` must be a string" + ))), + None => Err(Error::Parse(format!("{file}: `connector` is missing"))), + } +} + +/// A TOML error as `file:line: message`, or `file: message` when the +/// parser gives no position. +fn toml_error(file: &str, text: &str, e: &toml::de::Error) -> Error { + let line = e + .span() + .and_then(|span| text.get(..span.start)) + .map(|before| before.matches('\n').count() + 1); + match line { + Some(line) => Error::Parse(format!("{file}:{line}: {}", e.message())), + None => Error::Parse(format!("{file}: {}", e.message())), + } +} diff --git a/crates/compiler/src/directives.rs b/crates/compiler/src/directives.rs new file mode 100644 index 0000000..7b4d7b6 --- /dev/null +++ b/crates/compiler/src/directives.rs @@ -0,0 +1,215 @@ +//! The header grammar of RFC 0001: `-- @key: value` comments above the SQL. + +use infostash_model::{Error, Result}; + +/// A model's settings, read from the `-- @key: value` comments in its +/// header. Which of them a run accepts is decided elsewhere. +#[derive(Debug, Clone, Default, PartialEq, Eq)] +pub struct Directives { + /// `@unique_key`: the key columns, in the order written. + pub unique_key: Option>, + /// `@check`: every check, in file order. + pub checks: Vec, + /// `@full_refresh: true`. + pub full_refresh: bool, + /// `@description`. + pub description: Option, +} + +/// One `@check` directive. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Check { + /// The name written between `@check` and the colon, if any. + pub name: Option, + /// The SQL boolean expression, as written. + pub expr: String, + /// The line of the model file it is on, counting from 1. + pub line: usize, +} + +/// Read the directives in the header of a model file. The header is the run +/// of blank and `--` lines at the top; a directive starts at column one with +/// `-- @`, and anything after the header is SQL, not a directive. +pub(crate) fn parse(file: &str, sql: &str) -> Result { + let mut directives = Directives::default(); + let mut once = Vec::new(); + for (index, line) in sql.lines().enumerate() { + let trimmed = line.trim(); + if !trimmed.is_empty() && !trimmed.starts_with("--") { + break; + } + let Some(rest) = line.strip_prefix("-- @") else { + continue; + }; + let err = |message: String| Error::Parse(format!("{file}:{}: {message}", index + 1)); + let (head, value) = rest + .split_once(':') + .ok_or_else(|| err(format!("`-- @{}` needs a `:` and a value", rest.trim())))?; + let value = value.trim(); + let mut words = head.split_whitespace(); + let key = words.next().unwrap_or_default(); + let name = words.next(); + if words.next().is_some() { + return Err(err(format!( + "`@{}`: expected `@key` or `@check name`", + head.trim() + ))); + } + if !matches!(key, "unique_key" | "check" | "full_refresh" | "description") { + return Err(err(format!("unknown directive `@{key}`"))); + } + if value.is_empty() { + return Err(err(format!("`@{key}` needs a value"))); + } + if key == "check" { + directives + .checks + .push(check(name, value, index + 1).map_err(err)?); + continue; + } + if name.is_some() { + return Err(err(format!("only `@check` takes a name, not `@{key}`"))); + } + if once.contains(&key) { + return Err(err(format!("`@{key}` may appear only once"))); + } + once.push(key); + match key { + "unique_key" => directives.unique_key = Some(columns(value).map_err(err)?), + "full_refresh" if value == "true" => directives.full_refresh = true, + "full_refresh" => { + return Err(err(format!( + "`@full_refresh` takes only `true`, not `{value}`" + ))); + } + _ => directives.description = Some(value.to_string()), + } + } + check_unique_names(&directives.checks) + .map_err(|(line, message)| Error::Parse(format!("{file}:{line}: {message}")))?; + Ok(directives) +} + +fn check(name: Option<&str>, expr: &str, line: usize) -> std::result::Result { + if let Some(name) = name.filter(|n| !crate::is_name(n)) { + return Err(format!( + "check name `{name}` must use lower case letters, digits and `_`, and start with a letter" + )); + } + Ok(Check { + name: name.map(str::to_string), + expr: expr.to_string(), + line, + }) +} + +fn columns(value: &str) -> std::result::Result, String> { + let columns: Vec = value.split(',').map(|c| c.trim().to_string()).collect(); + if columns.iter().any(String::is_empty) { + return Err(format!("`@unique_key: {value}` has an empty column")); + } + Ok(columns) +} + +fn check_unique_names(checks: &[Check]) -> std::result::Result<(), (usize, String)> { + for (i, check) in checks.iter().enumerate() { + let Some(name) = &check.name else { continue }; + if checks[..i].iter().any(|c| c.name.as_ref() == Some(name)) { + return Err((check.line, format!("two checks are named `{name}`"))); + } + } + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn parse_ok(sql: &str) -> Directives { + parse("m.sql", sql).unwrap() + } + + fn parse_err(sql: &str) -> String { + match parse("m.sql", sql) { + Err(Error::Parse(message)) => message, + other => panic!("expected a parse error, got {other:?}"), + } + } + + #[test] + fn values_may_hold_colons_and_are_trimmed() { + let d = parse_ok("-- \n-- @check: amount::decimal > 0 \nselect 1"); + assert_eq!(d.checks[0].expr, "amount::decimal > 0"); + assert_eq!(d.checks[0].line, 2); + } + + #[test] + fn splits_a_compound_unique_key() { + let d = parse_ok("-- @unique_key: order_id, line_no\nselect 1"); + assert_eq!( + d.unique_key, + Some(vec!["order_id".into(), "line_no".into()]) + ); + } + + #[test] + fn other_comment_shapes_are_ordinary_comments() { + let d = parse_ok("--@unique_key: a\n -- @unique_key: b\n-- plain\n\nselect 1"); + assert_eq!(d, Directives::default()); + } + + #[test] + fn a_directive_after_the_sql_starts_is_a_comment() { + let d = parse_ok("select 1\n-- @nonsense: here\n-- @check: x"); + assert_eq!(d, Directives::default()); + } + + #[test] + fn rejects_malformed_directives() { + let cases = [ + ("-- @unique_key", "`-- @unique_key` needs a `:` and a value"), + ( + "-- @check a b: x", + "`@check a b`: expected `@key` or `@check name`", + ), + ("-- @Unique_key: a", "unknown directive `@Unique_key`"), + ("-- @description: ", "`@description` needs a value"), + ( + "-- @unique_key k: a", + "only `@check` takes a name, not `@unique_key`", + ), + ( + "-- @unique_key: a,,b", + "`@unique_key: a,,b` has an empty column", + ), + ( + "-- @full_refresh: yes", + "`@full_refresh` takes only `true`, not `yes`", + ), + ( + "-- @check Bad: x", + "check name `Bad` must use lower case letters, digits and `_`, and start with a letter", + ), + ]; + for (sql, message) in cases { + assert_eq!(parse_err(sql), format!("m.sql:1: {message}"), "{sql}"); + } + } + + #[test] + fn rejects_a_repeated_directive() { + let sql = "-- @description: a\n-- @description: b"; + assert_eq!( + parse_err(sql), + "m.sql:2: `@description` may appear only once" + ); + } + + #[test] + fn checks_may_repeat_but_names_may_not() { + let d = parse_ok("-- @check: a\n-- @check: b\n-- @check x: c"); + assert_eq!(d.checks.len(), 3); + let sql = "-- @check x: a\n-- @check: b\n-- @check x: c"; + assert_eq!(parse_err(sql), "m.sql:3: two checks are named `x`"); + } +} diff --git a/crates/compiler/src/graph.rs b/crates/compiler/src/graph.rs new file mode 100644 index 0000000..23673d5 --- /dev/null +++ b/crates/compiler/src/graph.rs @@ -0,0 +1,81 @@ +//! Checks the tables each model reads and puts the models in run order. + +use std::collections::{BTreeMap, BTreeSet}; + +use infostash_model::{Error, Result, TableRef}; + +/// A model as loaded, before its reads are checked. +pub(crate) struct Node { + pub table: TableRef, + /// The model file, for error messages. + pub file: String, + /// Each table read, with the first line that names it. + pub reads: BTreeMap, +} + +/// Check every read against the sources and models, then order the models +/// so that each comes after the models it reads. Ties go in table order, so +/// the order is the same on every run. +pub(crate) fn run_order(sources: &BTreeSet, nodes: Vec) -> Result> { + let mut pending: BTreeMap = + nodes.into_iter().map(|n| (n.table.clone(), n)).collect(); + for node in pending.values() { + for (table, line) in &node.reads { + let at = format!("{}:{line}", node.file); + if !sources.contains(table) && !pending.contains_key(table) { + return Err(Error::Parse(format!( + "{at}: no source or model defines `{table}`" + ))); + } + if table.layer > node.table.layer { + return Err(Error::Parse(format!( + "{at}: `{}` can't read `{table}`, which is in a later layer", + node.table + ))); + } + } + } + let mut ordered = Vec::with_capacity(pending.len()); + while let Some(next) = pending + .values() + .find(|n| n.reads.keys().all(|t| !pending.contains_key(t))) + .map(|n| n.table.clone()) + { + if let Some(node) = pending.remove(&next) { + ordered.push(node); + } + } + match pending.first_key_value() { + None => Ok(ordered), + Some((start, _)) => Err(cycle(&pending, start)), + } +} + +/// Name a cycle among models that can't be ordered. Each of them reads +/// another of them, so following those reads must come back round. +fn cycle(pending: &BTreeMap, start: &TableRef) -> Error { + let mut path: Vec<&TableRef> = vec![start]; + let mut current = start; + loop { + // Every model left reads another model left, or it would have run. + let next = pending + .get(current) + .and_then(|n| n.reads.keys().find(|t| pending.contains_key(*t))) + .unwrap_or(start); + if let Some(at) = path.iter().position(|t| *t == next) { + let first = path[at]; + let mut names: Vec = path[at..].iter().map(|t| t.to_string()).collect(); + names.push(next.to_string()); + let place = pending + .get(first) + .and_then(|n| { + let to = path.get(at + 1).copied().unwrap_or(next); + n.reads.get(to).map(|line| format!("{}:{line}", n.file)) + }) + .unwrap_or_default(); + return Error::Parse(format!("{place}: dependency cycle: {}", names.join(" -> "))); + } + path.push(next); + current = next; + } +} diff --git a/crates/compiler/src/lib.rs b/crates/compiler/src/lib.rs index 5b803ff..aa13466 100644 --- a/crates/compiler/src/lib.rs +++ b/crates/compiler/src/lib.rs @@ -2,29 +2,237 @@ //! incremental plans. Dependencies come from the table names each query //! reads; settings come from `-- @key: value` comments, so there is no //! templating language to learn. +//! +//! [`Project::load`] reads a project laid out as RFC 0001 describes, checks +//! every name, directive and table read, and returns the models in the order +//! they run. -use std::collections::BTreeMap; +use std::collections::{BTreeMap, BTreeSet}; +use std::fs; +use std::io; +use std::path::{Path, PathBuf}; -/// Settings parsed from `-- @key: value` comment lines. -pub fn directives(sql: &str) -> BTreeMap { - sql.lines() - .filter_map(|l| l.trim().strip_prefix("--")) - .filter_map(|l| l.trim().strip_prefix('@')) - .filter_map(|l| l.split_once(':')) - .map(|(k, v)| (k.trim().to_string(), v.trim().to_string())) - .collect() +use infostash_model::{Error, Layer, Result, TableRef}; + +mod config; +mod directives; +mod graph; +mod reads; + +pub use directives::{Check, Directives}; + +/// The project file at the root of every project. +const PROJECT_FILE: &str = "infostash.toml"; + +/// A loaded project: its sources, and its models in run order. +#[derive(Debug)] +pub struct Project { + name: String, + warehouse: PathBuf, + sources: Vec, + models: Vec, } -#[cfg(test)] -mod tests { - use super::*; +/// A Raw table, loaded by a connector. +#[derive(Debug, Clone, PartialEq)] +pub struct Source { + pub table: TableRef, + /// The source file, relative to the project root. + pub file: PathBuf, + pub connector: String, + /// Every key but `connector`. The connector checks them. + pub options: toml::Table, +} + +/// A Clean or Ready table, written by one SQL query. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Model { + pub table: TableRef, + /// The model file, relative to the project root. + pub file: PathBuf, + pub sql: String, + pub directives: Directives, + /// The tables the query reads. + pub inputs: BTreeSet, +} + +impl Project { + /// Load the project at `root`. Any error names the file, and the line + /// when there is one. + pub fn load(root: &Path) -> Result { + let text = match fs::read_to_string(root.join(PROJECT_FILE)) { + Ok(text) => text, + Err(e) if e.kind() == io::ErrorKind::NotFound => { + return Err(Error::NotFound(format!( + "no {PROJECT_FILE} in {}", + root.display() + ))); + } + Err(e) => return Err(io_error(&root.join(PROJECT_FILE), &e)), + }; + let file = config::project(PROJECT_FILE, &text)?; + let sources = load_sources(root)?; + let mut nodes = Vec::new(); + let mut models = BTreeMap::new(); + for layer in [Layer::Clean, Layer::Ready] { + for (table, path) in tables_in(root, layer.alias(), "sql", true)? { + let file = display(&path); + let sql = read(root, &path)?; + let directives = directives::parse(&file, &sql)?; + let reads = reads::tables_read(&file, &sql)?; + let table = TableRef::new(layer, table); + nodes.push(graph::Node { + table: table.clone(), + file, + reads: reads.clone(), + }); + models.insert( + table.clone(), + Model { + table, + file: path, + sql, + directives, + inputs: reads.into_keys().collect(), + }, + ); + } + } + let defined = sources.iter().map(|s| s.table.clone()).collect(); + let models = graph::run_order(&defined, nodes)? + .into_iter() + .filter_map(|node| models.remove(&node.table)) + .collect(); + Ok(Self { + name: file.project.name, + warehouse: root.join(file.warehouse.path), + sources, + models, + }) + } + + pub fn name(&self) -> &str { + &self.name + } + + /// The warehouse directory, resolved against the project root. + pub fn warehouse(&self) -> &Path { + &self.warehouse + } - #[test] - fn reads_directives() { - let sql = "-- @unique_key: order_id\n-- plain comment\n-- @partition_by: day\nselect 1"; - let d = directives(sql); - assert_eq!(d["unique_key"], "order_id"); - assert_eq!(d["partition_by"], "day"); - assert_eq!(d.len(), 2); + /// The sources, sorted by table. + pub fn sources(&self) -> &[Source] { + &self.sources } + + /// The models in run order: each comes after every model it reads. + pub fn models(&self) -> &[Model] { + &self.models + } +} + +/// Whether `name` is a valid table or check name: lower case letters, +/// digits and `_`, starting with a letter. +pub(crate) fn is_name(name: &str) -> bool { + name.starts_with(|c: char| c.is_ascii_lowercase()) + && name + .chars() + .all(|c| c.is_ascii_lowercase() || c.is_ascii_digit() || c == '_') } + +fn load_sources(root: &Path) -> Result> { + let mut sources = Vec::new(); + for (table, path) in tables_in(root, "sources", "toml", false)? { + let (connector, options) = config::source(&display(&path), &read(root, &path)?)?; + sources.push(Source { + table: TableRef::new(Layer::Raw, table), + file: path, + connector, + options, + }); + } + Ok(sources) +} + +/// The tables defined by the files with extension `ext` in `dir`, by name, +/// with each file's path relative to the root. A missing directory defines +/// none. Two files with one name are an error. +fn tables_in( + root: &Path, + dir: &str, + ext: &str, + recurse: bool, +) -> Result> { + let mut files = Vec::new(); + collect(root, Path::new(dir), ext, recurse, &mut files)?; + files.sort(); + let mut tables: BTreeMap = BTreeMap::new(); + for path in files { + let name = path + .file_stem() + .and_then(|s| s.to_str()) + .unwrap_or_default() + .to_string(); + if !is_name(&name) || name.ends_with("__quarantine") { + return Err(Error::Parse(format!( + "{}: a table name uses lower case letters, digits and `_`, starts with a letter \ + and does not end in `__quarantine`", + display(&path) + ))); + } + if let Some(first) = tables.get(&name) { + return Err(Error::Parse(format!( + "{} and {} both define `{name}`", + display(first), + display(&path) + ))); + } + tables.insert(name, path); + } + Ok(tables) +} + +/// Add the files under `dir` with extension `ext` to `files`. +fn collect( + root: &Path, + dir: &Path, + ext: &str, + recurse: bool, + files: &mut Vec, +) -> Result<()> { + let entries = match fs::read_dir(root.join(dir)) { + Ok(entries) => entries, + Err(e) if e.kind() == io::ErrorKind::NotFound => return Ok(()), + Err(e) => return Err(io_error(&root.join(dir), &e)), + }; + for entry in entries { + let entry = entry.map_err(|e| io_error(&root.join(dir), &e))?; + let path = dir.join(entry.file_name()); + let kind = fs::metadata(root.join(&path)).map_err(|e| io_error(&path, &e))?; + if kind.is_dir() && recurse { + collect(root, &path, ext, recurse, files)?; + } else if kind.is_file() && path.extension().is_some_and(|e| e == ext) { + files.push(path); + } + } + Ok(()) +} + +fn read(root: &Path, path: &Path) -> Result { + fs::read_to_string(root.join(path)).map_err(|e| io_error(path, &e)) +} + +/// A path as shown in messages, with `/` between parts on every platform. +fn display(path: &Path) -> String { + path.iter() + .map(|part| part.to_string_lossy()) + .collect::>() + .join("/") +} + +fn io_error(path: &Path, e: &io::Error) -> Error { + Error::Storage(format!("{}: {e}", path.display())) +} + +#[cfg(test)] +mod tests; diff --git a/crates/compiler/src/reads.rs b/crates/compiler/src/reads.rs new file mode 100644 index 0000000..98d0b4d --- /dev/null +++ b/crates/compiler/src/reads.rs @@ -0,0 +1,192 @@ +//! The tables a model's query reads, found in its parsed SQL. + +use std::collections::{BTreeMap, HashSet}; +use std::ops::ControlFlow; + +use datafusion::sql::parser::{DFParser, Statement}; +use datafusion::sql::sqlparser::ast::{ + Ident, ObjectName, Query, Statement as SqlStatement, TableFactor, Visit, Visitor, +}; +use infostash_model::{Error, Layer, Result, TableRef}; + +/// The tables a model's SQL reads, each with the first line that names it. +/// The SQL must be one query; names of its CTEs are not tables. +pub(crate) fn tables_read(file: &str, sql: &str) -> Result> { + let statements = DFParser::parse_sql(sql).map_err(|e| Error::Parse(format!("{file}: {e}")))?; + let query = match Vec::from(statements).as_slice() { + [Statement::Statement(statement)] => match statement.as_ref() { + SqlStatement::Query(query) => query.clone(), + _ => return Err(not_one_query(file)), + }, + _ => return Err(not_one_query(file)), + }; + let mut names = Names::default(); + let _ = query.visit(&mut names); + let mut tables = BTreeMap::new(); + for name in &names.tables { + if let Some((table, line)) = resolve(file, name, &names.ctes)? { + tables.entry(table).or_insert(line); + } + } + Ok(tables) +} + +fn not_one_query(file: &str) -> Error { + Error::Parse(format!("{file}: a model is one `select` statement")) +} + +/// The CTE names and table names in a query, collected in one walk. +#[derive(Default)] +struct Names { + ctes: HashSet, + tables: Vec, +} + +impl Visitor for Names { + type Break = (); + + fn pre_visit_query(&mut self, query: &Query) -> ControlFlow<()> { + if let Some(with) = &query.with { + self.ctes + .extend(with.cte_tables.iter().map(|cte| normalise(&cte.alias.name))); + } + ControlFlow::Continue(()) + } + + fn pre_visit_table_factor(&mut self, factor: &TableFactor) -> ControlFlow<()> { + // A table function such as `generate_series(1, 3)` reads no table. + if let TableFactor::Table { + name, args: None, .. + } = factor + { + self.tables.push(name.clone()); + } + ControlFlow::Continue(()) + } +} + +/// The table a name refers to and its line, or `None` for a CTE. +fn resolve( + file: &str, + name: &ObjectName, + ctes: &HashSet, +) -> Result> { + let parts: Vec<&Ident> = name.0.iter().filter_map(|part| part.as_ident()).collect(); + let line = parts.first().map_or(0, |ident| { + usize::try_from(ident.span.start.line).unwrap_or(0) + }); + let err = |message: String| Error::Parse(format!("{file}:{line}: {message}")); + if parts.len() != name.0.len() { + return Err(err(format!("`{name}` is not a table name"))); + } + match parts.as_slice() { + [table] if ctes.contains(&normalise(table)) => Ok(None), + [table] => { + let table = normalise(table); + Err(err(format!( + "`{table}` needs its layer, as in `raw.{table}`, `clean.{table}` or `ready.{table}`" + ))) + } + [layer, table] => match Layer::from_namespace(&normalise(layer)) { + Some(layer) => Ok(Some((TableRef::new(layer, normalise(table)), line))), + None => Err(err(format!( + "`{name}`: unknown layer `{}`; use raw, clean or ready", + normalise(layer) + ))), + }, + _ => Err(err(format!("`{name}`: name a table as `.`"))), + } +} + +/// An identifier as DataFusion resolves it: unquoted names are lower case. +fn normalise(ident: &Ident) -> String { + match ident.quote_style { + Some(_) => ident.value.clone(), + None => ident.value.to_lowercase(), + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn read(sql: &str) -> Vec<(String, usize)> { + tables_read("m.sql", sql) + .unwrap() + .into_iter() + .map(|(table, line)| (table.to_string(), line)) + .collect() + } + + fn read_err(sql: &str) -> String { + match tables_read("m.sql", sql) { + Err(Error::Parse(message)) => message, + other => panic!("expected a parse error, got {other:?}"), + } + } + + #[test] + fn finds_tables_in_joins_and_subqueries_with_their_first_line() { + let sql = "select *\nfrom raw.a\njoin (select * from clean.b) b on true\n\ + where exists (select 1 from raw.a)\nunion all select * from gold.c;"; + assert_eq!( + read(sql), + [ + ("bronze.a".to_string(), 2), + ("silver.b".to_string(), 3), + ("gold.c".to_string(), 5) + ] + ); + } + + #[test] + fn cte_names_are_not_tables() { + let sql = "with recent as (select * from raw.orders)\nselect * from recent"; + assert_eq!(read(sql), [("bronze.orders".to_string(), 1)]); + } + + #[test] + fn table_functions_read_no_table() { + assert!(read("select * from generate_series(1, 3)").is_empty()); + } + + #[test] + fn unquoted_names_are_lower_cased() { + assert_eq!( + read("select * from RAW.Orders"), + [("bronze.orders".to_string(), 1)] + ); + assert_eq!( + read("select * from raw.\"Orders\""), + [("bronze.Orders".to_string(), 1)] + ); + } + + #[test] + fn rejects_names_without_a_known_layer() { + assert_eq!( + read_err("select *\nfrom orders"), + "m.sql:2: `orders` needs its layer, as in `raw.orders`, `clean.orders` or `ready.orders`" + ); + assert_eq!( + read_err("select * from staging.orders"), + "m.sql:1: `staging.orders`: unknown layer `staging`; use raw, clean or ready" + ); + assert_eq!( + read_err("select * from cat.raw.orders"), + "m.sql:1: `cat.raw.orders`: name a table as `.
`" + ); + } + + #[test] + fn a_model_is_one_query() { + for sql in ["select 1; select 2", "create table x (a int)", ""] { + assert_eq!( + read_err(sql), + "m.sql: a model is one `select` statement", + "{sql}" + ); + } + assert!(read_err("select (").starts_with("m.sql: ")); + } +} diff --git a/crates/compiler/src/tests.rs b/crates/compiler/src/tests.rs new file mode 100644 index 0000000..325d2dc --- /dev/null +++ b/crates/compiler/src/tests.rs @@ -0,0 +1,304 @@ +//! Tests that load whole projects from temporary directories. + +use std::fs; +use std::path::Path; + +use infostash_model::{Error, Layer, TableRef}; +use tempfile::TempDir; + +use crate::{Check, Project}; + +const PROJECT: &str = + "[project]\nname = \"shop\"\nformat = 1\n\n[warehouse]\npath = \".infostash/warehouse\"\n"; +const ORDERS: &str = "connector = \"csv\"\npath = \"data/orders/*.csv\"\n"; + +/// A project directory holding `files`, each as (path, contents). +fn project(files: &[(&str, &str)]) -> TempDir { + let dir = TempDir::new().unwrap(); + for (path, text) in files { + let path = dir.path().join(path); + fs::create_dir_all(path.parent().unwrap()).unwrap(); + fs::write(path, text).unwrap(); + } + dir +} + +/// A shop project with the orders source and the given models. +fn shop(models: &[(&str, &str)]) -> TempDir { + let mut files = vec![("infostash.toml", PROJECT), ("sources/orders.toml", ORDERS)]; + files.extend_from_slice(models); + project(&files) +} + +fn parse_error(dir: &TempDir) -> String { + match Project::load(dir.path()) { + Err(Error::Parse(message)) => message, + other => panic!("expected a parse error, got {other:?}"), + } +} + +fn silver(name: &str) -> TableRef { + TableRef::new(Layer::Clean, name) +} + +fn gold(name: &str) -> TableRef { + TableRef::new(Layer::Ready, name) +} + +#[test] +fn loads_the_rfc_example() { + let dir = shop(&[ + ( + "clean/orders.sql", + "-- clean/orders.sql\n\ + -- @unique_key: order_id\n\ + -- @check not_negative: amount >= 0\n\ + -- @check: customer_id is not null\n\ + -- @description: One row per order, latest version wins.\n\ + select try_cast(amount as decimal(12, 2)) as amount from raw.orders\n", + ), + ( + "ready/daily_revenue.sql", + "-- @full_refresh: true\nselect sum(amount) as revenue from clean.orders\n", + ), + ]); + let project = Project::load(dir.path()).unwrap(); + assert_eq!(project.name(), "shop"); + assert_eq!(project.warehouse(), dir.path().join(".infostash/warehouse")); + let source = &project.sources()[0]; + assert_eq!(source.table, TableRef::new(Layer::Raw, "orders")); + assert_eq!(source.connector, "csv"); + assert_eq!(source.options["path"].as_str(), Some("data/orders/*.csv")); + assert!(!source.options.contains_key("connector")); + + let [orders, revenue] = project.models() else { + panic!("expected two models"); + }; + assert_eq!(orders.table, silver("orders")); + assert_eq!(orders.file, Path::new("clean/orders.sql")); + assert_eq!(orders.directives.unique_key, Some(vec!["order_id".into()])); + assert_eq!( + orders.directives.checks, + vec![ + Check { + name: Some("not_negative".into()), + expr: "amount >= 0".into(), + line: 3 + }, + Check { + name: None, + expr: "customer_id is not null".into(), + line: 4 + }, + ] + ); + assert_eq!( + orders.directives.description.as_deref(), + Some("One row per order, latest version wins.") + ); + assert!(!orders.directives.full_refresh); + assert_eq!(orders.inputs, [TableRef::new(Layer::Raw, "orders")].into()); + assert_eq!(revenue.table, gold("daily_revenue")); + assert!(revenue.directives.full_refresh); + assert_eq!(revenue.inputs, [silver("orders")].into()); +} + +#[test] +fn orders_models_after_what_they_read() { + // In name order `a` would run first, but it reads `b`, which reads `c`. + let dir = shop(&[ + ("ready/a.sql", "select * from gold.b"), + ("ready/b.sql", "select * from ready.c join clean.d on true"), + ("ready/c.sql", "select * from silver.d"), + ("clean/d.sql", "select * from bronze.orders"), + ("clean/e.sql", "select * from raw.orders"), + ]); + let project = Project::load(dir.path()).unwrap(); + let order: Vec = project + .models() + .iter() + .map(|m| m.table.to_string()) + .collect(); + assert_eq!( + order, + ["silver.d", "silver.e", "gold.c", "gold.b", "gold.a"] + ); +} + +#[test] +fn subdirectories_do_not_change_the_name() { + let dir = shop(&[("clean/sales/eu/orders.sql", "select * from raw.orders")]); + let project = Project::load(dir.path()).unwrap(); + assert_eq!(project.models()[0].table, silver("orders")); + assert_eq!( + project.models()[0].file, + Path::new("clean/sales/eu/orders.sql") + ); +} + +#[test] +fn ignores_other_files() { + let dir = shop(&[ + ("README.md", "# shop"), + ("clean/notes.txt", "not a model"), + ("sources/nested/extra.toml", "not = \"a source\""), + ]); + let project = Project::load(dir.path()).unwrap(); + assert_eq!(project.sources().len(), 1); + assert!(project.models().is_empty()); +} + +#[test] +fn a_missing_project_file_is_not_found() { + let dir = project(&[]); + assert!(matches!(Project::load(dir.path()), Err(Error::NotFound(_)))); +} + +#[test] +fn rejects_unknown_project_keys() { + let dir = project(&[("infostash.toml", &format!("{PROJECT}\n[extra]\nkey = 1\n"))]); + let message = parse_error(&dir); + assert!(message.starts_with("infostash.toml:8: "), "{message}"); + assert!(message.contains("extra"), "{message}"); +} + +#[test] +fn rejects_another_format() { + let dir = project(&[( + "infostash.toml", + &PROJECT.replace("format = 1", "format = 2"), + )]); + assert!(parse_error(&dir).contains("format 2 is not supported")); +} + +#[test] +fn a_source_needs_a_string_connector() { + let missing = shop(&[("sources/items.toml", "path = \"x.csv\"")]); + assert_eq!( + parse_error(&missing), + "sources/items.toml: `connector` is missing" + ); + let number = shop(&[("sources/items.toml", "connector = 1")]); + assert_eq!( + parse_error(&number), + "sources/items.toml: `connector` must be a string" + ); +} + +#[test] +fn rejects_bad_table_names() { + for name in ["Orders", "1orders", "orders__quarantine", "my-orders"] { + let dir = shop(&[(&format!("clean/{name}.sql"), "select 1")]); + let message = parse_error(&dir); + assert!( + message.starts_with(&format!("clean/{name}.sql: ")), + "{message}" + ); + } +} + +#[test] +fn rejects_two_files_with_one_name_in_a_layer() { + let dir = shop(&[ + ("clean/a/orders.sql", "select * from raw.orders"), + ("clean/b/orders.sql", "select * from raw.orders"), + ]); + assert_eq!( + parse_error(&dir), + "clean/a/orders.sql and clean/b/orders.sql both define `orders`" + ); +} + +#[test] +fn one_name_may_appear_in_each_layer() { + let dir = shop(&[ + ("clean/orders.sql", "select * from raw.orders"), + ("ready/orders.sql", "select * from clean.orders"), + ]); + assert_eq!(Project::load(dir.path()).unwrap().models().len(), 2); +} + +#[test] +fn rejects_a_read_of_an_undefined_table() { + let dir = shop(&[("clean/orders.sql", "-- note\nselect *\nfrom raw.items")]); + assert_eq!( + parse_error(&dir), + "clean/orders.sql:3: no source or model defines `bronze.items`" + ); +} + +#[test] +fn rejects_a_read_of_a_later_layer() { + let dir = shop(&[ + ("clean/orders.sql", "select * from ready.revenue"), + ("ready/revenue.sql", "select * from raw.orders"), + ]); + assert_eq!( + parse_error(&dir), + "clean/orders.sql:1: `silver.orders` can't read `gold.revenue`, which is in a later layer" + ); +} + +#[test] +fn ready_may_read_raw() { + let dir = shop(&[("ready/orders.sql", "select * from raw.orders")]); + assert!(Project::load(dir.path()).is_ok()); +} + +#[test] +fn names_a_cycle() { + let dir = shop(&[ + ("clean/a.sql", "select * from clean.b"), + ("clean/b.sql", "select * from\nclean.c"), + ("clean/c.sql", "select * from clean.b"), + ]); + assert_eq!( + parse_error(&dir), + "clean/b.sql:2: dependency cycle: silver.b -> silver.c -> silver.b" + ); +} + +#[test] +fn a_model_reading_itself_is_a_cycle() { + let dir = shop(&[("clean/a.sql", "select * from clean.a")]); + assert_eq!( + parse_error(&dir), + "clean/a.sql:1: dependency cycle: silver.a -> silver.a" + ); +} + +#[test] +fn rejects_a_bad_directive_with_its_line() { + let dir = shop(&[( + "clean/orders.sql", + "-- top\n-- @partition_by: day\nselect * from raw.orders", + )]); + assert_eq!( + parse_error(&dir), + "clean/orders.sql:2: unknown directive `@partition_by`" + ); +} + +#[test] +fn rejects_sql_that_is_not_one_query() { + let dir = shop(&[("clean/orders.sql", "select 1; select 2")]); + assert_eq!( + parse_error(&dir), + "clean/orders.sql: a model is one `select` statement" + ); +} + +#[test] +fn reports_a_project_file_that_cannot_be_read() { + let dir = project(&[("infostash.toml/x", "")]); + assert!(matches!(Project::load(dir.path()), Err(Error::Storage(_)))); +} + +#[test] +fn reports_a_model_directory_that_cannot_be_read() { + let dir = project(&[ + ("infostash.toml", PROJECT), + ("clean", "a file, not a directory"), + ]); + assert!(matches!(Project::load(dir.path()), Err(Error::Storage(_)))); +}