From ab8d8fe25430f81aa5382bc0eec3aa123383d694 Mon Sep 17 00:00:00 2001 From: Mayur Kale Date: Fri, 30 Oct 2026 17:07:46 +0000 Subject: [PATCH 1/6] feat(cli): infoctl query runs read-only SQL against the warehouse Tables are named by layer namespace or alias (gold.x or ready.x). Writes, DDL and SET are rejected. Output is a table, CSV or line-delimited JSON. An unknown table error lists the tables that exist. The warehouse comes from --warehouse, defaulting to .infostash/warehouse until project files (#5, #7) land. Closes #15 Signed-off-by: Mayur Kale --- Cargo.lock | 18 +++ Cargo.toml | 1 + crates/cli/Cargo.toml | 13 +++ crates/cli/src/lib.rs | 212 +++++++++++++++++++++++++++++++++++ crates/cli/src/main.rs | 23 +--- crates/cli/tests/query.rs | 140 +++++++++++++++++++++++ crates/cli/tests/usage.rs | 13 ++- crates/engine/Cargo.toml | 11 +- crates/engine/src/lib.rs | 79 +++++++++++++ crates/engine/tests/query.rs | 157 ++++++++++++++++++++++++++ crates/model/src/lib.rs | 16 ++- crates/storage/src/lib.rs | 22 +++- 12 files changed, 676 insertions(+), 29 deletions(-) create mode 100644 crates/cli/src/lib.rs create mode 100644 crates/cli/tests/query.rs create mode 100644 crates/engine/tests/query.rs diff --git a/Cargo.lock b/Cargo.lock index cae1913..ac79f65 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2583,6 +2583,18 @@ version = "0.1.0" [[package]] name = "infostash-cli" version = "0.1.0" +dependencies = [ + "arrow-array", + "arrow-csv", + "arrow-json", + "arrow-schema", + "datafusion", + "infostash-engine", + "infostash-model", + "infostash-storage", + "tempfile", + "tokio", +] [[package]] name = "infostash-compiler" @@ -2616,8 +2628,14 @@ dependencies = [ name = "infostash-engine" version = "0.1.0" dependencies = [ + "arrow-array", + "arrow-schema", "datafusion", "iceberg-datafusion", + "infostash-model", + "infostash-storage", + "tempfile", + "tokio", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index ccb7295..e7834fc 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -24,6 +24,7 @@ infostash-agent = { path = "crates/agent", version = "0.1.0" } # Arrow and parquet must match iceberg's. arrow-array = "59.2" arrow-csv = "59.2" +arrow-json = "59.2" arrow-schema = "59.2" datafusion = "55.2.0" iceberg = { git = "https://github.com/apache/iceberg-rust", rev = "5b82ab99aeaf50bf4f5b3b776ac9cd6380a5f3e3" } diff --git a/crates/cli/Cargo.toml b/crates/cli/Cargo.toml index 9b7eb0d..c29d198 100644 --- a/crates/cli/Cargo.toml +++ b/crates/cli/Cargo.toml @@ -7,6 +7,19 @@ license.workspace = true repository.workspace = true [dependencies] +arrow-csv.workspace = true +arrow-json.workspace = true +datafusion.workspace = true +infostash-engine.workspace = true +infostash-storage.workspace = true +tokio = { version = "1.54.1", features = ["rt-multi-thread"] } + +[dev-dependencies] +arrow-array.workspace = true +arrow-schema.workspace = true +infostash-model.workspace = true +tempfile = "3.27.0" +tokio = { version = "1.54.1", features = ["macros", "rt-multi-thread"] } [[bin]] name = "infoctl" diff --git a/crates/cli/src/lib.rs b/crates/cli/src/lib.rs new file mode 100644 index 0000000..dd4396c --- /dev/null +++ b/crates/cli/src/lib.rs @@ -0,0 +1,212 @@ +//! The commands behind `infoctl`. `main.rs` only passes in the arguments and +//! the output streams, so everything here can be tested in-process. + +use std::io::Write; +use std::path::PathBuf; + +use datafusion::arrow::util::pretty::pretty_format_batches_with_schema; +use infostash_engine::{QueryResult, query}; +use infostash_storage::Warehouse; + +pub const USAGE: &str = "\ +infoctl: medallion pipelines on open tables + +USAGE: + infoctl + +COMMANDS: + init create a new project + run build Raw, Clean and Ready tables (incremental) + query run SQL against the lake + +QUERY: + infoctl query [--warehouse ] [--format table|csv|json] \"\" + + Name tables ., as raw.orders or bronze.orders. Statements + that write are rejected. json prints one object per row. + --warehouse defaults to .infostash/warehouse. +"; + +/// Where `infoctl init` puts the warehouse (RFC 0001 draft). +const DEFAULT_WAREHOUSE: &str = ".infostash/warehouse"; + +/// Exit code for a command line infoctl can't parse. +const USAGE_ERROR: i32 = 2; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum Format { + Table, + Csv, + Json, +} + +#[derive(Debug, PartialEq, Eq)] +struct Query { + warehouse: PathBuf, + format: Format, + sql: String, +} + +/// Run `infoctl` with `args` (without the program name) and return the +/// exit code. +pub fn run(args: &[String], out: &mut dyn Write, err: &mut dyn Write) -> i32 { + let code = match args.first().map(String::as_str) { + Some(c @ ("init" | "run")) => { + let _ = writeln!(err, "infoctl {c}: not implemented yet"); + USAGE_ERROR + } + Some("query") => match parse_query(&args[1..]) { + Ok(q) => run_query(&q, out, err), + Err(msg) => { + let _ = writeln!(err, "infoctl query: {msg}\n\n{USAGE}"); + USAGE_ERROR + } + }, + _ => { + let _ = write!(out, "{USAGE}"); + 0 + } + }; + let _ = out.flush(); + code +} + +fn parse_query(args: &[String]) -> Result { + let mut warehouse = PathBuf::from(DEFAULT_WAREHOUSE); + let mut format = Format::Table; + let mut sql = None; + let mut args = args.iter(); + while let Some(arg) = args.next() { + match arg.as_str() { + "--warehouse" => { + warehouse = args.next().ok_or("--warehouse needs a directory")?.into(); + } + "--format" => { + format = match args.next().map(String::as_str) { + Some("table") => Format::Table, + Some("csv") => Format::Csv, + Some("json") => Format::Json, + _ => return Err("--format must be table, csv or json".into()), + }; + } + flag if flag.starts_with("--") => return Err(format!("unknown option {flag}")), + text if sql.is_none() => sql = Some(text.to_string()), + _ => return Err("pass the SQL as one quoted argument".into()), + } + } + let sql = sql.ok_or("no SQL given")?; + Ok(Query { + warehouse, + format, + sql, + }) +} + +fn run_query(q: &Query, out: &mut dyn Write, err: &mut dyn Write) -> i32 { + match execute(q).and_then(|result| print(&result, q.format, out)) { + Ok(()) => 0, + Err(e) => { + let _ = writeln!(err, "infoctl query: {e}"); + 1 + } + } +} + +fn execute(q: &Query) -> Result { + // Opening a missing directory would create a catalog there, which a + // read-only command must not do. + if !q.warehouse.is_dir() { + return Err(format!( + "no warehouse at {} (pass --warehouse)", + q.warehouse.display() + )); + } + let runtime = tokio::runtime::Runtime::new().map_err(|e| e.to_string())?; + runtime.block_on(async { + let warehouse = Warehouse::open_local(&q.warehouse) + .await + .map_err(|e| e.to_string())?; + query(&warehouse, &q.sql).await.map_err(|e| e.to_string()) + }) +} + +fn print(result: &QueryResult, format: Format, out: &mut dyn Write) -> Result<(), String> { + match format { + Format::Table => { + let table = pretty_format_batches_with_schema(result.schema.clone(), &result.batches) + .map_err(|e| e.to_string())?; + writeln!(out, "{table}").map_err(|e| e.to_string()) + } + Format::Csv => { + let mut writer = arrow_csv::WriterBuilder::new().with_header(true).build(out); + if result.batches.is_empty() { + let empty = datafusion::arrow::array::RecordBatch::new_empty(result.schema.clone()); + writer.write(&empty).map_err(|e| e.to_string())?; + } + for batch in &result.batches { + writer.write(batch).map_err(|e| e.to_string())?; + } + Ok(()) + } + Format::Json => { + let mut writer = arrow_json::WriterBuilder::new() + .with_explicit_nulls(true) + .build::<_, arrow_json::writer::LineDelimited>(out); + for batch in &result.batches { + writer.write(batch).map_err(|e| e.to_string())?; + } + writer.finish().map_err(|e| e.to_string()) + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn args(list: &[&str]) -> Vec { + list.iter().map(|s| s.to_string()).collect() + } + + #[test] + fn query_flags_parse() { + let q = parse_query(&args(&["--format", "csv", "--warehouse", "w", "select 1"])).unwrap(); + assert_eq!( + q, + Query { + warehouse: "w".into(), + format: Format::Csv, + sql: "select 1".into() + } + ); + let q = parse_query(&args(&["select 1"])).unwrap(); + assert_eq!(q.warehouse, PathBuf::from(DEFAULT_WAREHOUSE)); + assert_eq!(q.format, Format::Table); + assert_eq!( + parse_query(&args(&["--format", "json", "x"])) + .unwrap() + .format, + Format::Json + ); + assert_eq!( + parse_query(&args(&["--format", "table", "x"])) + .unwrap() + .format, + Format::Table + ); + } + + #[test] + fn bad_query_flags_are_usage_errors() { + for (bad, msg) in [ + (&[][..], "no SQL"), + (&["--warehouse"][..], "needs a directory"), + (&["--format", "xml", "x"][..], "table, csv or json"), + (&["--verbose", "x"][..], "unknown option --verbose"), + (&["select", "1"][..], "one quoted argument"), + ] { + let e = parse_query(&args(bad)).unwrap_err(); + assert!(e.contains(msg), "{bad:?}: {e}"); + } + } +} diff --git a/crates/cli/src/main.rs b/crates/cli/src/main.rs index 6293b7f..4ef6032 100644 --- a/crates/cli/src/main.rs +++ b/crates/cli/src/main.rs @@ -1,24 +1,7 @@ //! `infoctl`: the Infostash command line. -const USAGE: &str = "\ -infoctl: medallion pipelines on open tables - -USAGE: - infoctl - -COMMANDS: - init create a new project - run build Raw, Clean and Ready tables (incremental) - query run SQL against the lake -"; - fn main() { - let cmd = std::env::args().nth(1); - match cmd.as_deref() { - Some(c @ ("init" | "run" | "query")) => { - eprintln!("infoctl {c}: not implemented yet"); - std::process::exit(2); - } - _ => print!("{USAGE}"), - } + let args: Vec = std::env::args().skip(1).collect(); + let code = infostash_cli::run(&args, &mut std::io::stdout(), &mut std::io::stderr()); + std::process::exit(code); } diff --git a/crates/cli/tests/query.rs b/crates/cli/tests/query.rs new file mode 100644 index 0000000..b91028b --- /dev/null +++ b/crates/cli/tests/query.rs @@ -0,0 +1,140 @@ +//! `infoctl query` against a small committed table, run as a user would. + +#![allow(clippy::unwrap_used)] + +use std::process::{Command, Output}; +use std::sync::Arc; + +use arrow_array::{Int64Array, RecordBatch, StringArray}; +use arrow_schema::{DataType, Field, Schema}; +use infostash_model::{Layer, TableRef}; +use infostash_storage::Warehouse; + +/// A warehouse with `ready.daily_revenue` holding two days, one with no +/// region. +fn warehouse() -> tempfile::TempDir { + let dir = tempfile::tempdir().unwrap(); + let schema = Arc::new(Schema::new(vec![ + Field::new("day", DataType::Utf8, false), + Field::new("region", DataType::Utf8, true), + Field::new("amount", DataType::Int64, false), + ])); + let batch = RecordBatch::try_new( + schema.clone(), + vec![ + Arc::new(StringArray::from(vec!["2026-10-01", "2026-10-02"])), + Arc::new(StringArray::from(vec![Some("north"), None])), + Arc::new(Int64Array::from(vec![120, 80])), + ], + ) + .unwrap(); + let runtime = tokio::runtime::Runtime::new().unwrap(); + runtime.block_on(async { + let wh = Warehouse::open_local(dir.path()).await.unwrap(); + let table = TableRef::new(Layer::Ready, "daily_revenue"); + wh.create_table(&table, &schema).await.unwrap(); + wh.append(&table, &[batch]).await.unwrap(); + }); + dir +} + +fn infoctl(dir: &tempfile::TempDir, args: &[&str]) -> Output { + Command::new(env!("CARGO_BIN_EXE_infoctl")) + .arg("query") + .arg("--warehouse") + .arg(dir.path()) + .args(args) + .output() + .unwrap() +} + +fn stdout(out: &Output) -> String { + assert!( + out.status.success(), + "{}", + String::from_utf8_lossy(&out.stderr) + ); + String::from_utf8(out.stdout.clone()).unwrap() +} + +const SQL: &str = "select day, region, amount from ready.daily_revenue order by day"; + +#[test] +fn prints_a_table_by_default() { + let dir = warehouse(); + let text = stdout(&infoctl(&dir, &[SQL])); + assert!(text.contains("| day | region | amount |"), "{text}"); + assert!(text.contains("| 2026-10-01 | north | 120 |"), "{text}"); +} + +#[test] +fn prints_csv() { + let dir = warehouse(); + let text = stdout(&infoctl(&dir, &["--format", "csv", SQL])); + assert_eq!( + text, + "day,region,amount\n2026-10-01,north,120\n2026-10-02,,80\n" + ); +} + +#[test] +fn prints_one_json_object_per_row() { + let dir = warehouse(); + let text = stdout(&infoctl(&dir, &["--format", "json", SQL])); + assert_eq!( + text, + "{\"day\":\"2026-10-01\",\"region\":\"north\",\"amount\":120}\n\ + {\"day\":\"2026-10-02\",\"region\":null,\"amount\":80}\n" + ); +} + +#[test] +fn an_empty_result_still_shows_its_columns() { + let dir = warehouse(); + let none = "select day from gold.daily_revenue where amount < 0"; + assert!(stdout(&infoctl(&dir, &[none])).contains("| day |")); + assert_eq!(stdout(&infoctl(&dir, &["--format", "csv", none])), "day\n"); + assert_eq!(stdout(&infoctl(&dir, &["--format", "json", none])), ""); +} + +#[test] +fn a_write_is_rejected_and_changes_nothing() { + let dir = warehouse(); + let out = infoctl( + &dir, + &["insert into ready.daily_revenue values ('2026-10-03', 'x', 1)"], + ); + assert_eq!(out.status.code(), Some(1)); + let stderr = String::from_utf8(out.stderr).unwrap(); + assert!(stderr.contains("not supported"), "{stderr}"); + let count = "select count(*) as n from ready.daily_revenue"; + assert_eq!( + stdout(&infoctl(&dir, &["--format", "csv", count])), + "n\n2\n" + ); +} + +#[test] +fn an_unknown_table_lists_the_tables() { + let dir = warehouse(); + let out = infoctl(&dir, &["select * from ready.revenue"]); + assert_eq!(out.status.code(), Some(1)); + let stderr = String::from_utf8(out.stderr).unwrap(); + assert!(stderr.contains("tables: gold.daily_revenue"), "{stderr}"); +} + +#[test] +fn a_missing_warehouse_is_an_error_and_creates_nothing() { + let dir = tempfile::tempdir().unwrap(); + let missing = dir.path().join("nope"); + let out = Command::new(env!("CARGO_BIN_EXE_infoctl")) + .args(["query", "--warehouse"]) + .arg(&missing) + .arg("select 1") + .output() + .unwrap(); + assert_eq!(out.status.code(), Some(1)); + let stderr = String::from_utf8(out.stderr).unwrap(); + assert!(stderr.contains("no warehouse at"), "{stderr}"); + assert!(!missing.exists()); +} diff --git a/crates/cli/tests/usage.rs b/crates/cli/tests/usage.rs index 9e4d43b..2f71285 100644 --- a/crates/cli/tests/usage.rs +++ b/crates/cli/tests/usage.rs @@ -23,10 +23,21 @@ fn prints_usage_without_a_command() { #[test] fn unfinished_commands_exit_with_an_error() { - for cmd in ["init", "run", "query"] { + for cmd in ["init", "run"] { let out = infoctl(&[cmd]); assert_eq!(out.status.code(), Some(2)); let stderr = String::from_utf8(out.stderr).unwrap(); assert!(stderr.contains("not implemented yet"), "{stderr}"); } } + +#[test] +fn a_query_without_sql_is_a_usage_error() { + let out = infoctl(&["query"]); + assert_eq!(out.status.code(), Some(2)); + let stderr = String::from_utf8(out.stderr).unwrap(); + assert!( + stderr.contains("no SQL given") && stderr.contains("USAGE"), + "{stderr}" + ); +} diff --git a/crates/engine/Cargo.toml b/crates/engine/Cargo.toml index a9e2d8f..7806906 100644 --- a/crates/engine/Cargo.toml +++ b/crates/engine/Cargo.toml @@ -9,11 +9,14 @@ repository.workspace = true [dependencies] datafusion.workspace = true iceberg-datafusion.workspace = true +infostash-model.workspace = true +infostash-storage.workspace = true -# Not used until week 3, but kept so the build proves datafusion and iceberg -# agree on one arrow version. -[package.metadata.cargo-machete] -ignored = ["datafusion", "iceberg-datafusion"] +[dev-dependencies] +arrow-array.workspace = true +arrow-schema.workspace = true +tempfile = "3.27.0" +tokio = { version = "1.54.1", features = ["macros", "rt-multi-thread"] } [lints] workspace = true diff --git a/crates/engine/src/lib.rs b/crates/engine/src/lib.rs index 9c9eb08..a2111d4 100644 --- a/crates/engine/src/lib.rs +++ b/crates/engine/src/lib.rs @@ -1,2 +1,81 @@ //! Single-node vectorized execution on DataFusion and Arrow. Distributed //! plans in `infostash-dist` run these operators inside each task. + +use std::sync::Arc; + +use datafusion::arrow::array::RecordBatch; +use datafusion::arrow::datatypes::SchemaRef; +use datafusion::catalog::{MemorySchemaProvider, SchemaProvider}; +use datafusion::error::DataFusionError; +use datafusion::execution::context::{SQLOptions, SessionContext}; +use iceberg_datafusion::IcebergStaticTableProvider; +use infostash_model::{Error, Layer, Result}; +use infostash_storage::Warehouse; + +/// The rows a query returned, and their schema even when there are none. +pub struct QueryResult { + pub schema: SchemaRef, + pub batches: Vec, +} + +/// Run one read-only SQL statement against every table in the warehouse. +/// +/// Tables are named `.
`, where the layer is either the +/// namespace (`gold`) or its Raw, Clean and Ready alias (`ready`), as in +/// ADR 0003. Each table is read at the snapshot it had when the query +/// started. Statements that write or change the session are rejected. +pub async fn query(warehouse: &Warehouse, sql: &str) -> Result { + let ctx = SessionContext::new(); + let catalog = ctx + .catalog("datafusion") + .ok_or_else(|| Error::Plan("DataFusion has no default catalog".into()))?; + let tables = warehouse.tables().await?; + for layer in Layer::ALL { + let schema = Arc::new(MemorySchemaProvider::new()); + for table in tables.iter().filter(|t| t.layer == layer) { + let provider = + IcebergStaticTableProvider::try_new_from_table(warehouse.load(table).await?) + .await + .map_err(plan)?; + schema + .register_table(table.name.clone(), Arc::new(provider)) + .map_err(plan)?; + } + for name in [layer.namespace(), layer.alias()] { + catalog + .register_schema(name, schema.clone()) + .map_err(plan)?; + } + } + + let read_only = SQLOptions::new() + .with_allow_ddl(false) + .with_allow_dml(false) + .with_allow_statements(false); + let run = async { + let df = ctx.sql_with_options(sql, read_only).await?; + let schema = Arc::new(df.schema().as_arrow().clone()); + Ok::<_, DataFusionError>(QueryResult { + schema, + batches: df.collect().await?, + }) + }; + run.await.map_err(|e| match e.find_root() { + DataFusionError::Plan(m) if m.contains("not found") && m.starts_with("table") => { + let names: Vec<_> = tables.iter().map(ToString::to_string).collect(); + let known = if names.is_empty() { + "the warehouse has no tables".to_string() + } else { + format!("tables: {}", names.join(", ")) + }; + Error::NotFound(format!("{m}; {known}")) + } + DataFusionError::SQL(..) => Error::Parse(e.to_string()), + DataFusionError::Plan(..) => Error::Plan(e.to_string()), + _ => Error::Execution(e.to_string()), + }) +} + +fn plan(e: impl std::fmt::Display) -> Error { + Error::Plan(e.to_string()) +} diff --git a/crates/engine/tests/query.rs b/crates/engine/tests/query.rs new file mode 100644 index 0000000..7cf380d --- /dev/null +++ b/crates/engine/tests/query.rs @@ -0,0 +1,157 @@ +//! SQL over warehouse tables, through the same path `infoctl query` uses. + +#![allow(clippy::unwrap_used)] + +use std::sync::Arc; + +use arrow_array::{Int64Array, RecordBatch, StringArray}; +use arrow_schema::{DataType, Field, Schema}; +use datafusion::arrow::util::pretty::pretty_format_batches; +use infostash_engine::query; +use infostash_model::{Error, Layer, TableRef}; +use infostash_storage::Warehouse; + +fn revenue() -> RecordBatch { + let schema = Arc::new(Schema::new(vec![ + Field::new("day", DataType::Utf8, false), + Field::new("amount", DataType::Int64, false), + ])); + RecordBatch::try_new( + schema, + vec![ + Arc::new(StringArray::from(vec!["2026-10-01", "2026-10-02"])), + Arc::new(Int64Array::from(vec![120, 80])), + ], + ) + .unwrap() +} + +async fn warehouse(dir: &tempfile::TempDir) -> Warehouse { + let wh = Warehouse::open_local(dir.path()).await.unwrap(); + let table = TableRef::new(Layer::Ready, "daily_revenue"); + wh.create_table(&table, &revenue().schema()).await.unwrap(); + wh.append(&table, &[revenue()]).await.unwrap(); + wh +} + +fn text(batches: &[RecordBatch]) -> String { + pretty_format_batches(batches).unwrap().to_string() +} + +#[tokio::test] +async fn a_layer_reads_the_same_by_namespace_and_alias() { + let dir = tempfile::tempdir().unwrap(); + let wh = warehouse(&dir).await; + for name in ["gold.daily_revenue", "ready.daily_revenue"] { + let sql = format!("select day, amount from {name} order by day"); + let out = query(&wh, &sql).await.unwrap(); + let shown = text(&out.batches); + assert!( + shown.contains("2026-10-01") && shown.contains("120"), + "{shown}" + ); + assert_eq!(out.batches.iter().map(|b| b.num_rows()).sum::(), 2); + } +} + +#[tokio::test] +async fn sql_runs_over_the_table() { + let dir = tempfile::tempdir().unwrap(); + let wh = warehouse(&dir).await; + let out = query(&wh, "select sum(amount) as total from ready.daily_revenue") + .await + .unwrap(); + assert!(text(&out.batches).contains("200")); +} + +#[tokio::test] +async fn an_empty_result_keeps_its_schema() { + let dir = tempfile::tempdir().unwrap(); + let wh = warehouse(&dir).await; + let out = query( + &wh, + "select day from ready.daily_revenue where amount > 1000", + ) + .await + .unwrap(); + assert_eq!(out.schema.field(0).name(), "day"); + assert_eq!(out.batches.iter().map(|b| b.num_rows()).sum::(), 0); +} + +#[tokio::test] +async fn a_table_with_no_snapshot_reads_as_empty() { + let dir = tempfile::tempdir().unwrap(); + let wh = Warehouse::open_local(dir.path()).await.unwrap(); + let table = TableRef::new(Layer::Raw, "listings"); + wh.create_table(&table, &revenue().schema()).await.unwrap(); + let out = query(&wh, "select count(*) as n from raw.listings") + .await + .unwrap(); + assert!( + text(&out.batches).contains("| 0 "), + "{}", + text(&out.batches) + ); +} + +#[tokio::test] +async fn statements_that_write_are_rejected() { + let dir = tempfile::tempdir().unwrap(); + let wh = warehouse(&dir).await; + for sql in [ + "insert into ready.daily_revenue values ('2026-10-03', 5)", + "create table ready.copy as select * from ready.daily_revenue", + "drop table ready.daily_revenue", + "set datafusion.execution.batch_size = 1", + ] { + let err = query(&wh, sql).await.err().unwrap(); + assert!(matches!(err, Error::Plan(_)), "{sql}: {err}"); + assert!(err.to_string().contains("not supported"), "{sql}: {err}"); + } + let table = TableRef::new(Layer::Ready, "daily_revenue"); + assert_eq!(wh.read_all(&table).await.unwrap()[0].num_rows(), 2); + assert_eq!(wh.tables().await.unwrap(), vec![table]); +} + +#[tokio::test] +async fn an_unknown_table_lists_the_tables_that_exist() { + let dir = tempfile::tempdir().unwrap(); + let wh = warehouse(&dir).await; + let err = query(&wh, "select * from ready.missing") + .await + .err() + .unwrap(); + assert!(matches!(err, Error::NotFound(_)), "{err}"); + let msg = err.to_string(); + assert!( + msg.contains("missing") && msg.contains("gold.daily_revenue"), + "{msg}" + ); +} + +#[tokio::test] +async fn an_empty_warehouse_says_so() { + let dir = tempfile::tempdir().unwrap(); + let wh = Warehouse::open_local(dir.path()).await.unwrap(); + let err = query(&wh, "select * from raw.x").await.err().unwrap(); + assert!(err.to_string().contains("no tables"), "{err}"); +} + +#[tokio::test] +async fn bad_sql_is_a_parse_error() { + let dir = tempfile::tempdir().unwrap(); + let wh = warehouse(&dir).await; + let err = query(&wh, "select (1").await.err().unwrap(); + assert!(matches!(err, Error::Parse(_)), "{err}"); +} + +#[tokio::test] +async fn a_failing_expression_is_an_execution_error() { + let dir = tempfile::tempdir().unwrap(); + let wh = warehouse(&dir).await; + let err = query(&wh, "select cast(day as int) from ready.daily_revenue") + .await + .err() + .unwrap(); + assert!(matches!(err, Error::Execution(_)), "{err}"); +} diff --git a/crates/model/src/lib.rs b/crates/model/src/lib.rs index 120a4e1..b33f8f0 100644 --- a/crates/model/src/lib.rs +++ b/crates/model/src/lib.rs @@ -21,6 +21,9 @@ pub enum Layer { } impl Layer { + /// Every layer, in pipeline order. + pub const ALL: [Layer; 3] = [Layer::Raw, Layer::Clean, Layer::Ready]; + /// Namespace used in SQL and in the catalog (`bronze.orders`). pub fn namespace(self) -> &'static str { match self { @@ -30,6 +33,15 @@ impl Layer { } } + /// The analyst-facing name, also accepted in SQL (`ready.orders`). + pub fn alias(self) -> &'static str { + match self { + Layer::Raw => "raw", + Layer::Clean => "clean", + Layer::Ready => "ready", + } + } + pub fn from_namespace(ns: &str) -> Option { match ns { "bronze" | "raw" => Some(Layer::Raw), @@ -198,10 +210,10 @@ mod tests { #[test] fn every_layer_round_trips() { - for layer in [Layer::Raw, Layer::Clean, Layer::Ready] { + for layer in Layer::ALL { assert_eq!(Layer::from_namespace(layer.namespace()), Some(layer)); + assert_eq!(Layer::from_namespace(layer.alias()), Some(layer)); } - assert_eq!(Layer::from_namespace("ready"), Some(Layer::Ready)); } #[test] diff --git a/crates/storage/src/lib.rs b/crates/storage/src/lib.rs index 421cd33..2d8f8ac 100644 --- a/crates/storage/src/lib.rs +++ b/crates/storage/src/lib.rs @@ -35,7 +35,7 @@ use iceberg::{ use iceberg_catalog_sql::{ SQL_CATALOG_PROP_URI, SQL_CATALOG_PROP_WAREHOUSE, SqlCatalog, SqlCatalogBuilder, }; -use infostash_model::{Error, Result, SnapshotId, SnapshotRange, TableRef}; +use infostash_model::{Error, Layer, Result, SnapshotId, SnapshotRange, TableRef}; use parquet::file::properties::WriterProperties; /// The result of one atomic append. @@ -286,7 +286,25 @@ impl Warehouse { Ok(snaps.into_iter().map(|(_, id)| id).collect()) } - async fn load(&self, table: &TableRef) -> Result
{ + /// Every table in the Raw, Clean and Ready namespaces, sorted. + pub async fn tables(&self) -> Result> { + let mut tables = Vec::new(); + for layer in Layer::ALL { + let ns = NamespaceIdent::new(layer.namespace().to_string()); + if !self.catalog.namespace_exists(&ns).await.map_err(storage)? { + continue; + } + for t in self.catalog.list_tables(&ns).await.map_err(storage)? { + tables.push(TableRef::new(layer, t.name())); + } + } + tables.sort(); + Ok(tables) + } + + /// Load a table at its current snapshot, for engines that read it + /// directly. + pub async fn load(&self, table: &TableRef) -> Result
{ self.catalog .load_table(&ident(table)) .await From 10140381f7a5120c4620e4392c0076ef198815a5 Mon Sep 17 00:00:00 2001 From: Mayur Kale Date: Fri, 30 Oct 2026 17:10:38 +0000 Subject: [PATCH 2/6] fix(engine): reject COPY TO in tests and name unknown tables as typed Signed-off-by: Mayur Kale --- crates/engine/src/lib.rs | 5 ++++- crates/engine/tests/query.rs | 4 ++++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/crates/engine/src/lib.rs b/crates/engine/src/lib.rs index a2111d4..a2e7dee 100644 --- a/crates/engine/src/lib.rs +++ b/crates/engine/src/lib.rs @@ -68,7 +68,10 @@ pub async fn query(warehouse: &Warehouse, sql: &str) -> Result { } else { format!("tables: {}", names.join(", ")) }; - Error::NotFound(format!("{m}; {known}")) + // DataFusion names it 'datafusion.ready.x'; show what the user typed. + let name = m.split('\'').nth(1).unwrap_or(m); + let name = name.strip_prefix("datafusion.").unwrap_or(name); + Error::NotFound(format!("table {name}; {known}")) } DataFusionError::SQL(..) => Error::Parse(e.to_string()), DataFusionError::Plan(..) => Error::Plan(e.to_string()), diff --git a/crates/engine/tests/query.rs b/crates/engine/tests/query.rs index 7cf380d..0b422b4 100644 --- a/crates/engine/tests/query.rs +++ b/crates/engine/tests/query.rs @@ -98,7 +98,10 @@ async fn a_table_with_no_snapshot_reads_as_empty() { async fn statements_that_write_are_rejected() { let dir = tempfile::tempdir().unwrap(); let wh = warehouse(&dir).await; + let copied = dir.path().join("copied.csv"); + let copy = format!("copy (select 1) to '{}'", copied.display()); for sql in [ + copy.as_str(), "insert into ready.daily_revenue values ('2026-10-03', 5)", "create table ready.copy as select * from ready.daily_revenue", "drop table ready.daily_revenue", @@ -108,6 +111,7 @@ async fn statements_that_write_are_rejected() { assert!(matches!(err, Error::Plan(_)), "{sql}: {err}"); assert!(err.to_string().contains("not supported"), "{sql}: {err}"); } + assert!(!copied.exists()); let table = TableRef::new(Layer::Ready, "daily_revenue"); assert_eq!(wh.read_all(&table).await.unwrap()[0].num_rows(), 2); assert_eq!(wh.tables().await.unwrap(), vec![table]); From 7436e92c05475ef04ec960fe67de393ffe96b5ec Mon Sep 17 00:00:00 2001 From: Mayur Kale Date: Fri, 30 Oct 2026 17:10:57 +0000 Subject: [PATCH 3/6] test(engine): pin the unknown-table message Signed-off-by: Mayur Kale --- crates/engine/tests/query.rs | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/crates/engine/tests/query.rs b/crates/engine/tests/query.rs index 0b422b4..ae81ddc 100644 --- a/crates/engine/tests/query.rs +++ b/crates/engine/tests/query.rs @@ -126,10 +126,9 @@ async fn an_unknown_table_lists_the_tables_that_exist() { .err() .unwrap(); assert!(matches!(err, Error::NotFound(_)), "{err}"); - let msg = err.to_string(); - assert!( - msg.contains("missing") && msg.contains("gold.daily_revenue"), - "{msg}" + assert_eq!( + err.to_string(), + "not found: table ready.missing; tables: gold.daily_revenue" ); } From 6f1a64e4b07cfb6afe3ceb01f73e2bf4921768f5 Mon Sep 17 00:00:00 2001 From: Mayur Kale Date: Fri, 30 Oct 2026 17:43:23 +0000 Subject: [PATCH 4/6] fix(cli): fail when the output cannot be flushed A query whose final flush failed, as when the reader closed the pipe, exited 0 with incomplete output. The exit code is now 1 unless the command had already failed. Also adds a storage test that Warehouse::tables lists every layer, skips an absent one, and sorts. Signed-off-by: Mayur Kale --- crates/cli/src/lib.rs | 63 ++++++++++++++++++++--- crates/storage/tests/iceberg_roundtrip.rs | 25 +++++++++ 2 files changed, 81 insertions(+), 7 deletions(-) diff --git a/crates/cli/src/lib.rs b/crates/cli/src/lib.rs index dd4396c..139b54e 100644 --- a/crates/cli/src/lib.rs +++ b/crates/cli/src/lib.rs @@ -48,7 +48,7 @@ struct Query { } /// Run `infoctl` with `args` (without the program name) and return the -/// exit code. +/// exit code: 0 on success, 1 when a command fails, 2 for a usage error. pub fn run(args: &[String], out: &mut dyn Write, err: &mut dyn Write) -> i32 { let code = match args.first().map(String::as_str) { Some(c @ ("init" | "run")) => { @@ -62,13 +62,23 @@ pub fn run(args: &[String], out: &mut dyn Write, err: &mut dyn Write) -> i32 { USAGE_ERROR } }, - _ => { - let _ = write!(out, "{USAGE}"); - 0 - } + _ => match write!(out, "{USAGE}") { + Ok(()) => 0, + Err(e) => output_error(err, &e), + }, }; - let _ = out.flush(); - code + // stdout is buffered, so a reader that closed the pipe can show up here. + match out.flush() { + Err(e) if code == 0 => output_error(err, &e), + _ => code, + } +} + +/// Report a failed write to stdout. The output is incomplete, so the +/// command fails. +fn output_error(err: &mut dyn Write, e: &std::io::Error) -> i32 { + let _ = writeln!(err, "infoctl: output error: {e}"); + 1 } fn parse_query(args: &[String]) -> Result { @@ -209,4 +219,43 @@ mod tests { assert!(e.contains(msg), "{bad:?}: {e}"); } } + + /// Stdout after the reader closed the pipe. With `buffered`, writes + /// still succeed and only the flush fails. + struct ClosedPipe { + buffered: bool, + } + + impl Write for ClosedPipe { + fn write(&mut self, buf: &[u8]) -> std::io::Result { + if self.buffered { + Ok(buf.len()) + } else { + Err(std::io::ErrorKind::BrokenPipe.into()) + } + } + + fn flush(&mut self) -> std::io::Result<()> { + Err(std::io::ErrorKind::BrokenPipe.into()) + } + } + + #[test] + fn output_that_cannot_be_delivered_fails_the_command() { + for buffered in [true, false] { + let mut err = Vec::new(); + let code = run(&[], &mut ClosedPipe { buffered }, &mut err); + assert_eq!(code, 1, "buffered={buffered}"); + let err = String::from_utf8(err).unwrap(); + assert!(err.contains("output error"), "{err}"); + } + // A command that already failed keeps its own exit code. + let mut err = Vec::new(); + let code = run( + &args(&["query"]), + &mut ClosedPipe { buffered: true }, + &mut err, + ); + assert_eq!(code, USAGE_ERROR); + } } diff --git a/crates/storage/tests/iceberg_roundtrip.rs b/crates/storage/tests/iceberg_roundtrip.rs index f8c4c56..f353640 100644 --- a/crates/storage/tests/iceberg_roundtrip.rs +++ b/crates/storage/tests/iceberg_roundtrip.rs @@ -115,6 +115,31 @@ async fn tables_share_a_layer_and_names_are_unique() { assert!(wh.snapshots(&customers).await.unwrap().is_empty()); } +#[tokio::test] +async fn tables_lists_every_layer_in_order() { + let dir = tempfile::tempdir().unwrap(); + let wh = Warehouse::open_local(dir.path()).await.unwrap(); + // No namespace exists yet. + assert!(wh.tables().await.unwrap().is_empty()); + + // Created out of order, with no Clean table at all. + let zeta = TableRef::new(Layer::Ready, "zeta"); + let beta = TableRef::new(Layer::Raw, "beta"); + let alpha = TableRef::new(Layer::Raw, "alpha"); + for table in [&zeta, &beta, &alpha] { + wh.create_table(table, &schema()).await.unwrap(); + } + assert_eq!( + wh.tables().await.unwrap(), + [alpha.clone(), beta.clone(), zeta.clone()] + ); + + // Clean appears once it has a table, between Raw and Ready. + let mid = TableRef::new(Layer::Clean, "mid"); + wh.create_table(&mid, &schema()).await.unwrap(); + assert_eq!(wh.tables().await.unwrap(), [alpha, beta, mid, zeta]); +} + #[tokio::test] async fn missing_tables_and_directories_are_errors() { let dir = tempfile::tempdir().unwrap(); From b3017eb464c26f2774663591a3910bbbb404de71 Mon Sep 17 00:00:00 2001 From: Mayur Kale Date: Mon, 9 Nov 2026 11:57:41 +0000 Subject: [PATCH 5/6] fix(cli): refuse a warehouse directory without a catalog infoctl query opened any existing directory, and Warehouse::open_local creates catalog.db on open, so a read-only command turned an empty directory into a warehouse. The command now requires /catalog.db before opening and otherwise fails the way it does for a missing directory. Engine tests pin that CREATE EXTERNAL TABLE, EXPLAIN ANALYZE INSERT and a file path in FROM are rejected and write nothing. Closes #93 Signed-off-by: Mayur Kale --- crates/cli/src/lib.rs | 11 +++++--- crates/cli/tests/query.rs | 10 ++++++++ crates/engine/tests/query.rs | 49 ++++++++++++++++++++++++++++++++++++ 3 files changed, 67 insertions(+), 3 deletions(-) diff --git a/crates/cli/src/lib.rs b/crates/cli/src/lib.rs index 139b54e..da2f873 100644 --- a/crates/cli/src/lib.rs +++ b/crates/cli/src/lib.rs @@ -30,6 +30,11 @@ QUERY: /// Where `infoctl init` puts the warehouse (RFC 0001 draft). const DEFAULT_WAREHOUSE: &str = ".infostash/warehouse"; +/// The catalog file `Warehouse::open_local` keeps in the warehouse directory. +/// It repeats the private constant in `crates/storage/src/lib.rs` rather than +/// widening the storage API for one check. +const CATALOG_FILE: &str = "catalog.db"; + /// Exit code for a command line infoctl can't parse. const USAGE_ERROR: i32 = 2; @@ -123,9 +128,9 @@ fn run_query(q: &Query, out: &mut dyn Write, err: &mut dyn Write) -> i32 { } fn execute(q: &Query) -> Result { - // Opening a missing directory would create a catalog there, which a - // read-only command must not do. - if !q.warehouse.is_dir() { + // Opening a directory without a catalog (or a missing one) would create a + // catalog there, which a read-only command must not do. + if !q.warehouse.join(CATALOG_FILE).is_file() { return Err(format!( "no warehouse at {} (pass --warehouse)", q.warehouse.display() diff --git a/crates/cli/tests/query.rs b/crates/cli/tests/query.rs index b91028b..d33d44b 100644 --- a/crates/cli/tests/query.rs +++ b/crates/cli/tests/query.rs @@ -138,3 +138,13 @@ fn a_missing_warehouse_is_an_error_and_creates_nothing() { assert!(stderr.contains("no warehouse at"), "{stderr}"); assert!(!missing.exists()); } + +#[test] +fn a_directory_without_a_catalog_is_an_error_and_stays_empty() { + let dir = tempfile::tempdir().unwrap(); + let out = infoctl(&dir, &["select 1"]); + assert_eq!(out.status.code(), Some(1)); + let stderr = String::from_utf8(out.stderr).unwrap(); + assert!(stderr.contains("no warehouse at"), "{stderr}"); + assert_eq!(std::fs::read_dir(dir.path()).unwrap().count(), 0); +} diff --git a/crates/engine/tests/query.rs b/crates/engine/tests/query.rs index ae81ddc..50e363c 100644 --- a/crates/engine/tests/query.rs +++ b/crates/engine/tests/query.rs @@ -117,6 +117,55 @@ async fn statements_that_write_are_rejected() { assert_eq!(wh.tables().await.unwrap(), vec![table]); } +/// A small CSV next to the warehouse, as a query might try to read. +fn csv_file(dir: &tempfile::TempDir) -> std::path::PathBuf { + let path = dir.path().join("in.csv"); + std::fs::write(&path, "day,amount\n2026-10-03,5\n").unwrap(); + path +} + +#[tokio::test] +async fn an_external_table_cannot_be_declared() { + let dir = tempfile::tempdir().unwrap(); + let wh = warehouse(&dir).await; + let sql = format!( + "create external table t stored as csv location '{}'", + csv_file(&dir).display() + ); + let err = query(&wh, &sql).await.err().unwrap(); + assert!(matches!(err, Error::Plan(_)), "{err}"); + let shown = err.to_string(); + assert!( + shown.contains("not supported") && shown.contains("CreateExternalTable"), + "{shown}" + ); +} + +#[tokio::test] +async fn explain_analyze_does_not_run_a_write() { + let dir = tempfile::tempdir().unwrap(); + let wh = warehouse(&dir).await; + let err = query( + &wh, + "explain analyze insert into ready.daily_revenue values ('2026-10-03', 5)", + ) + .await + .err() + .unwrap(); + assert!(matches!(err, Error::Plan(_)), "{err}"); + assert!(err.to_string().contains("not supported"), "{err}"); + let table = TableRef::new(Layer::Ready, "daily_revenue"); + assert_eq!(wh.read_all(&table).await.unwrap()[0].num_rows(), 2); +} + +#[tokio::test] +async fn a_file_path_is_not_a_table() { + let dir = tempfile::tempdir().unwrap(); + let wh = warehouse(&dir).await; + let sql = format!("select * from '{}'", csv_file(&dir).display()); + assert!(query(&wh, &sql).await.is_err()); +} + #[tokio::test] async fn an_unknown_table_lists_the_tables_that_exist() { let dir = tempfile::tempdir().unwrap(); From 56b6217ccd36c04fb903e6535fd0e42560172b39 Mon Sep 17 00:00:00 2001 From: Mayur Kale Date: Mon, 9 Nov 2026 11:57:42 +0000 Subject: [PATCH 6/6] docs: add infoctl query to the changelog CONTRIBUTING asks for a line under Unreleased for anything users will notice, and a new command qualifies. Signed-off-by: Mayur Kale --- CHANGELOG.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5d039d4..6a9621d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,3 +16,7 @@ include breaking changes; the release notes say so. from its metadata file with no catalog. - The table catalog is kept in `catalog.db` (SQLite) in the warehouse directory, so tables and their snapshot history survive between runs. +- `infoctl query` runs one read-only SQL statement against the warehouse and + prints the result as a table, CSV or line-delimited JSON. Tables are named + `.
`, by namespace or alias; statements that write are rejected, + and a directory without a catalog is refused rather than initialised.