From ca8ea2c8bfb071bfca5b4d7958866f12e4140314 Mon Sep 17 00:00:00 2001 From: broisnees Date: Fri, 20 Nov 2026 16:19:26 +0000 Subject: [PATCH 1/6] Live tests for SQL Server, Oracle and ClickHouse; drop finished SQL Server sessions Transactions round-trip on real SQL Server and Oracle servers (rollback, commit, abandoned and released). ClickHouse streams 2M rows and Stop leaves nothing in system.processes. A transaction's SQL Server client now leaves the cancel registry when it closes instead of keeping its login there. --- src-tauri/src/db/mssql.rs | 7 +++ src-tauri/src/db/query.rs | 45 ++++++++++++++++ src-tauri/src/db/tx.rs | 110 +++++++++++++++++++++++++++++++++++++- 3 files changed, 161 insertions(+), 1 deletion(-) diff --git a/src-tauri/src/db/mssql.rs b/src-tauri/src/db/mssql.rs index 4c0293cf..d1ea3bc2 100644 --- a/src-tauri/src/db/mssql.rs +++ b/src-tauri/src/db/mssql.rs @@ -41,6 +41,13 @@ pub async fn remember(handle: &MssqlHandle, cfg: &MssqlConfig) { } } +/// Drop a handle's registry entry once the handle is done with (a +/// transaction's own client). The entry holds the login, and the address it +/// is keyed by can be reused by the next handle. +pub fn forget(handle: &MssqlHandle) { + SESSIONS.lock().unwrap_or_else(|e| e.into_inner()).remove(&key(handle)); +} + /// A new client on the same server and login, for a transaction: on the /// shared client every other query would have run inside it. pub async fn open_tx_client(handle: &MssqlHandle) -> Result { diff --git a/src-tauri/src/db/query.rs b/src-tauri/src/db/query.rs index fd399066..6f0d796a 100644 --- a/src-tauri/src/db/query.rs +++ b/src-tauri/src/db/query.rs @@ -6250,6 +6250,51 @@ mod stream_tests { check(&store, "ch-big", r, before, 120).await; } + // STROKE_TEST_CH=18123:password cargo test --lib stream_tests::clickhouse_live + fn live_clickhouse() -> Option { + let spec = std::env::var("STROKE_TEST_CH").ok()?; + let (port, pw) = spec.split_once(':')?; + let mut cfg = fake_clickhouse_cfg(port.parse().ok()?); + cfg.password = pw.into(); + Some(cfg) + } + + #[tokio::test] + async fn clickhouse_live_streams_and_stops() { + let Some(cfg) = live_clickhouse() else { return }; + let store = super::super::result_store::ResultStore::default(); + let mut sink = sink_into(&store, "ch-live"); + let before = rss_mb(); + let (_tx, rx) = tokio::sync::oneshot::channel(); + let stop = super::super::stop::Stop::from_rx(rx); + let sql = format!("SELECT number + 1 AS id, hex(MD5(toString(number))) AS v FROM numbers({N})"); + let r = super::super::clickhouse::stream_into(&cfg, &sql, &mut sink, &stop).await.unwrap(); + assert_eq!(r.columns[0].name, "id"); + assert_eq!(r.row_count, Some(N as i64)); + let stored = store.get("ch-live").unwrap(); + assert_eq!(stored.len(), N); + // UInt64 arrives quoted (output_format_json_quote_64bit_integers), as + // it did collected: values past 2^53 stay exact in the grid. + let deep: Vec = serde_json::from_str(&stored.read_window(N - 1, 1).unwrap()).unwrap(); + assert_eq!(deep[0][0], serde_json::json!(N.to_string())); + let grew = rss_mb().saturating_sub(before); + assert!(grew < 120, "memory grew {grew} MB while streaming"); + + // Stop: the server must not keep running the query. + let mut sink = sink_into(&store, "ch-stop"); + let (tx, rx) = tokio::sync::oneshot::channel(); + let stop = super::super::stop::Stop::from_rx(rx); + tokio::spawn(async move { tokio::time::sleep(std::time::Duration::from_millis(800)).await; let _ = tx.send(()); }); + let marker = "stroke_live_stop_marker"; + let long = format!("SELECT count() AS {marker} FROM numbers(100000000000) WHERE sipHash64(number) = 1"); + let t0 = std::time::Instant::now(); + let e = super::super::clickhouse::stream_into(&cfg, &long, &mut sink, &stop).await.unwrap_err(); + assert_eq!(e, QUERY_CANCELLED); + assert!(t0.elapsed() < std::time::Duration::from_secs(15)); + let left = super::super::clickhouse::query(&cfg, &format!("SELECT count() FROM system.processes WHERE query LIKE '%{marker}%' AND query NOT LIKE '%system.processes%'")).await.unwrap(); + assert_eq!(left.rows[0][0].as_str().map(|s| s.to_string()).unwrap_or(left.rows[0][0].to_string()), "0", "query still running on the server"); + } + #[tokio::test] async fn clickhouse_error_after_the_first_rows_is_reported() { let port = fake_clickhouse(500, "Code: 241. DB::Exception: Memory limit exceeded\n").await; diff --git a/src-tauri/src/db/tx.rs b/src-tauri/src/db/tx.rs index 2deced08..005ab560 100644 --- a/src-tauri/src/db/tx.rs +++ b/src-tauri/src/db/tx.rs @@ -77,7 +77,10 @@ impl TxConn { // Their own connection: dropping it closes it, and the server // rolls back what it held. TxConn::Duckdb(h) => drop(h), - TxConn::Mssql(h) => drop(h), + TxConn::Mssql(h) => { + super::mssql::forget(&h); + drop(h) + } TxConn::Oracle(h) => drop(h), } } @@ -686,6 +689,111 @@ async fn fetch_rows(conn: &mut TxConn, stmt: &str) -> Result mod tests { use super::leading_keyword; + fn live_session(conn: super::TxConn) -> super::TxSession { + super::TxSession { + conn, + epoch: 0, + blocker: None, + statements: 0, + rows_affected: 0, + started: std::time::Instant::now(), + last_used: std::time::Instant::now(), + } + } + + fn count_of(r: &super::SqlResult) -> i64 { + let v = &r.rows[0][0]; + v.as_i64().or_else(|| v.as_str().and_then(|s| s.parse().ok())).or_else(|| v.as_f64().map(|f| f as i64)).unwrap_or(-1) + } + + /// One engine's transaction, end to end through tx.rs: BEGIN, a write + /// that only the transaction sees, ROLLBACK; then BEGIN, write, COMMIT; + /// then a transaction abandoned mid-way, which must roll back on release. + /// `outside` reads through the shared handle. + async fn tx_round_trip(open: impl Fn() -> Fut, outside: F, table: &str) + where + Fut: std::future::Future, + F: Fn(String) -> std::pin::Pin>>, + { + use super::{execute_in, release, run_statement}; + let count = format!("SELECT COUNT(*) FROM {table}"); + let insert = format!("INSERT INTO {table} (v) VALUES (1)"); + + let mut c = open().await; + run_statement(&mut c, "BEGIN").await.unwrap(); + let w = execute_in(&mut c, &insert).await.unwrap(); + assert!(w.message.unwrap_or_default().contains("not committed")); + assert_eq!(count_of(&execute_in(&mut c, &count).await.unwrap()), 1, "the transaction sees its own row"); + run_statement(&mut c, "ROLLBACK").await.unwrap(); + assert_eq!(outside(count.clone()).await, 0, "rolled back"); + + run_statement(&mut c, "BEGIN").await.unwrap(); + execute_in(&mut c, &insert).await.unwrap(); + run_statement(&mut c, "COMMIT").await.unwrap(); + assert_eq!(outside(count.clone()).await, 1, "committed"); + + let mut c2 = open().await; + run_statement(&mut c2, "BEGIN").await.unwrap(); + execute_in(&mut c2, &insert).await.unwrap(); + release(live_session(c2)).await; + drop(c); + assert_eq!(outside(count.clone()).await, 1, "abandoned transaction rolled back"); + } + + // STROKE_TEST_MSSQL=127.0.0.1:11433:sa:password cargo test --lib tx::tests::mssql_live + #[tokio::test] + async fn mssql_live_transaction() { + let Ok(spec) = std::env::var("STROKE_TEST_MSSQL") else { return }; + let p: Vec<&str> = spec.splitn(4, ':').collect(); + let cfg = super::super::connection::MssqlConfig { + name: "live".into(), host: p[0].into(), port: p[1].parse().unwrap(), database: "master".into(), + user: p[2].into(), password: p[3].into(), encrypt: false, trust_cert: true, ssh: None, via: None, + }; + let shared = super::super::connection::open_mssql(&cfg).await.unwrap(); + super::super::mssql::batch(&shared, "IF OBJECT_ID('stroke_tx_live') IS NOT NULL DROP TABLE stroke_tx_live; CREATE TABLE stroke_tx_live (v int)").await.unwrap(); + let s2 = shared.clone(); + tx_round_trip( + || { let s = shared.clone(); async move { super::TxConn::Mssql(super::super::mssql::open_tx_client(&s).await.unwrap()) } }, + move |sql| { let s = s2.clone(); Box::pin(async move { count_of(&super::super::mssql::execute_sql(&s, &sql).await.unwrap()) }) }, + "stroke_tx_live", + ).await; + } + + // STROKE_TEST_ORACLE=app/pw@//127.0.0.1:11521/FREEPDB1 STROKE_ORACLE_CLIENT_DIR=... cargo test --lib tx::tests::oracle_live + #[tokio::test] + async fn oracle_live_transaction() { + let Ok(url) = std::env::var("STROKE_TEST_ORACLE") else { return }; + let (creds, cs) = url.split_once('@').unwrap(); + let (user, password) = creds.split_once('/').unwrap(); + let cfg = super::super::connection::OracleConfig { + ssh: None, name: "live".into(), host: String::new(), port: 1521, service_name: String::new(), sid: String::new(), + connect_string: cs.into(), user: user.into(), password: password.into(), + client_lib_dir: std::env::var("STROKE_ORACLE_CLIENT_DIR").unwrap_or_default(), + }; + super::super::oracle::prepare_client(&cfg.client_lib_dir).await.unwrap(); + let shared = super::super::oracle::connect(&cfg).await.unwrap(); + let _ = super::super::oracle::execute_sql(&shared, "DROP TABLE stroke_tx_live").await; + super::super::oracle::execute_sql(&shared, "CREATE TABLE stroke_tx_live (v NUMBER)").await.unwrap(); + let s2 = shared.clone(); + tx_round_trip( + || { let s = shared.clone(); async move { super::TxConn::Oracle(super::super::oracle::open_tx_session(&s).await.unwrap()) } }, + move |sql| { let s = s2.clone(); Box::pin(async move { count_of(&super::super::oracle::execute_sql(&s, &sql).await.unwrap()) }) }, + "stroke_tx_live", + ).await; + } + + #[tokio::test] + async fn duckdb_transaction_round_trip() { + let shared: super::super::connection::DuckdbHandle = std::sync::Arc::new(std::sync::Mutex::new(::duckdb::Connection::open_in_memory().unwrap())); + super::super::duckdb::batch(&shared, "CREATE TABLE stroke_tx_live (v INTEGER)").await.unwrap(); + let s2 = shared.clone(); + tx_round_trip( + || { let s = shared.clone(); async move { super::TxConn::Duckdb(super::super::duckdb::open_tx_conn(&s).await.unwrap()) } }, + move |sql| { let s = s2.clone(); Box::pin(async move { count_of(&super::super::duckdb::execute_sql(&s, &sql).await.unwrap()) }) }, + "stroke_tx_live", + ).await; + } + // Live: a released session must give back a connection that is out of its // transaction. Before `release`, the idle sweep dropped the PoolConnection, // sqlx returned it to the pool still inside the BEGIN, and the next user of From 3f72bfa32d61528d1e2fc0a31994f911fe459133 Mon Sep 17 00:00:00 2001 From: broisnees Date: Fri, 20 Nov 2026 16:34:27 +0000 Subject: [PATCH 2/6] Stream Oracle and MongoDB results Oracle queries feed the grid or result store as the session fetches them instead of collecting up to the budget; Stop breaks the call. MongoDB find and aggregate stream from the cursor past the console's 5,000-document cap, with fields the first 1,000 documents lacked kept in (other fields). Both verified on real servers. --- src-tauri/src/db/mongo.rs | 127 +++++++++++++++++++++++++++++++++++++ src-tauri/src/db/oracle.rs | 105 ++++++++++++++++++++++++++++++ src-tauri/src/db/query.rs | 79 +++++++++++++++++++++++ 3 files changed, 311 insertions(+) diff --git a/src-tauri/src/db/mongo.rs b/src-tauri/src/db/mongo.rs index a27fc656..16cda137 100644 --- a/src-tauri/src/db/mongo.rs +++ b/src-tauri/src/db/mongo.rs @@ -1433,6 +1433,133 @@ fn arg_doc(args: &[Value], i: usize, what: &str) -> Result { const CONSOLE_CAP: i64 = 5_000; +/// Fields the first batch did not have go here, as one object, so a later +/// document's extra keys are not lost from a streamed result. +pub const OTHER_FIELDS: &str = "(other fields)"; + +/// Whether `stream_into` takes this statement: a find or aggregate with no +/// limit under the console's cap (those answer in the reply as before). +pub(crate) fn streams(stmt: &str) -> bool { + let Ok(Stmt::Coll { calls, .. }) = parse(stmt) else { return false }; + let Some(head) = calls.first() else { return false }; + if head.name == "explain" || calls.iter().any(|c| c.name == "explain") { + return false; + } + if head.name != "find" && head.name != "aggregate" { + return false; + } + let limit = calls.iter().find(|c| c.name == "limit").and_then(|c| parse_args(&c.args).ok()).and_then(|a| a.first().and_then(|v| v.as_i64())); + limit.map_or(true, |l| l <= 0 || l > CONSOLE_CAP) +} + +/// A find or aggregate, streamed into `sink` from the cursor. +/// +/// The console collected at most 5,000 documents. Here the cursor's batches go +/// straight to the grid or the result store. Columns come from the first +/// 1,000 documents (`_id` first, as in the collected result); a later +/// document's keys outside them land in `(other fields)`. Stop drops the +/// cursor, which the driver kills on the server. +pub async fn stream_into( + m: &MongoConn, + stmt: &str, + sink: &mut super::query::RowSink, + stop: &super::stop::Stop, +) -> Result { + let run = stream_rows(m, stmt, sink); + tokio::pin!(run); + tokio::select! { + r = &mut run => r, + _ = stop.wait() => Err(super::query::QUERY_CANCELLED.into()), + } +} + +async fn stream_rows(m: &MongoConn, stmt: &str, sink: &mut super::query::RowSink) -> Result { + use futures::StreamExt; + const SAMPLE: usize = 1_000; + let t0 = std::time::Instant::now(); + let db_name = m.current_db.lock().map(|g| g.clone()).unwrap_or_default(); + let Stmt::Coll { coll, calls } = parse(stmt)? else { return Err("Not a find or aggregate".into()) }; + let c = m.client.database(&db_name).collection::(&coll); + let (head, mods) = calls.split_first().ok_or("Missing method")?; + let args = parse_args(&head.args)?; + let (mut sort, mut limit, mut skip) = (None, None, None); + for md in mods { + let a = parse_args(&md.args)?; + match md.name.as_str() { + "sort" => sort = Some(arg_doc(&a, 0, "sort")?), + "limit" => limit = a.first().and_then(|v| v.as_i64()).filter(|l| *l > 0), + "skip" => skip = a.first().and_then(|v| v.as_u64()), + "pretty" | "toArray" => {} + other => return Err(format!(".{other}() isn't supported here")), + } + } + let mut cursor = match head.name.as_str() { + "find" => { + let mut q = c.find(arg_doc(&args, 0, "filter")?).batch_size(1_000); + if let Some(p) = args.get(1).filter(|v| v.is_object()) { + q = q.projection(to_doc(p, None)?); + } + if let Some(s) = sort { + q = q.sort(s); + } + if let Some(k) = skip { + q = q.skip(k); + } + if let Some(l) = limit { + q = q.limit(l); + } + q.await.map_err(err)? + } + _ => { + let pipeline = match args.first() { + Some(Value::Array(stages)) => stages.iter().map(|s| to_doc(s, None)).collect::, _>>()?, + _ => return Err("aggregate() takes an array of stages".into()), + }; + c.aggregate(pipeline).batch_size(1_000).await.map_err(err)? + } + }; + + let mut sample: Vec = Vec::with_capacity(SAMPLE); + while sample.len() < SAMPLE { + match cursor.next().await { + Some(d) => sample.push(doc_to_json(&d.map_err(err)?)), + None => break, + } + } + let exhausted = sample.len() < SAMPLE; + if exhausted { + // Small: the collected shape, exactly as before. + return Ok(super::couchdb::docs_result(sample, None, stmt, t0)); + } + let shape = super::couchdb::docs_result(sample.clone(), None, stmt, t0); + let mut columns = shape.columns; + let names: Vec = columns.iter().map(|c| c.name.clone()).collect(); + let known: std::collections::HashSet = names.iter().cloned().collect(); + columns.push(super::query::ColumnInfo::new(OTHER_FIELDS, "object")); + sink.start(columns.clone())?; + let row_of = |d: &Value| -> Vec { + let mut row: Vec = names.iter().map(|n| super::couchdb::grid_cell(d.get(n).unwrap_or(&Value::Null)).0).collect(); + let extra: serde_json::Map = d + .as_object() + .map(|o| o.iter().filter(|(k, _)| !known.contains(*k)).map(|(k, v)| (k.clone(), v.clone())).collect()) + .unwrap_or_default(); + row.push(if extra.is_empty() { Value::Null } else { Value::Object(extra) }); + row + }; + let mut total = 0usize; + for d in &sample { + sink.push(row_of(d))?; + total += 1; + } + drop(sample); + while let Some(d) = cursor.next().await { + sink.push(row_of(&doc_to_json(&d.map_err(err)?)))?; + total += 1; + } + sink.finish()?; + Ok(super::query::SqlResult { columns, rows: vec![], row_count: Some(total as i64), message: None, query_ms: t0.elapsed().as_millis() as u64, sql: stmt.to_string() }) +} + pub async fn query(m: &MongoConn, stmt: &str) -> Result { let t0 = std::time::Instant::now(); let db_name = m.current_db.lock().map(|g| g.clone()).unwrap_or_default(); diff --git a/src-tauri/src/db/oracle.rs b/src-tauri/src/db/oracle.rs index ee04d246..3810b9e1 100644 --- a/src-tauri/src/db/oracle.rs +++ b/src-tauri/src/db/oracle.rs @@ -931,6 +931,111 @@ fn run_one(conn: &Connection, sql: &str, binds: &[Bind]) -> Result bool { + reads_only(sql) && !split_statements(sql).is_empty() && split_statements(sql).len() == 1 +} + +/// A query, streamed into `sink` a batch at a time while Oracle fetches it. +/// +/// `run_one` collected every row before the grid saw one, so a large result +/// stopped at the collect budget. Here the session's thread fetches 500 rows +/// per round trip and hands them over in batches of 1,000, so memory stays +/// flat whatever the size. Stop breaks the call on the server. +pub async fn stream_into( + handle: &OracleHandle, + sql: &str, + sink: &mut super::query::RowSink, + stop: &super::stop::Stop, +) -> Result { + enum Msg { + Columns(Vec), + Rows(Vec>), + } + const BATCH: usize = 1_000; + let stmt_sql = split_statements(sql).into_iter().next().unwrap_or_default(); + let (tx, mut rx) = tokio::sync::mpsc::channel::(8); + let reader = run_blocking(handle, move |conn| { + let mut stmt = conn.statement(&stmt_sql).fetch_array_size(500).build().map_err(ora_err)?; + let rs = stmt.query(&[]).map_err(ora_err)?; + let infos = rs.column_info().to_vec(); + let columns: Vec = infos + .iter() + .map(|c| { + let mut ci = ColumnInfo::new(c.name().to_string(), type_label(c.oracle_type())); + ci.nullable = c.nullable(); + ci + }) + .collect(); + if tx.blocking_send(Msg::Columns(columns)).is_err() { + return Ok(()); + } + let mut batch = Vec::with_capacity(BATCH); + for row in rs { + let row = row.map_err(ora_err)?; + batch.push(row.sql_values().iter().zip(infos.iter()).map(|(v, c)| cell_to_json(v, c.oracle_type())).collect()); + if batch.len() >= BATCH && tx.blocking_send(Msg::Rows(std::mem::replace(&mut batch, Vec::with_capacity(BATCH)))).is_err() { + return Ok(()); + } + } + if !batch.is_empty() { + let _ = tx.blocking_send(Msg::Rows(batch)); + } + Ok(()) + }); + tokio::pin!(reader); + let t0 = Instant::now(); + let mut columns = Vec::new(); + let mut total = 0usize; + let mut sink_err: Option = None; + let mut reader_out: Option> = None; + let mut stopped = false; + loop { + tokio::select! { + msg = rx.recv() => match msg { + Some(Msg::Columns(c)) => columns = c, + Some(Msg::Rows(rows)) => { + if sink_err.is_some() || stopped { + continue; + } + for r in rows { + if total == 0 { + if let Err(e) = sink.start(columns.clone()) { sink_err = Some(e); break; } + } + if let Err(e) = sink.push(r) { sink_err = Some(e); break; } + total += 1; + } + if sink_err.is_some() { + rx.close(); + } + } + None => break, + }, + out = &mut reader, if reader_out.is_none() => reader_out = Some(out), + _ = stop.wait(), if !stopped => { + stopped = true; + rx.close(); + cancel(handle).await; + } + } + } + let out = match reader_out { + Some(o) => o, + None => reader.await, + }; + if stopped { + return Err(super::query::QUERY_CANCELLED.into()); + } + if let Some(e) = sink_err { + return Err(e); + } + out?; + if total > 0 { + sink.finish()?; + } + Ok(SqlResult { columns, rows: vec![], row_count: Some(total as i64), message: None, query_ms: t0.elapsed().as_millis() as u64, sql: sql.to_string() }) +} + /// `run_one`, committed when it wrote (a query has nothing to commit). fn run_commit(conn: &Connection, sql: &str, binds: &[Bind]) -> Result { let r = run_one(conn, sql, binds); diff --git a/src-tauri/src/db/query.rs b/src-tauri/src/db/query.rs index 6f0d796a..45bce3e3 100644 --- a/src-tauri/src/db/query.rs +++ b/src-tauri/src/db/query.rs @@ -4671,6 +4671,8 @@ async fn stream_one( } } } + ActiveConnection::Doc(super::docdb::DocConn::Mongo(m)) if super::mongo::streams(stmt) => super::mongo::stream_into(m, stmt, sink, stop).await, + ActiveConnection::Oracle(h) if super::oracle::reads(stmt) => super::oracle::stream_into(h, stmt, sink, stop).await, ActiveConnection::Clickhouse(cfg) if super::clickhouse::reads(stmt) => super::clickhouse::stream_into(cfg, stmt, sink, stop).await, _ => return None, }) @@ -6295,6 +6297,83 @@ mod stream_tests { assert_eq!(left.rows[0][0].as_str().map(|s| s.to_string()).unwrap_or(left.rows[0][0].to_string()), "0", "query still running on the server"); } + // STROKE_TEST_ORACLE=user/pw@//host:1521/FREEPDB1 STROKE_ORACLE_CLIENT_DIR=... cargo test --lib stream_tests::oracle_live + #[tokio::test] + async fn oracle_live_streams_and_stops() { + let Ok(url) = std::env::var("STROKE_TEST_ORACLE") else { return }; + let (creds, cs) = url.split_once('@').unwrap(); + let (user, password) = creds.split_once('/').unwrap(); + let cfg = super::super::connection::OracleConfig { + ssh: None, name: "live".into(), host: String::new(), port: 1521, service_name: String::new(), sid: String::new(), + connect_string: cs.into(), user: user.into(), password: password.into(), + client_lib_dir: std::env::var("STROKE_ORACLE_CLIENT_DIR").unwrap_or_default(), + }; + super::super::oracle::prepare_client(&cfg.client_lib_dir).await.unwrap(); + let h = super::super::oracle::connect(&cfg).await.unwrap(); + let store = super::super::result_store::ResultStore::default(); + let mut sink = sink_into(&store, "ora-big"); + let before = rss_mb(); + let (_tx, rx) = tokio::sync::oneshot::channel(); + let stop = super::super::stop::Stop::from_rx(rx); + let sql = format!("SELECT a.n * 1000 + b.n + 1 AS id, RPAD('v', 64, 'x') AS v FROM (SELECT level - 1 AS n FROM dual CONNECT BY level <= {}) a CROSS JOIN (SELECT level - 1 AS n FROM dual CONNECT BY level <= 1000) b ORDER BY 1", N / 1000); + let r = super::super::oracle::stream_into(&h, &sql, &mut sink, &stop).await.unwrap(); + assert_eq!(r.columns[0].name, "ID"); + assert_eq!(r.row_count, Some(N as i64)); + let stored = store.get("ora-big").unwrap(); + assert_eq!(stored.len(), N); + let grew = rss_mb().saturating_sub(before); + assert!(grew < 120, "memory grew {grew} MB while streaming"); + + // Stop mid-stream: the call breaks and the session stays usable. + let mut sink = sink_into(&store, "ora-stop"); + let (tx, rx) = tokio::sync::oneshot::channel(); + let stop = super::super::stop::Stop::from_rx(rx); + tokio::spawn(async move { tokio::time::sleep(std::time::Duration::from_millis(700)).await; let _ = tx.send(()); }); + let slow = "SELECT COUNT(*) FROM (SELECT level n FROM dual CONNECT BY level <= 3000) a, (SELECT level n FROM dual CONNECT BY level <= 3000) b, (SELECT level n FROM dual CONNECT BY level <= 3000) c"; + let t0 = std::time::Instant::now(); + let e = super::super::oracle::stream_into(&h, slow, &mut sink, &stop).await.unwrap_err(); + assert_eq!(e, QUERY_CANCELLED); + assert!(t0.elapsed() < std::time::Duration::from_secs(10)); + super::super::oracle::ping(&h).await.unwrap(); + } + + // STROKE_TEST_MONGO=mongodb://127.0.0.1:27018 cargo test --lib stream_tests::mongo_live + #[tokio::test] + async fn mongo_live_streams_past_the_console_cap() { + let Ok(uri) = std::env::var("STROKE_TEST_MONGO") else { return }; + use mongodb::bson::doc; + const M: usize = 400_000; + let m = super::super::mongo::connect(&super::super::mongo::MongoConfig { ssh: None, name: "t".into(), uri, database: Some("stroke_stream".into()) }).await.unwrap(); + let c = m.client.database("stroke_stream").collection::("big"); + let _ = c.drop().await; + let mut batch = Vec::with_capacity(10_000); + for i in 0..M { + let mut d = doc! { "_id": i as i64 + 1, "n": i as i64, "v": format!("{:064x}", i) }; + // A field the first 1,000 documents never had. + if i == 250_000 { d.insert("late", "here"); } + batch.push(d); + if batch.len() == 10_000 { c.insert_many(std::mem::take(&mut batch)).await.unwrap(); } + } + let store = super::super::result_store::ResultStore::default(); + let mut sink = sink_into(&store, "mongo-big"); + let before = rss_mb(); + let (_tx, rx) = tokio::sync::oneshot::channel(); + let stop = super::super::stop::Stop::from_rx(rx); + let stmt = "db.big.find({}).sort({_id: 1})"; + assert!(super::super::mongo::streams(stmt)); + assert!(!super::super::mongo::streams("db.big.find({}).limit(10)")); + let r = super::super::mongo::stream_into(&m, stmt, &mut sink, &stop).await.unwrap(); + assert_eq!(r.row_count, Some(M as i64), "past the 5,000-document cap"); + assert_eq!(r.columns.last().unwrap().name, super::super::mongo::OTHER_FIELDS); + let stored = store.get("mongo-big").unwrap(); + assert_eq!(stored.len(), M); + let late: Vec> = serde_json::from_str(&stored.read_window(250_000, 1).unwrap()).unwrap(); + assert_eq!(late[0].last().unwrap(), &serde_json::json!({ "late": "here" })); + let grew = rss_mb().saturating_sub(before); + assert!(grew < 120, "memory grew {grew} MB while streaming"); + let _ = c.drop().await; + } + #[tokio::test] async fn clickhouse_error_after_the_first_rows_is_reported() { let port = fake_clickhouse(500, "Code: 241. DB::Exception: Memory limit exceeded\n").await; From bbc3ffc1af92c55d658df00f1800312b3ce92399 Mon Sep 17 00:00:00 2001 From: broisnees Date: Fri, 20 Nov 2026 16:44:18 +0000 Subject: [PATCH 3/6] Stop on the server for Elasticsearch, OpenSearch, PostHog and MongoDB Elasticsearch and OpenSearch requests carry an X-Opaque-Id and Stop cancels the tasks holding it. PostHog queries carry a client_query_id and Stop sends DELETE for it. A streamed MongoDB find or aggregate is tagged with a comment and Stop runs killOp on it. Elasticsearch and MongoDB verified on real servers; PostHog against a fake that checks the cancel call. --- src-tauri/src/db/elastic.rs | 84 ++++++++++++++++++++++++++++++++++++ src-tauri/src/db/mongo.rs | 40 ++++++++++++++--- src-tauri/src/db/posthog.rs | 85 ++++++++++++++++++++++++++++++++++++- src-tauri/src/db/query.rs | 21 +++++++++ 4 files changed, 224 insertions(+), 6 deletions(-) diff --git a/src-tauri/src/db/elastic.rs b/src-tauri/src/db/elastic.rs index 47c5296c..d8360662 100644 --- a/src-tauri/src/db/elastic.rs +++ b/src-tauri/src/db/elastic.rs @@ -232,6 +232,48 @@ enum Body<'a> { Ndjson(String), } +tokio::task_local! { + /// The console run's id, sent as X-Opaque-Id so Stop can find its tasks. + static OPAQUE_ID: String; +} + +/// `query`, stoppable on the server. Every request the statement makes +/// carries an X-Opaque-Id; Stop drops the request and cancels the tasks that +/// carry that id. Searches already end when their connection closes, but +/// `_sql`, `_delete_by_query`, `_update_by_query` and the like run on. +pub async fn query_cancellable(cfg: &EsConfig, stmt: &str, stop: &super::stop::Stop) -> Result { + static SEQ: std::sync::atomic::AtomicU64 = std::sync::atomic::AtomicU64::new(0); + let id = format!("stroke-{:x}-{:x}", std::process::id(), SEQ.fetch_add(1, std::sync::atomic::Ordering::Relaxed)); + let run = OPAQUE_ID.scope(id.clone(), query(cfg, stmt)); + tokio::pin!(run); + tokio::select! { + r = &mut run => r, + _ = stop.wait() => { + drop(run); + let _ = tokio::time::timeout(Duration::from_secs(10), cancel_tasks(cfg, &id)).await; + Err(super::query::QUERY_CANCELLED.into()) + } + } +} + +/// Cancel every task whose X-Opaque-Id is `id`. Returns how many. +pub async fn cancel_tasks(cfg: &EsConfig, id: &str) -> Result { + let v = send(cfg, "GET", "_tasks?detailed=true&group_by=none", None).await?; + let ids: Vec = v["tasks"] + .as_array() + .map(|t| { + t.iter() + .filter(|t| t["headers"]["X-Opaque-Id"].as_str() == Some(id) && t["cancellable"].as_bool() != Some(false)) + .filter_map(|t| Some(format!("{}:{}", t["node"].as_str()?, t["id"].as_i64()?))) + .collect() + }) + .unwrap_or_default(); + for t in &ids { + let _ = send(cfg, "POST", &format!("_tasks/{t}/_cancel"), None).await; + } + Ok(ids.len()) +} + async fn send(cfg: &EsConfig, method: &str, path: &str, body: Option>) -> Result { let url = base(cfg)?.join(path.trim_start_matches('/')).map_err(|e| format!("Invalid path \"{path}\": {e}"))?; let m = reqwest::Method::from_bytes(method.as_bytes()).map_err(|e| e.to_string())?; @@ -244,6 +286,9 @@ async fn send(cfg: &EsConfig, method: &str, path: &str, body: Option>) } else if let Some(u) = cfg.user.as_deref().filter(|u| !u.is_empty()) { req = req.basic_auth(u, cfg.password.as_deref()); } + if let Ok(id) = OPAQUE_ID.try_with(Clone::clone) { + req = req.header("X-Opaque-Id", id); + } match body { Some(Body::Json(b)) => req = req.json(b), Some(Body::Ndjson(s)) => req = req.header("Content-Type", "application/x-ndjson").body(s), @@ -862,6 +907,45 @@ mod tests { assert_eq!(urlencoding::encode("a/b"), "a%2Fb"); } + /// Stop on a slow search: while it runs the server lists a task with our + /// X-Opaque-Id, and after Stop none is left. + // STROKE_TEST_ES=http://127.0.0.1:19200 cargo test --lib elastic::tests::live_stop + #[tokio::test] + async fn live_stop_cancels_the_task() { + let Ok(url) = std::env::var("STROKE_TEST_ES") else { return }; + let cfg = EsConfig { ssh: None, via: None, name: "t".into(), url, user: None, password: None, api_key: None, flavor: None, insecure: true }; + let cfg = connect(cfg).await.unwrap(); + let _ = send(&cfg, "DELETE", "stroke_slow", None).await; + let mut nd = String::new(); + for i in 0..20_000 { + nd.push_str(&format!("{{\"index\":{{\"_index\":\"stroke_slow\"}}}}\n{{\"n\":{i}}}\n")); + } + send(&cfg, "POST", "_bulk?refresh=true", Some(Body::Ndjson(nd))).await.unwrap(); + let ours = |v: &Value| -> usize { + v["tasks"].as_array().map(|t| t.iter().filter(|t| t["headers"]["X-Opaque-Id"].as_str().is_some_and(|h| h.starts_with("stroke-"))).count()).unwrap_or(0) + }; + let (tx, rx) = tokio::sync::oneshot::channel(); + let stop = super::super::stop::Stop::from_rx(rx); + let probe = cfg.clone(); + let during = tokio::spawn(async move { + tokio::time::sleep(Duration::from_millis(1500)).await; + let v = send(&probe, "GET", "_tasks?detailed=true&group_by=none", None).await.unwrap(); + let n = ours(&v); + let _ = tx.send(()); + n + }); + let slow = "GET stroke_slow/_search\n{\"query\": {\"script\": {\"script\": \"long x = 0; for (int i = 0; i < 900000; i++) { x += i; } return x > 0;\"}}}"; + let t0 = Instant::now(); + let e = query_cancellable(&cfg, slow, &stop).await.unwrap_err(); + assert_eq!(e, super::super::query::QUERY_CANCELLED); + assert!(during.await.unwrap() > 0, "the search carried our X-Opaque-Id"); + assert!(t0.elapsed() < Duration::from_secs(20)); + tokio::time::sleep(Duration::from_millis(1500)).await; + let v = send(&cfg, "GET", "_tasks?detailed=true&group_by=none", None).await.unwrap(); + assert_eq!(ours(&v), 0, "a task is still running after Stop"); + let _ = send(&cfg, "DELETE", "stroke_slow", None).await; + } + /// Against a local server: `cargo test --lib es_live -- --ignored`, with /// ES_URL (default http://localhost:9200). #[tokio::test] diff --git a/src-tauri/src/db/mongo.rs b/src-tauri/src/db/mongo.rs index 16cda137..06538eed 100644 --- a/src-tauri/src/db/mongo.rs +++ b/src-tauri/src/db/mongo.rs @@ -1465,15 +1465,45 @@ pub async fn stream_into( sink: &mut super::query::RowSink, stop: &super::stop::Stop, ) -> Result { - let run = stream_rows(m, stmt, sink); + let tag = op_tag(); + let run = stream_rows(m, stmt, sink, &tag); tokio::pin!(run); tokio::select! { r = &mut run => r, - _ = stop.wait() => Err(super::query::QUERY_CANCELLED.into()), + _ = stop.wait() => { + drop(run); + let _ = tokio::time::timeout(std::time::Duration::from_secs(10), kill_tagged(m, &tag)).await; + Err(super::query::QUERY_CANCELLED.into()) + } + } +} + +/// A comment to tag a run's operations with, so Stop can find them. +fn op_tag() -> String { + static SEQ: std::sync::atomic::AtomicU64 = std::sync::atomic::AtomicU64::new(0); + format!("stroke-{:x}-{:x}", std::process::id(), SEQ.fetch_add(1, std::sync::atomic::Ordering::Relaxed)) +} + +/// killOp every operation carrying `tag` as its comment. Dropping the cursor +/// only kills it once a batch has come back; a find still scanning for its +/// first batch, or an aggregate's $group, runs on until then. Returns how many. +pub async fn kill_tagged(m: &MongoConn, tag: &str) -> Result { + let admin = m.client.database("admin"); + let pipeline = vec![ + doc! { "$currentOp": { "allUsers": false, "idleCursors": false } }, + doc! { "$match": { "$or": [ { "command.comment": tag }, { "cursor.originatingCommand.comment": tag } ] } }, + doc! { "$project": { "opid": 1 } }, + ]; + let ops: Vec = admin.aggregate(pipeline).await.map_err(err)?.try_collect().await.map_err(err)?; + for op in &ops { + if let Some(id) = op.get("opid") { + let _ = admin.run_command(doc! { "killOp": 1, "op": id.clone() }).await; + } } + Ok(ops.len()) } -async fn stream_rows(m: &MongoConn, stmt: &str, sink: &mut super::query::RowSink) -> Result { +async fn stream_rows(m: &MongoConn, stmt: &str, sink: &mut super::query::RowSink, tag: &str) -> Result { use futures::StreamExt; const SAMPLE: usize = 1_000; let t0 = std::time::Instant::now(); @@ -1495,7 +1525,7 @@ async fn stream_rows(m: &MongoConn, stmt: &str, sink: &mut super::query::RowSink } let mut cursor = match head.name.as_str() { "find" => { - let mut q = c.find(arg_doc(&args, 0, "filter")?).batch_size(1_000); + let mut q = c.find(arg_doc(&args, 0, "filter")?).batch_size(1_000).comment(Bson::String(tag.to_string())); if let Some(p) = args.get(1).filter(|v| v.is_object()) { q = q.projection(to_doc(p, None)?); } @@ -1515,7 +1545,7 @@ async fn stream_rows(m: &MongoConn, stmt: &str, sink: &mut super::query::RowSink Some(Value::Array(stages)) => stages.iter().map(|s| to_doc(s, None)).collect::, _>>()?, _ => return Err("aggregate() takes an array of stages".into()), }; - c.aggregate(pipeline).batch_size(1_000).await.map_err(err)? + c.aggregate(pipeline).batch_size(1_000).comment(Bson::String(tag.to_string())).await.map_err(err)? } }; diff --git a/src-tauri/src/db/posthog.rs b/src-tauri/src/db/posthog.rs index 02dd8844..c7609830 100644 --- a/src-tauri/src/db/posthog.rs +++ b/src-tauri/src/db/posthog.rs @@ -41,6 +41,50 @@ fn client() -> &'static reqwest::Client { /// POST one query object to the project's query endpoint. async fn post_query(config: &PosthogConfig, query: Value) -> Result { + post_query_as(config, query, None).await +} + +/// `query`, stoppable on PostHog's side: the request carries a +/// client_query_id, and Stop asks PostHog to cancel that query (DELETE on the +/// query endpoint) instead of leaving it running against the 3-at-once limit. +pub async fn query_cancellable(config: &PosthogConfig, sql: &str, stop: &super::stop::Stop) -> Result { + let trimmed = sql.trim().trim_end_matches(';'); + if !is_read_query(trimmed) { + return Err("PostHog is read-only here: HogQL runs SELECT (and WITH) queries.".into()); + } + static SEQ: std::sync::atomic::AtomicU64 = std::sync::atomic::AtomicU64::new(0); + let id = format!( + "stroke-{:x}-{:x}-{:x}", + std::process::id(), + std::time::SystemTime::now().duration_since(std::time::UNIX_EPOCH).map(|d| d.as_millis() as u64).unwrap_or(0), + SEQ.fetch_add(1, std::sync::atomic::Ordering::Relaxed) + ); + let t0 = Instant::now(); + let run = post_query_as(config, json!({ "kind": "HogQLQuery", "query": trimmed }), Some(&id)); + tokio::pin!(run); + tokio::select! { + r = &mut run => Ok(to_sql_result(&r?, sql, t0.elapsed().as_millis() as u64)), + _ = stop.wait() => { + drop(run); + let _ = tokio::time::timeout(Duration::from_secs(10), cancel_query(config, &id)).await; + Err(super::query::QUERY_CANCELLED.into()) + } + } +} + +/// DELETE /api/projects/{id}/query/{client_query_id}/: PostHog kills it in ClickHouse. +pub async fn cancel_query(config: &PosthogConfig, client_query_id: &str) -> Result<(), String> { + let url = format!( + "{}/api/projects/{}/query/{}/", + config.base_url(), + urlencoding::encode(config.project_id.trim()), + urlencoding::encode(client_query_id) + ); + let resp = client().delete(&url).bearer_auth(config.api_key.trim()).send().await.map_err(|e| format!("PostHog request failed: {e}"))?; + if resp.status().is_success() { Ok(()) } else { Err(format!("PostHog could not cancel the query (HTTP {})", resp.status().as_u16())) } +} + +async fn post_query_as(config: &PosthogConfig, query: Value, client_query_id: Option<&str>) -> Result { let url = format!( "{}/api/projects/{}/query/", config.base_url(), @@ -49,7 +93,10 @@ async fn post_query(config: &PosthogConfig, query: Value) -> Result json!({ "query": query, "name": "stroke", "client_query_id": id }), + None => json!({ "query": query, "name": "stroke" }), + }) .send() .await .map_err(|e| format!("PostHog request failed: {e}"))?; @@ -392,6 +439,42 @@ pub async fn get_ddl(config: &PosthogConfig, table: &str) -> Result(); + tokio::spawn(async move { + let mut held = Vec::new(); + loop { + let (mut s, _) = l.accept().await.unwrap(); + let mut buf = vec![0u8; 16384]; + let n = s.read(&mut buf).await.unwrap(); + let req = String::from_utf8_lossy(&buf[..n]).to_string(); + let _ = seen_tx.send(req.clone()); + if req.starts_with("DELETE") { + s.write_all(b"HTTP/1.1 204 No Content\r\nContent-Length: 0\r\n\r\n").await.unwrap(); + } else { + held.push(s); // never answered: a long query + } + } + }); + let cfg = PosthogConfig { name: "t".into(), host: format!("http://127.0.0.1:{port}"), project_id: "7".into(), api_key: "k".into() }; + let (tx, rx) = tokio::sync::oneshot::channel(); + let stop = crate::db::stop::Stop::from_rx(rx); + tokio::spawn(async move { tokio::time::sleep(Duration::from_millis(300)).await; let _ = tx.send(()); }); + let e = query_cancellable(&cfg, "SELECT count() FROM events", &stop).await.unwrap_err(); + assert_eq!(e, crate::db::query::QUERY_CANCELLED); + let post = seen_rx.recv().await.unwrap(); + assert!(post.starts_with("POST /api/projects/7/query/"), "{post}"); + let id = post.split("\"client_query_id\":\"").nth(1).and_then(|r| r.split('"').next()).expect("client_query_id sent").to_string(); + let del = seen_rx.recv().await.unwrap(); + assert!(del.starts_with(&format!("DELETE /api/projects/7/query/{id}/")), "{del}"); + } + #[test] fn hogql_response_maps_columns_types_and_rows() { let body = json!({ diff --git a/src-tauri/src/db/query.rs b/src-tauri/src/db/query.rs index 45bce3e3..a56b86c5 100644 --- a/src-tauri/src/db/query.rs +++ b/src-tauri/src/db/query.rs @@ -4628,6 +4628,9 @@ pub(crate) async fn run_one_cancellable( } } } + // Cancel by client_query_id / X-Opaque-Id on the server. + ActiveConnection::Posthog(cfg) => super::posthog::query_cancellable(cfg, stmt, stop).await, + ActiveConnection::Doc(super::docdb::DocConn::Elastic(cfg)) => super::elastic::query_cancellable(cfg, stmt, stop).await, // No server-side stop exists: stop waiting, and say the server may not. other => tokio::select! { r = run_one_other(other, stmt) => r, @@ -6371,6 +6374,24 @@ mod stream_tests { assert_eq!(late[0].last().unwrap(), &serde_json::json!({ "late": "here" })); let grew = rss_mb().saturating_sub(before); assert!(grew < 120, "memory grew {grew} MB while streaming"); + // Stop while the server is still working: the tagged operation is killed. + let slow = "db.big.aggregate([{$group: {_id: {$mod: [\"$n\", 7]}, s: {$sum: {$function: {body: 'function(n){var x=0; for (var i=0;i<20000;i++){x+=i;} return x;}', args: [\"$n\"], lang: 'js'}}}}}])"; + assert!(super::super::mongo::streams(slow)); + let mut sink = sink_into(&store, "mongo-stop"); + let (tx, rx) = tokio::sync::oneshot::channel(); + let stop = super::super::stop::Stop::from_rx(rx); + tokio::spawn(async move { tokio::time::sleep(std::time::Duration::from_millis(1500)).await; let _ = tx.send(()); }); + let t0 = std::time::Instant::now(); + let e = super::super::mongo::stream_into(&m, slow, &mut sink, &stop).await.unwrap_err(); + assert_eq!(e, QUERY_CANCELLED); + assert!(t0.elapsed() < std::time::Duration::from_secs(15)); + tokio::time::sleep(std::time::Duration::from_millis(1000)).await; + let admin = m.client.database("admin"); + let left: Vec = futures::TryStreamExt::try_collect(admin.aggregate(vec![ + doc! { "$currentOp": {} }, + doc! { "$match": { "command.aggregate": "big" } }, + ]).await.unwrap()).await.unwrap(); + assert!(left.is_empty(), "the aggregate still runs on the server: {left:?}"); let _ = c.drop().await; } From 2c83ef0190ae1c85be17947b982810cbeb9d262b Mon Sep 17 00:00:00 2001 From: broisnees Date: Fri, 20 Nov 2026 16:52:19 +0000 Subject: [PATCH 4/6] Query plans for MongoDB, Elasticsearch, OpenSearch, CouchDB and libSQL Explain draws MongoDB's executed stages (aggregate stages above the cursor), a search's profile per shard and Lucene query, CouchDB's _explain index choice, and libSQL's SQLite plan in the plan view. The full-scan warning knows COLLSCAN and CouchDB's all-documents scan. Verified on real MongoDB 7, Elasticsearch 8 and CouchDB 3. --- src-tauri/src/commands.rs | 27 +++++- src-tauri/src/db/couchdb.rs | 39 ++++++++ src-tauri/src/db/elastic.rs | 72 +++++++++++++++ src-tauri/src/db/mongo.rs | 173 ++++++++++++++++++++++++++++------- src-tauri/src/db/query.rs | 57 ++++++++++++ src/lib/explain-plan.js | 7 +- src/lib/explain-plan.test.js | 30 ++++++ 7 files changed, 370 insertions(+), 35 deletions(-) diff --git a/src-tauri/src/commands.rs b/src-tauri/src/commands.rs index 019f20ec..76071146 100644 --- a/src-tauri/src/commands.rs +++ b/src-tauri/src/commands.rs @@ -1234,7 +1234,32 @@ pub async fn pg_explain_sql( let lines = crate::db::oracle::explain_lines(&h, &sql).await?; Ok(explain_from_text_lines(lines, "oracle")) } - _ => Err("EXPLAIN isn't supported for this engine yet".into()), + ActiveConnection::LibSql(cfg) => { + // libSQL is SQLite: the same (id, parent, notused, detail) rows. + let res = crate::db::libsql::query(&cfg, &format!("EXPLAIN QUERY PLAN {sql}"), vec![]).await?; + Ok(explain_from_sqlite_plan(&res, "libsql")) + } + ActiveConnection::Doc(crate::db::docdb::DocConn::Mongo(m)) => Ok(ExplainResult { + plan: crate::db::mongo::explain_plan(&m, &sql).await?, + planning_time: 0.0, + execution_time: 0.0, + driver: "mongodb".into(), + }), + ActiveConnection::Doc(crate::db::docdb::DocConn::Elastic(cfg)) => Ok(ExplainResult { + plan: crate::db::elastic::explain_plan(&cfg, &sql).await?, + planning_time: 0.0, + execution_time: 0.0, + driver: "elasticsearch".into(), + }), + ActiveConnection::Doc(crate::db::docdb::DocConn::Couch(cfg)) => Ok(ExplainResult { + plan: crate::db::couchdb::explain_plan(&cfg, &sql).await?, + planning_time: 0.0, + execution_time: 0.0, + driver: "couchdb".into(), + }), + // Redis, DynamoDB, Firestore, Cassandra/ScyllaDB and PostHog's HogQL + // API expose no query plan. + _ => Err("This engine doesn't expose a query plan".into()), } } diff --git a/src-tauri/src/db/couchdb.rs b/src-tauri/src/db/couchdb.rs index 35ad5fdb..10b22b9f 100644 --- a/src-tauri/src/db/couchdb.rs +++ b/src-tauri/src/db/couchdb.rs @@ -227,6 +227,45 @@ pub async fn query(cfg: &CouchConfig, stmt: &str) -> Result Result { + let Statement::Find { db, selector, opts } = parse(stmt)? else { + return Err("Explain works on find queries: db.find({selector})".into()); + }; + let mut body = json!({ "selector": selector }); + if let Some(o) = opts.as_object() { + for (k, v) in o { + body[k] = v.clone(); + } + } + let out = request(cfg, "POST", &format!("{}/_explain", enc(&db)), Some(&body)).await?; + Ok(explain_tree(&out, &db)) +} + +/// `_explain` output → plan tree. Pure, so it is tested on recorded output. +pub fn explain_tree(out: &Value, db: &str) -> Value { + let ix = &out["index"]; + let kind = ix["type"].as_str().unwrap_or(""); + let name = ix["name"].as_str().unwrap_or(""); + let scan = match kind { + "special" => json!({ "Node Type": "All documents scan", "Relation Name": db }), + _ => json!({ + "Node Type": format!("{} index scan", if kind.is_empty() { "Index" } else { kind }), + "Relation Name": db, + "Index Name": if name.is_empty() { ix["ddoc"].as_str().unwrap_or("") } else { name }, + "Index Cond": ix["def"]["fields"].to_string(), + }), + }; + json!({ + "Node Type": "Find", + "Relation Name": db, + "Filter": out["selector"].to_string(), + "Plan Rows": out["limit"].as_i64(), + "Plans": [scan], + }) +} + /// A result-grid cell: Extended JSON scalars (`{"$oid": ...}`, `{"$date": ...}`, /// `{"$numberLong": ...}`, `{"$numberDecimal": ...}`) as their plain value, and /// the type name for the column header. diff --git a/src-tauri/src/db/elastic.rs b/src-tauri/src/db/elastic.rs index d8360662..b0d249fd 100644 --- a/src-tauri/src/db/elastic.rs +++ b/src-tauri/src/db/elastic.rs @@ -321,6 +321,78 @@ async fn send(cfg: &EsConfig, method: &str, path: &str, body: Option>) Ok(v) } +/// A search's plan: the statement run once with `"profile": true`, and the +/// profile drawn as the plan view's tree (search, then each shard, then its +/// Lucene queries and aggregations with their times). Only searches have one. +pub async fn explain_plan(cfg: &EsConfig, stmt: &str) -> Result { + let (index, mut body) = match parse(stmt)? { + Statement::Search { index, body } => (index, body), + Statement::Count { index, body } => (index, json!({ "query": body.get("query").cloned().unwrap_or(json!({"match_all": {}})), "size": 0 })), + Statement::Raw { method, path, body } if (method == "GET" || method == "POST") && path.trim_end_matches('/').ends_with("_search") => { + let index = path.trim_start_matches('/').split('/').next().unwrap_or("").to_string(); + (index, body.unwrap_or(json!({}))) + } + _ => return Err("Explain works on searches: GET index/_search, index.search({...}) or index.count({...})".into()), + }; + if !body.is_object() { + body = json!({}); + } + body["profile"] = json!(true); + let path = if index.is_empty() || index == "_search" { "_search".to_string() } else { format!("{}/_search", enc(&index)) }; + let v = request(cfg, "POST", &path, Some(&body)).await?; + Ok(profile_tree(&v, &index)) +} + +/// A profiled search response → plan tree. Pure, so it is tested on recorded output. +pub fn profile_tree(v: &Value, index: &str) -> Value { + fn ms(nanos: &Value) -> Value { + nanos.as_f64().map(|n| json!((n / 1e6 * 1000.0).round() / 1000.0)).unwrap_or(Value::Null) + } + fn part(q: &Value) -> Value { + let mut n = json!({ + "Node Type": q["type"].as_str().unwrap_or("Query"), + "Filter": q["description"].as_str().unwrap_or(""), + "Actual Total Time": ms(&q["time_in_nanos"]), + }); + if let Some(kids) = q["children"].as_array().filter(|k| !k.is_empty()) { + n["Plans"] = Value::Array(kids.iter().map(part).collect()); + } + n + } + let shards: Vec = v["profile"]["shards"] + .as_array() + .map(|sh| { + sh.iter() + .map(|shard| { + let mut plans: Vec = shard["searches"] + .as_array() + .map(|ss| ss.iter().flat_map(|s| s["query"].as_array().cloned().unwrap_or_default()).map(|q| part(&q)).collect()) + .unwrap_or_default(); + plans.extend(shard["aggregations"].as_array().map(|a| a.iter().map(part).collect::>()).unwrap_or_default()); + let total: f64 = plans.iter().filter_map(|p| p["Actual Total Time"].as_f64()).sum(); + let id = shard["id"].as_str().unwrap_or(""); + // `[node][index][shard]`: the last two say which. + let label = id.split(']').filter(|p| !p.is_empty()).map(|p| p.trim_start_matches('[')).skip(1).collect::>().join(" shard "); + json!({ + "Node Type": "Shard", + "Relation Name": if label.is_empty() { id.to_string() } else { label }, + "Actual Total Time": (total * 1000.0).round() / 1000.0, + "Plans": plans, + }) + }) + .collect() + }) + .unwrap_or_default(); + let hits = v["hits"]["total"]["value"].as_i64().or_else(|| v["hits"]["total"].as_i64()); + json!({ + "Node Type": "Search", + "Relation Name": index, + "Actual Rows": hits, + "Actual Total Time": v["took"].as_f64(), + "Plans": shards, + }) +} + /// What `GET /` says about the server. #[derive(Debug, Clone, Serialize)] #[serde(rename_all = "camelCase")] diff --git a/src-tauri/src/db/mongo.rs b/src-tauri/src/db/mongo.rs index 06538eed..355831fd 100644 --- a/src-tauri/src/db/mongo.rs +++ b/src-tauri/src/db/mongo.rs @@ -1431,6 +1431,147 @@ fn arg_doc(args: &[Value], i: usize, what: &str) -> Result { } } +/// The command a console find/aggregate/count/distinct explains as. +fn explain_cmd(coll: &str, head: &Call, args: &[Value], sort: Option, limit: Option, skip: Option) -> Result { + let cmd = match head.name.as_str() { + "find" => { + let mut f = doc! { "find": coll, "filter": arg_doc(&args, 0, "filter")? }; + if let Some(p) = args.get(1).filter(|v| v.is_object()) { + f.insert("projection", to_doc(p, None)?); + } + if let Some(s) = sort { + f.insert("sort", s); + } + if let Some(l) = limit { + f.insert("limit", l); + } + if let Some(k) = skip { + f.insert("skip", k as i64); + } + f + } + "aggregate" => { + let pipeline = match args.first() { + Some(Value::Array(stages)) => stages.iter().map(|s| to_doc(s, None)).collect::, _>>()?, + _ => return Err("aggregate() takes an array of stages".into()), + }; + doc! { "aggregate": coll, "pipeline": pipeline, "cursor": {} } + } + "countDocuments" => doc! { "count": coll, "query": arg_doc(&args, 0, "filter")? }, + "distinct" => { + let field = args.first().and_then(|v| v.as_str()).ok_or("distinct() takes a field name")?; + doc! { "distinct": coll, "key": field, "query": arg_doc(&args, 1, "filter")? } + } + other => return Err(format!("{other}() can't be explained here; try find() or aggregate()")), + }; + Ok(cmd) +} + + +/// A console statement's plan as the plan view's tree (the Postgres-shaped +/// `{Node Type, Plans, Actual Rows, ...}` it draws for every engine): the +/// executed stages with their counts and timings, aggregate stages stacked +/// above the cursor's tree. +pub async fn explain_plan(m: &MongoConn, stmt: &str) -> Result { + let db_name = m.current_db.lock().map(|g| g.clone()).unwrap_or_default(); + let Stmt::Coll { coll, calls } = parse(stmt)? else { + return Err("Explain works on db..find(...), aggregate(...), countDocuments(...) or distinct(...)".into()); + }; + let calls: Vec = calls.into_iter().filter(|c| c.name != "explain").collect(); + let (head, mods) = calls.split_first().ok_or("Missing method")?; + let args = parse_args(&head.args)?; + let (mut sort, mut limit, mut skip) = (None, None, None); + for md in mods { + let a = parse_args(&md.args)?; + match md.name.as_str() { + "sort" => sort = Some(arg_doc(&a, 0, "sort")?), + "limit" => limit = a.first().and_then(|v| v.as_i64()), + "skip" => skip = a.first().and_then(|v| v.as_u64()), + _ => {} + } + } + let cmd = explain_cmd(&coll, head, &args, sort, limit, skip)?; + let raw = m.client.database(&db_name).run_command(doc! { "explain": cmd, "verbosity": "executionStats" }).await.map_err(err)?; + Ok(plan_tree(&doc_to_json(&raw), &coll)) +} + +/// Explain output → plan tree. Pure, so it is tested on recorded output. +pub fn plan_tree(raw: &Value, coll: &str) -> Value { + fn stage_node(st: &Value, coll: &str) -> Value { + let name = st["stage"].as_str().unwrap_or("STAGE"); + let mut n = serde_json::Map::new(); + n.insert("Node Type".into(), json!(name)); + if matches!(name, "COLLSCAN" | "IXSCAN" | "FETCH" | "IDHACK" | "EXPRESS_IXSCAN" | "CLUSTERED_IXSCAN") { + n.insert("Relation Name".into(), json!(coll)); + } + if let Some(ix) = st["indexName"].as_str() { + n.insert("Index Name".into(), json!(ix)); + } + let num = |k: &str| st[k].as_i64().or_else(|| st[k].as_f64().map(|f| f as i64)).or_else(|| st[k].as_str().and_then(|s| s.parse().ok())); + if let Some(r) = num("nReturned") { + n.insert("Actual Rows".into(), json!(r)); + } + if let Some(ms) = num("executionTimeMillisEstimate") { + n.insert("Actual Total Time".into(), json!(ms)); + } + if let Some(d) = num("docsExamined") { + n.insert("Docs Examined".into(), json!(d)); + if let Some(r) = num("nReturned") { + if d > r && st.get("filter").is_some() { + n.insert("Rows Removed by Filter".into(), json!(d - r)); + } + } + } + if let Some(k) = num("keysExamined") { + n.insert("Keys Examined".into(), json!(k)); + } + if let Some(f) = st.get("filter") { + n.insert("Filter".into(), json!(f.to_string())); + } + if let Some(kp) = st.get("keyPattern") { + n.insert("Index Cond".into(), json!(kp.to_string())); + } + let mut kids: Vec = vec![]; + if let Some(c) = st.get("inputStage") { + kids.push(stage_node(c, coll)); + } + if let Some(cs) = st["inputStages"].as_array() { + kids.extend(cs.iter().map(|c| stage_node(c, coll))); + } + if !kids.is_empty() { + n.insert("Plans".into(), Value::Array(kids)); + } + Value::Object(n) + } + // aggregate: the find part sits in stages[0].$cursor, the rest above it. + let (base, later): (&Value, Vec<&Value>) = match raw["stages"].as_array() { + Some(st) if st.first().is_some_and(|s| s.get("$cursor").is_some()) => (&st[0]["$cursor"], st.iter().skip(1).collect()), + _ => (raw, vec![]), + }; + let exec = &base["executionStats"]["executionStages"]; + let mut tree = if exec.get("stage").is_some() { + stage_node(exec, coll) + } else { + let wp = &base["queryPlanner"]["winningPlan"]; + // SBE plans nest the classic tree under queryPlan. + stage_node(if wp.get("queryPlan").is_some() { &wp["queryPlan"] } else { wp }, coll) + }; + for st in later { + let Some((name, _)) = st.as_object().and_then(|o| o.iter().find(|(k, _)| k.starts_with('$'))) else { continue }; + let mut n = serde_json::Map::new(); + n.insert("Node Type".into(), json!(name)); + if let Some(r) = st["nReturned"].as_i64() { + n.insert("Actual Rows".into(), json!(r)); + } + if let Some(ms) = st["executionTimeMillisEstimate"].as_i64() { + n.insert("Actual Total Time".into(), json!(ms)); + } + n.insert("Plans".into(), json!([tree])); + tree = Value::Object(n); + } + tree +} + const CONSOLE_CAP: i64 = 5_000; /// Fields the first batch did not have go here, as one object, so a later @@ -1637,37 +1778,7 @@ pub async fn query(m: &MongoConn, stmt: &str) -> Result { - let mut f = doc! { "find": coll.as_str(), "filter": arg_doc(&args, 0, "filter")? }; - if let Some(p) = args.get(1).filter(|v| v.is_object()) { - f.insert("projection", to_doc(p, None)?); - } - if let Some(s) = sort { - f.insert("sort", s); - } - if let Some(l) = limit { - f.insert("limit", l); - } - if let Some(k) = skip { - f.insert("skip", k as i64); - } - f - } - "aggregate" => { - let pipeline = match args.first() { - Some(Value::Array(stages)) => stages.iter().map(|s| to_doc(s, None)).collect::, _>>()?, - _ => return Err("aggregate() takes an array of stages".into()), - }; - doc! { "aggregate": coll.as_str(), "pipeline": pipeline, "cursor": {} } - } - "countDocuments" => doc! { "count": coll.as_str(), "query": arg_doc(&args, 0, "filter")? }, - "distinct" => { - let field = args.first().and_then(|v| v.as_str()).ok_or("distinct() takes a field name")?; - doc! { "distinct": coll.as_str(), "key": field, "query": arg_doc(&args, 1, "filter")? } - } - other => return Err(format!("{other}() can't be explained here; try find() or aggregate()")), - }; + let cmd = explain_cmd(&coll, head, &args, sort, limit, skip)?; let raw = m.client.database(&db_name).run_command(doc! { "explain": cmd, "verbosity": "executionStats" }).await.map_err(err)?; let sum = summarize_explain(&raw); let rows = sum diff --git a/src-tauri/src/db/query.rs b/src-tauri/src/db/query.rs index a56b86c5..90939d54 100644 --- a/src-tauri/src/db/query.rs +++ b/src-tauri/src/db/query.rs @@ -6395,6 +6395,63 @@ mod stream_tests { let _ = c.drop().await; } + /// Explain on the document engines, against real servers: + /// STROKE_TEST_MONGO=mongodb://127.0.0.1:27018 STROKE_TEST_ES=http://127.0.0.1:19200 + /// STROKE_TEST_COUCH=http://admin:admin@127.0.0.1:15984 cargo test --lib live_doc_plans + #[tokio::test] + async fn live_doc_plans() { + fn types(n: &Value, out: &mut Vec) { + out.push(n["Node Type"].as_str().unwrap_or("").to_string()); + for k in n["Plans"].as_array().into_iter().flatten() { + types(k, out); + } + } + if let Ok(uri) = std::env::var("STROKE_TEST_MONGO") { + use mongodb::bson::doc; + let m = super::super::mongo::connect(&super::super::mongo::MongoConfig { ssh: None, name: "t".into(), uri, database: Some("stroke_plan".into()) }).await.unwrap(); + let c = m.client.database("stroke_plan").collection::("t"); + let _ = c.drop().await; + c.insert_many((0..2000).map(|i| doc! { "n": i, "k": i % 10 })).await.unwrap(); + c.create_index(mongodb::IndexModel::builder().keys(doc! { "n": 1 }).build()).await.unwrap(); + let p = super::super::mongo::explain_plan(&m, "db.t.find({n: {$gt: 1500}}).sort({n: 1})").await.unwrap(); + let mut t = vec![]; + types(&p, &mut t); + assert!(t.iter().any(|s| s == "IXSCAN"), "{p}"); + assert!(p.to_string().contains("n_1"), "{p}"); + let p = super::super::mongo::explain_plan(&m, "db.t.find({k: 3})").await.unwrap(); + assert!(p.to_string().contains("COLLSCAN"), "{p}"); + assert!(p.to_string().contains("Rows Removed by Filter"), "{p}"); + let p = super::super::mongo::explain_plan(&m, "db.t.aggregate([{$match: {k: 3}}, {$group: {_id: null, s: {$sum: \"$n\"}}}])").await.unwrap(); + assert!(p["Node Type"].is_string(), "{p}"); + println!("mongo aggregate plan: {p}"); + let _ = c.drop().await; + } + if let Ok(url) = std::env::var("STROKE_TEST_ES") { + let cfg = super::super::elastic::EsConfig { ssh: None, via: None, name: "t".into(), url, user: None, password: None, api_key: None, flavor: None, insecure: true }; + let cfg = super::super::elastic::connect(cfg).await.unwrap(); + super::super::elastic::request(&cfg, "PUT", "stroke_plan/_doc/1?refresh=wait_for", Some(&serde_json::json!({"level": "error", "ms": 12}))).await.unwrap(); + let p = super::super::elastic::explain_plan(&cfg, r#"stroke_plan.search({"query": {"bool": {"must": [{"term": {"level.keyword": "error"}}], "filter": [{"range": {"ms": {"gte": 5}}}]}}})"#).await.unwrap(); + let mut t = vec![]; + types(&p, &mut t); + assert_eq!(t[0], "Search"); + assert!(t.iter().any(|s| s == "Shard"), "{p}"); + assert!(t.iter().any(|s| s == "BooleanQuery"), "{p}"); + assert_eq!(p["Actual Rows"], serde_json::json!(1)); + let p = super::super::elastic::explain_plan(&cfg, "GET stroke_plan/_search\n{\"query\": {\"match_all\": {}}}").await.unwrap(); + assert!(p.to_string().contains("MatchAllDocsQuery"), "{p}"); + let _ = super::super::elastic::request(&cfg, "DELETE", "stroke_plan", None).await; + } + if let Ok(url) = std::env::var("STROKE_TEST_COUCH") { + let cfg = super::super::couchdb::CouchConfig { ssh: None, via: None, name: "t".into(), url, user: Some("admin".into()), password: Some("admin".into()) }; + // Set up outside (creating databases is refused here): stroke_plan + // with a "by-level" index on level. + let p = super::super::couchdb::explain_plan(&cfg, r#"stroke_plan.find({"other": 1})"#).await.unwrap(); + assert!(p.to_string().contains("All documents scan"), "{p}"); + let p = super::super::couchdb::explain_plan(&cfg, r#"stroke_plan.find({"level": "error"})"#).await.unwrap(); + assert!(p.to_string().contains("by-level"), "{p}"); + } + } + #[tokio::test] async fn clickhouse_error_after_the_first_rows_is_reported() { let port = fake_clickhouse(500, "Code: 241. DB::Exception: Memory limit exceeded\n").await; diff --git a/src/lib/explain-plan.js b/src/lib/explain-plan.js index 4f9f1e48..911b69f6 100644 --- a/src/lib/explain-plan.js +++ b/src/lib/explain-plan.js @@ -290,7 +290,8 @@ export function foldLabel(f) { // ── Findings ───────────────────────────────────────────────────────────────── const fmt = (/** @type {number} */ n) => (n >= 1e6 ? `${(n / 1e6).toFixed(1)}M` : n >= 1e4 ? `${Math.round(n / 1e3)}k` : n.toLocaleString('en-US')) -const isFullScan = (/** @type {string} */ t) => /^(Seq Scan|Full Table Scan|Parallel Seq Scan)$/i.test(t) || /^SCAN \w+$/i.test(t) +// MongoDB's COLLSCAN and CouchDB's all-documents scan read everything too. +const isFullScan = (/** @type {string} */ t) => /^(Seq Scan|Full Table Scan|Parallel Seq Scan|COLLSCAN|All documents scan)$/i.test(t) || /^SCAN \w+$/i.test(t) /** * What is worth fixing, worst first. Every node also gets its own warnings, @@ -315,8 +316,8 @@ export function planInsights(a) { const filter = r.Filter ? ` filtered by ${String(r.Filter)}` : '' if (n.removed && n.removed > Math.max(1000, rows * 4)) { add(n, 'high', `Full scan${where} throws away most rows`, `Read every row and kept ${fmt(rows)}, discarding ${fmt(n.removed)}${filter}. An index on the filtered columns would let it read only the matches.`) - } else if (rows > 10000 || (num(r['Rows Examined']) ?? 0) > 10000) { - add(n, 'medium', `Full scan${where}`, `Reads all ${fmt(Math.max(rows, num(r['Rows Examined']) ?? 0))} rows${filter}. Fine for small tables or when most rows are needed; otherwise look at an index.`) + } else if (rows > 10000 || (num(r['Rows Examined'] ?? r['Docs Examined']) ?? 0) > 10000) { + add(n, 'medium', `Full scan${where}`, `Reads all ${fmt(Math.max(rows, num(r['Rows Examined'] ?? r['Docs Examined']) ?? 0))} rows${filter}. Fine for small tables or when most rows are needed; otherwise look at an index.`) } } if (n.misestimate != null && Math.abs(n.misestimate) >= 10 && Math.max(n.actualRows ?? 0, n.planRows ?? 0) >= 100) { diff --git a/src/lib/explain-plan.test.js b/src/lib/explain-plan.test.js index 97ff47c1..df996708 100644 --- a/src/lib/explain-plan.test.js +++ b/src/lib/explain-plan.test.js @@ -129,3 +129,33 @@ describe('planOutline', () => { expect(entries.every((e) => e.kind === 'node')).toBe(true) }) }) + +describe('document engine plans', () => { + // Shape built by mongo::plan_tree from a real MongoDB 7 explain. + const mongo = { + 'Node Type': '$group', 'Actual Rows': 1, 'Actual Total Time': 40, + Plans: [{ + 'Node Type': 'COLLSCAN', 'Relation Name': 'orders', 'Actual Rows': 2000, 'Actual Total Time': 38, + 'Docs Examined': 200000, 'Rows Removed by Filter': 198000, Filter: '{"k":{"$eq":3}}', + }], + } + + it('flags a MongoDB collection scan that throws away most documents', () => { + const a = annotatePlan(normalizePlan(mongo, 'mongodb')) + expect(a.root?.type).toBe('$group') + expect(a.nodes).toHaveLength(2) + const insights = planInsights(a) + expect(insights[0].title).toBe('Full scan on orders throws away most rows') + }) + + it('reads an Elasticsearch profile as a timed tree', () => { + const es = { + 'Node Type': 'Search', 'Relation Name': 'logs', 'Actual Rows': 1, 'Actual Total Time': 3, + Plans: [{ 'Node Type': 'Shard', 'Relation Name': 'logs shard 0', 'Actual Total Time': 0.4, + Plans: [{ 'Node Type': 'BooleanQuery', Filter: '+level:error', 'Actual Total Time': 0.4 }] }], + } + const a = annotatePlan(normalizePlan(es, 'elasticsearch')) + expect(a.nodes.map((n) => n.type)).toEqual(['Search', 'Shard', 'BooleanQuery']) + expect(a.analyzed).toBe(true) + }) +}) From d0d1ab15b9262f079c0584477d2eac959a9ceafb Mon Sep 17 00:00:00 2001 From: broisnees Date: Fri, 20 Nov 2026 16:56:26 +0000 Subject: [PATCH 5/6] Outline the filter rows in menus and the terminal's find box Menu and popover filter rows (chart picker, database switcher, data diff pickers, column finder, model picker) are now an inset search-shell, the same outline as every page search. The terminal's find box drops its drop shadow for the outline and lights its border on focus. --- DESIGN_SYSTEM.md | 6 ++++-- src/lib/components/AiModelPicker.svelte | 2 +- src/lib/components/ChartView.svelte | 2 +- src/lib/components/DataDiffPage.svelte | 4 ++-- src/lib/components/StatusBar.svelte | 2 +- src/lib/components/TableTextView.svelte | 2 +- src/lib/components/TerminalPage.svelte | 2 +- 7 files changed, 11 insertions(+), 9 deletions(-) diff --git a/DESIGN_SYSTEM.md b/DESIGN_SYSTEM.md index 71b59f30..2fd5e119 100644 --- a/DESIGN_SYSTEM.md +++ b/DESIGN_SYSTEM.md @@ -357,8 +357,10 @@ pass `rank={null}` for a field `/` should not focus. Where a field keeps its own keyboard handling or toolbar cluster, put `search-box no-field-frame` on the input (icon positioned over it), or `search-shell` on a wrapper row that holds icon, input and buttons. Both draw -the same outline. Filter rows at the top of a menu or popover list -(`data-slot="menu-search"`) stay frameless: the menu is already the box. +the same outline. A filter row at the top of a menu or popover list +(`data-slot="menu-search"`) is a `search-shell` inset with `m-1.5` (`h-8`), +so it reads as the same field inside the menu. The command palette's title +row is the one exception: the palette itself is the field. **Prominent pick lists** (provider DB lists, account lists): ``` diff --git a/src/lib/components/AiModelPicker.svelte b/src/lib/components/AiModelPicker.svelte index 3ff93fee..64d7c1bd 100644 --- a/src/lib/components/AiModelPicker.svelte +++ b/src/lib/components/AiModelPicker.svelte @@ -320,7 +320,7 @@
-