diff --git a/DESIGN_SYSTEM.md b/DESIGN_SYSTEM.md index 71b59f30..2fd5e119 100644 --- a/DESIGN_SYSTEM.md +++ b/DESIGN_SYSTEM.md @@ -357,8 +357,10 @@ pass `rank={null}` for a field `/` should not focus. Where a field keeps its own keyboard handling or toolbar cluster, put `search-box no-field-frame` on the input (icon positioned over it), or `search-shell` on a wrapper row that holds icon, input and buttons. Both draw -the same outline. Filter rows at the top of a menu or popover list -(`data-slot="menu-search"`) stay frameless: the menu is already the box. +the same outline. A filter row at the top of a menu or popover list +(`data-slot="menu-search"`) is a `search-shell` inset with `m-1.5` (`h-8`), +so it reads as the same field inside the menu. The command palette's title +row is the one exception: the palette itself is the field. **Prominent pick lists** (provider DB lists, account lists): ``` diff --git a/src-tauri/src/commands.rs b/src-tauri/src/commands.rs index 019f20ec..76071146 100644 --- a/src-tauri/src/commands.rs +++ b/src-tauri/src/commands.rs @@ -1234,7 +1234,32 @@ pub async fn pg_explain_sql( let lines = crate::db::oracle::explain_lines(&h, &sql).await?; Ok(explain_from_text_lines(lines, "oracle")) } - _ => Err("EXPLAIN isn't supported for this engine yet".into()), + ActiveConnection::LibSql(cfg) => { + // libSQL is SQLite: the same (id, parent, notused, detail) rows. + let res = crate::db::libsql::query(&cfg, &format!("EXPLAIN QUERY PLAN {sql}"), vec![]).await?; + Ok(explain_from_sqlite_plan(&res, "libsql")) + } + ActiveConnection::Doc(crate::db::docdb::DocConn::Mongo(m)) => Ok(ExplainResult { + plan: crate::db::mongo::explain_plan(&m, &sql).await?, + planning_time: 0.0, + execution_time: 0.0, + driver: "mongodb".into(), + }), + ActiveConnection::Doc(crate::db::docdb::DocConn::Elastic(cfg)) => Ok(ExplainResult { + plan: crate::db::elastic::explain_plan(&cfg, &sql).await?, + planning_time: 0.0, + execution_time: 0.0, + driver: "elasticsearch".into(), + }), + ActiveConnection::Doc(crate::db::docdb::DocConn::Couch(cfg)) => Ok(ExplainResult { + plan: crate::db::couchdb::explain_plan(&cfg, &sql).await?, + planning_time: 0.0, + execution_time: 0.0, + driver: "couchdb".into(), + }), + // Redis, DynamoDB, Firestore, Cassandra/ScyllaDB and PostHog's HogQL + // API expose no query plan. + _ => Err("This engine doesn't expose a query plan".into()), } } diff --git a/src-tauri/src/db/couchdb.rs b/src-tauri/src/db/couchdb.rs index 35ad5fdb..10b22b9f 100644 --- a/src-tauri/src/db/couchdb.rs +++ b/src-tauri/src/db/couchdb.rs @@ -227,6 +227,45 @@ pub async fn query(cfg: &CouchConfig, stmt: &str) -> Result Result { + let Statement::Find { db, selector, opts } = parse(stmt)? else { + return Err("Explain works on find queries: db.find({selector})".into()); + }; + let mut body = json!({ "selector": selector }); + if let Some(o) = opts.as_object() { + for (k, v) in o { + body[k] = v.clone(); + } + } + let out = request(cfg, "POST", &format!("{}/_explain", enc(&db)), Some(&body)).await?; + Ok(explain_tree(&out, &db)) +} + +/// `_explain` output → plan tree. Pure, so it is tested on recorded output. +pub fn explain_tree(out: &Value, db: &str) -> Value { + let ix = &out["index"]; + let kind = ix["type"].as_str().unwrap_or(""); + let name = ix["name"].as_str().unwrap_or(""); + let scan = match kind { + "special" => json!({ "Node Type": "All documents scan", "Relation Name": db }), + _ => json!({ + "Node Type": format!("{} index scan", if kind.is_empty() { "Index" } else { kind }), + "Relation Name": db, + "Index Name": if name.is_empty() { ix["ddoc"].as_str().unwrap_or("") } else { name }, + "Index Cond": ix["def"]["fields"].to_string(), + }), + }; + json!({ + "Node Type": "Find", + "Relation Name": db, + "Filter": out["selector"].to_string(), + "Plan Rows": out["limit"].as_i64(), + "Plans": [scan], + }) +} + /// A result-grid cell: Extended JSON scalars (`{"$oid": ...}`, `{"$date": ...}`, /// `{"$numberLong": ...}`, `{"$numberDecimal": ...}`) as their plain value, and /// the type name for the column header. diff --git a/src-tauri/src/db/duckdb.rs b/src-tauri/src/db/duckdb.rs index 425efc8e..e3df418d 100644 --- a/src-tauri/src/db/duckdb.rs +++ b/src-tauri/src/db/duckdb.rs @@ -165,14 +165,45 @@ fn for_each_row( params: &[DuckValue], on_columns: impl FnOnce(Vec), mut on_row: impl FnMut(Vec) -> bool, +) -> Result<(), String> { + for_each_row_in(conn, sql, params, false, on_columns, on_row) +} + +/// `for_each_row` on DuckDB's streaming result: chunks are produced as they +/// are read, so DuckDB never holds the whole result either. For reads only +/// (streaming execution refuses DML that returns rows). +fn for_each_row_streaming( + conn: &::duckdb::Connection, + sql: &str, + on_columns: impl FnOnce(Vec), + on_row: impl FnMut(Vec) -> bool, +) -> Result<(), String> { + for_each_row_in(conn, sql, &[], true, on_columns, on_row) +} + +fn for_each_row_in( + conn: &::duckdb::Connection, + sql: &str, + params: &[DuckValue], + streaming: bool, + on_columns: impl FnOnce(Vec), + mut on_row: impl FnMut(Vec) -> bool, ) -> Result<(), String> { let mut stmt = conn.prepare(sql).map_err(|e| format!("DuckDB error: {e}"))?; // column_names()/column_count() panic before the statement is executed, so we // run query() first (it executes eagerly) and read metadata via rows.as_ref(). - let mut rows = stmt - .query(::duckdb::params_from_iter(params.iter())) - .map_err(|e| format!("DuckDB error: {e}"))?; + // + // Streaming: stream_arrow executes in DuckDB's streaming mode, and + // raw_query reads that same result through the row API. Its own Arrow + // iterator panics on a mid-stream error (fatal with panic=abort); the + // row API returns the error instead. + let mut rows = if streaming { + drop(stmt.stream_arrow(::duckdb::params_from_iter(params.iter())).map_err(|e| format!("DuckDB error: {e}"))?); + stmt.raw_query() + } else { + stmt.query(::duckdb::params_from_iter(params.iter())).map_err(|e| format!("DuckDB error: {e}"))? + }; let (col_names, col_count): (Vec, usize) = match rows.as_ref() { Some(s) => (s.column_names(), s.column_count()), @@ -365,10 +396,9 @@ pub async fn stream_into( g.1 = Some(conn.interrupt_handle()); } let mut batch = Vec::with_capacity(BATCH); - let out = for_each_row( + let out = for_each_row_streaming( conn, &sql_owned, - &[], |c| { let _ = tx.blocking_send(Msg::Columns(c)); }, diff --git a/src-tauri/src/db/elastic.rs b/src-tauri/src/db/elastic.rs index 47c5296c..b0d249fd 100644 --- a/src-tauri/src/db/elastic.rs +++ b/src-tauri/src/db/elastic.rs @@ -232,6 +232,48 @@ enum Body<'a> { Ndjson(String), } +tokio::task_local! { + /// The console run's id, sent as X-Opaque-Id so Stop can find its tasks. + static OPAQUE_ID: String; +} + +/// `query`, stoppable on the server. Every request the statement makes +/// carries an X-Opaque-Id; Stop drops the request and cancels the tasks that +/// carry that id. Searches already end when their connection closes, but +/// `_sql`, `_delete_by_query`, `_update_by_query` and the like run on. +pub async fn query_cancellable(cfg: &EsConfig, stmt: &str, stop: &super::stop::Stop) -> Result { + static SEQ: std::sync::atomic::AtomicU64 = std::sync::atomic::AtomicU64::new(0); + let id = format!("stroke-{:x}-{:x}", std::process::id(), SEQ.fetch_add(1, std::sync::atomic::Ordering::Relaxed)); + let run = OPAQUE_ID.scope(id.clone(), query(cfg, stmt)); + tokio::pin!(run); + tokio::select! { + r = &mut run => r, + _ = stop.wait() => { + drop(run); + let _ = tokio::time::timeout(Duration::from_secs(10), cancel_tasks(cfg, &id)).await; + Err(super::query::QUERY_CANCELLED.into()) + } + } +} + +/// Cancel every task whose X-Opaque-Id is `id`. Returns how many. +pub async fn cancel_tasks(cfg: &EsConfig, id: &str) -> Result { + let v = send(cfg, "GET", "_tasks?detailed=true&group_by=none", None).await?; + let ids: Vec = v["tasks"] + .as_array() + .map(|t| { + t.iter() + .filter(|t| t["headers"]["X-Opaque-Id"].as_str() == Some(id) && t["cancellable"].as_bool() != Some(false)) + .filter_map(|t| Some(format!("{}:{}", t["node"].as_str()?, t["id"].as_i64()?))) + .collect() + }) + .unwrap_or_default(); + for t in &ids { + let _ = send(cfg, "POST", &format!("_tasks/{t}/_cancel"), None).await; + } + Ok(ids.len()) +} + async fn send(cfg: &EsConfig, method: &str, path: &str, body: Option>) -> Result { let url = base(cfg)?.join(path.trim_start_matches('/')).map_err(|e| format!("Invalid path \"{path}\": {e}"))?; let m = reqwest::Method::from_bytes(method.as_bytes()).map_err(|e| e.to_string())?; @@ -244,6 +286,9 @@ async fn send(cfg: &EsConfig, method: &str, path: &str, body: Option>) } else if let Some(u) = cfg.user.as_deref().filter(|u| !u.is_empty()) { req = req.basic_auth(u, cfg.password.as_deref()); } + if let Ok(id) = OPAQUE_ID.try_with(Clone::clone) { + req = req.header("X-Opaque-Id", id); + } match body { Some(Body::Json(b)) => req = req.json(b), Some(Body::Ndjson(s)) => req = req.header("Content-Type", "application/x-ndjson").body(s), @@ -276,6 +321,78 @@ async fn send(cfg: &EsConfig, method: &str, path: &str, body: Option>) Ok(v) } +/// A search's plan: the statement run once with `"profile": true`, and the +/// profile drawn as the plan view's tree (search, then each shard, then its +/// Lucene queries and aggregations with their times). Only searches have one. +pub async fn explain_plan(cfg: &EsConfig, stmt: &str) -> Result { + let (index, mut body) = match parse(stmt)? { + Statement::Search { index, body } => (index, body), + Statement::Count { index, body } => (index, json!({ "query": body.get("query").cloned().unwrap_or(json!({"match_all": {}})), "size": 0 })), + Statement::Raw { method, path, body } if (method == "GET" || method == "POST") && path.trim_end_matches('/').ends_with("_search") => { + let index = path.trim_start_matches('/').split('/').next().unwrap_or("").to_string(); + (index, body.unwrap_or(json!({}))) + } + _ => return Err("Explain works on searches: GET index/_search, index.search({...}) or index.count({...})".into()), + }; + if !body.is_object() { + body = json!({}); + } + body["profile"] = json!(true); + let path = if index.is_empty() || index == "_search" { "_search".to_string() } else { format!("{}/_search", enc(&index)) }; + let v = request(cfg, "POST", &path, Some(&body)).await?; + Ok(profile_tree(&v, &index)) +} + +/// A profiled search response → plan tree. Pure, so it is tested on recorded output. +pub fn profile_tree(v: &Value, index: &str) -> Value { + fn ms(nanos: &Value) -> Value { + nanos.as_f64().map(|n| json!((n / 1e6 * 1000.0).round() / 1000.0)).unwrap_or(Value::Null) + } + fn part(q: &Value) -> Value { + let mut n = json!({ + "Node Type": q["type"].as_str().unwrap_or("Query"), + "Filter": q["description"].as_str().unwrap_or(""), + "Actual Total Time": ms(&q["time_in_nanos"]), + }); + if let Some(kids) = q["children"].as_array().filter(|k| !k.is_empty()) { + n["Plans"] = Value::Array(kids.iter().map(part).collect()); + } + n + } + let shards: Vec = v["profile"]["shards"] + .as_array() + .map(|sh| { + sh.iter() + .map(|shard| { + let mut plans: Vec = shard["searches"] + .as_array() + .map(|ss| ss.iter().flat_map(|s| s["query"].as_array().cloned().unwrap_or_default()).map(|q| part(&q)).collect()) + .unwrap_or_default(); + plans.extend(shard["aggregations"].as_array().map(|a| a.iter().map(part).collect::>()).unwrap_or_default()); + let total: f64 = plans.iter().filter_map(|p| p["Actual Total Time"].as_f64()).sum(); + let id = shard["id"].as_str().unwrap_or(""); + // `[node][index][shard]`: the last two say which. + let label = id.split(']').filter(|p| !p.is_empty()).map(|p| p.trim_start_matches('[')).skip(1).collect::>().join(" shard "); + json!({ + "Node Type": "Shard", + "Relation Name": if label.is_empty() { id.to_string() } else { label }, + "Actual Total Time": (total * 1000.0).round() / 1000.0, + "Plans": plans, + }) + }) + .collect() + }) + .unwrap_or_default(); + let hits = v["hits"]["total"]["value"].as_i64().or_else(|| v["hits"]["total"].as_i64()); + json!({ + "Node Type": "Search", + "Relation Name": index, + "Actual Rows": hits, + "Actual Total Time": v["took"].as_f64(), + "Plans": shards, + }) +} + /// What `GET /` says about the server. #[derive(Debug, Clone, Serialize)] #[serde(rename_all = "camelCase")] @@ -862,6 +979,45 @@ mod tests { assert_eq!(urlencoding::encode("a/b"), "a%2Fb"); } + /// Stop on a slow search: while it runs the server lists a task with our + /// X-Opaque-Id, and after Stop none is left. + // STROKE_TEST_ES=http://127.0.0.1:19200 cargo test --lib elastic::tests::live_stop + #[tokio::test] + async fn live_stop_cancels_the_task() { + let Ok(url) = std::env::var("STROKE_TEST_ES") else { return }; + let cfg = EsConfig { ssh: None, via: None, name: "t".into(), url, user: None, password: None, api_key: None, flavor: None, insecure: true }; + let cfg = connect(cfg).await.unwrap(); + let _ = send(&cfg, "DELETE", "stroke_slow", None).await; + let mut nd = String::new(); + for i in 0..20_000 { + nd.push_str(&format!("{{\"index\":{{\"_index\":\"stroke_slow\"}}}}\n{{\"n\":{i}}}\n")); + } + send(&cfg, "POST", "_bulk?refresh=true", Some(Body::Ndjson(nd))).await.unwrap(); + let ours = |v: &Value| -> usize { + v["tasks"].as_array().map(|t| t.iter().filter(|t| t["headers"]["X-Opaque-Id"].as_str().is_some_and(|h| h.starts_with("stroke-"))).count()).unwrap_or(0) + }; + let (tx, rx) = tokio::sync::oneshot::channel(); + let stop = super::super::stop::Stop::from_rx(rx); + let probe = cfg.clone(); + let during = tokio::spawn(async move { + tokio::time::sleep(Duration::from_millis(1500)).await; + let v = send(&probe, "GET", "_tasks?detailed=true&group_by=none", None).await.unwrap(); + let n = ours(&v); + let _ = tx.send(()); + n + }); + let slow = "GET stroke_slow/_search\n{\"query\": {\"script\": {\"script\": \"long x = 0; for (int i = 0; i < 900000; i++) { x += i; } return x > 0;\"}}}"; + let t0 = Instant::now(); + let e = query_cancellable(&cfg, slow, &stop).await.unwrap_err(); + assert_eq!(e, super::super::query::QUERY_CANCELLED); + assert!(during.await.unwrap() > 0, "the search carried our X-Opaque-Id"); + assert!(t0.elapsed() < Duration::from_secs(20)); + tokio::time::sleep(Duration::from_millis(1500)).await; + let v = send(&cfg, "GET", "_tasks?detailed=true&group_by=none", None).await.unwrap(); + assert_eq!(ours(&v), 0, "a task is still running after Stop"); + let _ = send(&cfg, "DELETE", "stroke_slow", None).await; + } + /// Against a local server: `cargo test --lib es_live -- --ignored`, with /// ES_URL (default http://localhost:9200). #[tokio::test] diff --git a/src-tauri/src/db/mongo.rs b/src-tauri/src/db/mongo.rs index a27fc656..355831fd 100644 --- a/src-tauri/src/db/mongo.rs +++ b/src-tauri/src/db/mongo.rs @@ -1431,8 +1431,306 @@ fn arg_doc(args: &[Value], i: usize, what: &str) -> Result { } } +/// The command a console find/aggregate/count/distinct explains as. +fn explain_cmd(coll: &str, head: &Call, args: &[Value], sort: Option, limit: Option, skip: Option) -> Result { + let cmd = match head.name.as_str() { + "find" => { + let mut f = doc! { "find": coll, "filter": arg_doc(&args, 0, "filter")? }; + if let Some(p) = args.get(1).filter(|v| v.is_object()) { + f.insert("projection", to_doc(p, None)?); + } + if let Some(s) = sort { + f.insert("sort", s); + } + if let Some(l) = limit { + f.insert("limit", l); + } + if let Some(k) = skip { + f.insert("skip", k as i64); + } + f + } + "aggregate" => { + let pipeline = match args.first() { + Some(Value::Array(stages)) => stages.iter().map(|s| to_doc(s, None)).collect::, _>>()?, + _ => return Err("aggregate() takes an array of stages".into()), + }; + doc! { "aggregate": coll, "pipeline": pipeline, "cursor": {} } + } + "countDocuments" => doc! { "count": coll, "query": arg_doc(&args, 0, "filter")? }, + "distinct" => { + let field = args.first().and_then(|v| v.as_str()).ok_or("distinct() takes a field name")?; + doc! { "distinct": coll, "key": field, "query": arg_doc(&args, 1, "filter")? } + } + other => return Err(format!("{other}() can't be explained here; try find() or aggregate()")), + }; + Ok(cmd) +} + + +/// A console statement's plan as the plan view's tree (the Postgres-shaped +/// `{Node Type, Plans, Actual Rows, ...}` it draws for every engine): the +/// executed stages with their counts and timings, aggregate stages stacked +/// above the cursor's tree. +pub async fn explain_plan(m: &MongoConn, stmt: &str) -> Result { + let db_name = m.current_db.lock().map(|g| g.clone()).unwrap_or_default(); + let Stmt::Coll { coll, calls } = parse(stmt)? else { + return Err("Explain works on db..find(...), aggregate(...), countDocuments(...) or distinct(...)".into()); + }; + let calls: Vec = calls.into_iter().filter(|c| c.name != "explain").collect(); + let (head, mods) = calls.split_first().ok_or("Missing method")?; + let args = parse_args(&head.args)?; + let (mut sort, mut limit, mut skip) = (None, None, None); + for md in mods { + let a = parse_args(&md.args)?; + match md.name.as_str() { + "sort" => sort = Some(arg_doc(&a, 0, "sort")?), + "limit" => limit = a.first().and_then(|v| v.as_i64()), + "skip" => skip = a.first().and_then(|v| v.as_u64()), + _ => {} + } + } + let cmd = explain_cmd(&coll, head, &args, sort, limit, skip)?; + let raw = m.client.database(&db_name).run_command(doc! { "explain": cmd, "verbosity": "executionStats" }).await.map_err(err)?; + Ok(plan_tree(&doc_to_json(&raw), &coll)) +} + +/// Explain output → plan tree. Pure, so it is tested on recorded output. +pub fn plan_tree(raw: &Value, coll: &str) -> Value { + fn stage_node(st: &Value, coll: &str) -> Value { + let name = st["stage"].as_str().unwrap_or("STAGE"); + let mut n = serde_json::Map::new(); + n.insert("Node Type".into(), json!(name)); + if matches!(name, "COLLSCAN" | "IXSCAN" | "FETCH" | "IDHACK" | "EXPRESS_IXSCAN" | "CLUSTERED_IXSCAN") { + n.insert("Relation Name".into(), json!(coll)); + } + if let Some(ix) = st["indexName"].as_str() { + n.insert("Index Name".into(), json!(ix)); + } + let num = |k: &str| st[k].as_i64().or_else(|| st[k].as_f64().map(|f| f as i64)).or_else(|| st[k].as_str().and_then(|s| s.parse().ok())); + if let Some(r) = num("nReturned") { + n.insert("Actual Rows".into(), json!(r)); + } + if let Some(ms) = num("executionTimeMillisEstimate") { + n.insert("Actual Total Time".into(), json!(ms)); + } + if let Some(d) = num("docsExamined") { + n.insert("Docs Examined".into(), json!(d)); + if let Some(r) = num("nReturned") { + if d > r && st.get("filter").is_some() { + n.insert("Rows Removed by Filter".into(), json!(d - r)); + } + } + } + if let Some(k) = num("keysExamined") { + n.insert("Keys Examined".into(), json!(k)); + } + if let Some(f) = st.get("filter") { + n.insert("Filter".into(), json!(f.to_string())); + } + if let Some(kp) = st.get("keyPattern") { + n.insert("Index Cond".into(), json!(kp.to_string())); + } + let mut kids: Vec = vec![]; + if let Some(c) = st.get("inputStage") { + kids.push(stage_node(c, coll)); + } + if let Some(cs) = st["inputStages"].as_array() { + kids.extend(cs.iter().map(|c| stage_node(c, coll))); + } + if !kids.is_empty() { + n.insert("Plans".into(), Value::Array(kids)); + } + Value::Object(n) + } + // aggregate: the find part sits in stages[0].$cursor, the rest above it. + let (base, later): (&Value, Vec<&Value>) = match raw["stages"].as_array() { + Some(st) if st.first().is_some_and(|s| s.get("$cursor").is_some()) => (&st[0]["$cursor"], st.iter().skip(1).collect()), + _ => (raw, vec![]), + }; + let exec = &base["executionStats"]["executionStages"]; + let mut tree = if exec.get("stage").is_some() { + stage_node(exec, coll) + } else { + let wp = &base["queryPlanner"]["winningPlan"]; + // SBE plans nest the classic tree under queryPlan. + stage_node(if wp.get("queryPlan").is_some() { &wp["queryPlan"] } else { wp }, coll) + }; + for st in later { + let Some((name, _)) = st.as_object().and_then(|o| o.iter().find(|(k, _)| k.starts_with('$'))) else { continue }; + let mut n = serde_json::Map::new(); + n.insert("Node Type".into(), json!(name)); + if let Some(r) = st["nReturned"].as_i64() { + n.insert("Actual Rows".into(), json!(r)); + } + if let Some(ms) = st["executionTimeMillisEstimate"].as_i64() { + n.insert("Actual Total Time".into(), json!(ms)); + } + n.insert("Plans".into(), json!([tree])); + tree = Value::Object(n); + } + tree +} + const CONSOLE_CAP: i64 = 5_000; +/// Fields the first batch did not have go here, as one object, so a later +/// document's extra keys are not lost from a streamed result. +pub const OTHER_FIELDS: &str = "(other fields)"; + +/// Whether `stream_into` takes this statement: a find or aggregate with no +/// limit under the console's cap (those answer in the reply as before). +pub(crate) fn streams(stmt: &str) -> bool { + let Ok(Stmt::Coll { calls, .. }) = parse(stmt) else { return false }; + let Some(head) = calls.first() else { return false }; + if head.name == "explain" || calls.iter().any(|c| c.name == "explain") { + return false; + } + if head.name != "find" && head.name != "aggregate" { + return false; + } + let limit = calls.iter().find(|c| c.name == "limit").and_then(|c| parse_args(&c.args).ok()).and_then(|a| a.first().and_then(|v| v.as_i64())); + limit.map_or(true, |l| l <= 0 || l > CONSOLE_CAP) +} + +/// A find or aggregate, streamed into `sink` from the cursor. +/// +/// The console collected at most 5,000 documents. Here the cursor's batches go +/// straight to the grid or the result store. Columns come from the first +/// 1,000 documents (`_id` first, as in the collected result); a later +/// document's keys outside them land in `(other fields)`. Stop drops the +/// cursor, which the driver kills on the server. +pub async fn stream_into( + m: &MongoConn, + stmt: &str, + sink: &mut super::query::RowSink, + stop: &super::stop::Stop, +) -> Result { + let tag = op_tag(); + let run = stream_rows(m, stmt, sink, &tag); + tokio::pin!(run); + tokio::select! { + r = &mut run => r, + _ = stop.wait() => { + drop(run); + let _ = tokio::time::timeout(std::time::Duration::from_secs(10), kill_tagged(m, &tag)).await; + Err(super::query::QUERY_CANCELLED.into()) + } + } +} + +/// A comment to tag a run's operations with, so Stop can find them. +fn op_tag() -> String { + static SEQ: std::sync::atomic::AtomicU64 = std::sync::atomic::AtomicU64::new(0); + format!("stroke-{:x}-{:x}", std::process::id(), SEQ.fetch_add(1, std::sync::atomic::Ordering::Relaxed)) +} + +/// killOp every operation carrying `tag` as its comment. Dropping the cursor +/// only kills it once a batch has come back; a find still scanning for its +/// first batch, or an aggregate's $group, runs on until then. Returns how many. +pub async fn kill_tagged(m: &MongoConn, tag: &str) -> Result { + let admin = m.client.database("admin"); + let pipeline = vec![ + doc! { "$currentOp": { "allUsers": false, "idleCursors": false } }, + doc! { "$match": { "$or": [ { "command.comment": tag }, { "cursor.originatingCommand.comment": tag } ] } }, + doc! { "$project": { "opid": 1 } }, + ]; + let ops: Vec = admin.aggregate(pipeline).await.map_err(err)?.try_collect().await.map_err(err)?; + for op in &ops { + if let Some(id) = op.get("opid") { + let _ = admin.run_command(doc! { "killOp": 1, "op": id.clone() }).await; + } + } + Ok(ops.len()) +} + +async fn stream_rows(m: &MongoConn, stmt: &str, sink: &mut super::query::RowSink, tag: &str) -> Result { + use futures::StreamExt; + const SAMPLE: usize = 1_000; + let t0 = std::time::Instant::now(); + let db_name = m.current_db.lock().map(|g| g.clone()).unwrap_or_default(); + let Stmt::Coll { coll, calls } = parse(stmt)? else { return Err("Not a find or aggregate".into()) }; + let c = m.client.database(&db_name).collection::(&coll); + let (head, mods) = calls.split_first().ok_or("Missing method")?; + let args = parse_args(&head.args)?; + let (mut sort, mut limit, mut skip) = (None, None, None); + for md in mods { + let a = parse_args(&md.args)?; + match md.name.as_str() { + "sort" => sort = Some(arg_doc(&a, 0, "sort")?), + "limit" => limit = a.first().and_then(|v| v.as_i64()).filter(|l| *l > 0), + "skip" => skip = a.first().and_then(|v| v.as_u64()), + "pretty" | "toArray" => {} + other => return Err(format!(".{other}() isn't supported here")), + } + } + let mut cursor = match head.name.as_str() { + "find" => { + let mut q = c.find(arg_doc(&args, 0, "filter")?).batch_size(1_000).comment(Bson::String(tag.to_string())); + if let Some(p) = args.get(1).filter(|v| v.is_object()) { + q = q.projection(to_doc(p, None)?); + } + if let Some(s) = sort { + q = q.sort(s); + } + if let Some(k) = skip { + q = q.skip(k); + } + if let Some(l) = limit { + q = q.limit(l); + } + q.await.map_err(err)? + } + _ => { + let pipeline = match args.first() { + Some(Value::Array(stages)) => stages.iter().map(|s| to_doc(s, None)).collect::, _>>()?, + _ => return Err("aggregate() takes an array of stages".into()), + }; + c.aggregate(pipeline).batch_size(1_000).comment(Bson::String(tag.to_string())).await.map_err(err)? + } + }; + + let mut sample: Vec = Vec::with_capacity(SAMPLE); + while sample.len() < SAMPLE { + match cursor.next().await { + Some(d) => sample.push(doc_to_json(&d.map_err(err)?)), + None => break, + } + } + let exhausted = sample.len() < SAMPLE; + if exhausted { + // Small: the collected shape, exactly as before. + return Ok(super::couchdb::docs_result(sample, None, stmt, t0)); + } + let shape = super::couchdb::docs_result(sample.clone(), None, stmt, t0); + let mut columns = shape.columns; + let names: Vec = columns.iter().map(|c| c.name.clone()).collect(); + let known: std::collections::HashSet = names.iter().cloned().collect(); + columns.push(super::query::ColumnInfo::new(OTHER_FIELDS, "object")); + sink.start(columns.clone())?; + let row_of = |d: &Value| -> Vec { + let mut row: Vec = names.iter().map(|n| super::couchdb::grid_cell(d.get(n).unwrap_or(&Value::Null)).0).collect(); + let extra: serde_json::Map = d + .as_object() + .map(|o| o.iter().filter(|(k, _)| !known.contains(*k)).map(|(k, v)| (k.clone(), v.clone())).collect()) + .unwrap_or_default(); + row.push(if extra.is_empty() { Value::Null } else { Value::Object(extra) }); + row + }; + let mut total = 0usize; + for d in &sample { + sink.push(row_of(d))?; + total += 1; + } + drop(sample); + while let Some(d) = cursor.next().await { + sink.push(row_of(&doc_to_json(&d.map_err(err)?)))?; + total += 1; + } + sink.finish()?; + Ok(super::query::SqlResult { columns, rows: vec![], row_count: Some(total as i64), message: None, query_ms: t0.elapsed().as_millis() as u64, sql: stmt.to_string() }) +} + pub async fn query(m: &MongoConn, stmt: &str) -> Result { let t0 = std::time::Instant::now(); let db_name = m.current_db.lock().map(|g| g.clone()).unwrap_or_default(); @@ -1480,37 +1778,7 @@ pub async fn query(m: &MongoConn, stmt: &str) -> Result { - let mut f = doc! { "find": coll.as_str(), "filter": arg_doc(&args, 0, "filter")? }; - if let Some(p) = args.get(1).filter(|v| v.is_object()) { - f.insert("projection", to_doc(p, None)?); - } - if let Some(s) = sort { - f.insert("sort", s); - } - if let Some(l) = limit { - f.insert("limit", l); - } - if let Some(k) = skip { - f.insert("skip", k as i64); - } - f - } - "aggregate" => { - let pipeline = match args.first() { - Some(Value::Array(stages)) => stages.iter().map(|s| to_doc(s, None)).collect::, _>>()?, - _ => return Err("aggregate() takes an array of stages".into()), - }; - doc! { "aggregate": coll.as_str(), "pipeline": pipeline, "cursor": {} } - } - "countDocuments" => doc! { "count": coll.as_str(), "query": arg_doc(&args, 0, "filter")? }, - "distinct" => { - let field = args.first().and_then(|v| v.as_str()).ok_or("distinct() takes a field name")?; - doc! { "distinct": coll.as_str(), "key": field, "query": arg_doc(&args, 1, "filter")? } - } - other => return Err(format!("{other}() can't be explained here; try find() or aggregate()")), - }; + let cmd = explain_cmd(&coll, head, &args, sort, limit, skip)?; let raw = m.client.database(&db_name).run_command(doc! { "explain": cmd, "verbosity": "executionStats" }).await.map_err(err)?; let sum = summarize_explain(&raw); let rows = sum diff --git a/src-tauri/src/db/mssql.rs b/src-tauri/src/db/mssql.rs index 4c0293cf..d1ea3bc2 100644 --- a/src-tauri/src/db/mssql.rs +++ b/src-tauri/src/db/mssql.rs @@ -41,6 +41,13 @@ pub async fn remember(handle: &MssqlHandle, cfg: &MssqlConfig) { } } +/// Drop a handle's registry entry once the handle is done with (a +/// transaction's own client). The entry holds the login, and the address it +/// is keyed by can be reused by the next handle. +pub fn forget(handle: &MssqlHandle) { + SESSIONS.lock().unwrap_or_else(|e| e.into_inner()).remove(&key(handle)); +} + /// A new client on the same server and login, for a transaction: on the /// shared client every other query would have run inside it. pub async fn open_tx_client(handle: &MssqlHandle) -> Result { diff --git a/src-tauri/src/db/oracle.rs b/src-tauri/src/db/oracle.rs index ee04d246..3810b9e1 100644 --- a/src-tauri/src/db/oracle.rs +++ b/src-tauri/src/db/oracle.rs @@ -931,6 +931,111 @@ fn run_one(conn: &Connection, sql: &str, binds: &[Bind]) -> Result bool { + reads_only(sql) && !split_statements(sql).is_empty() && split_statements(sql).len() == 1 +} + +/// A query, streamed into `sink` a batch at a time while Oracle fetches it. +/// +/// `run_one` collected every row before the grid saw one, so a large result +/// stopped at the collect budget. Here the session's thread fetches 500 rows +/// per round trip and hands them over in batches of 1,000, so memory stays +/// flat whatever the size. Stop breaks the call on the server. +pub async fn stream_into( + handle: &OracleHandle, + sql: &str, + sink: &mut super::query::RowSink, + stop: &super::stop::Stop, +) -> Result { + enum Msg { + Columns(Vec), + Rows(Vec>), + } + const BATCH: usize = 1_000; + let stmt_sql = split_statements(sql).into_iter().next().unwrap_or_default(); + let (tx, mut rx) = tokio::sync::mpsc::channel::(8); + let reader = run_blocking(handle, move |conn| { + let mut stmt = conn.statement(&stmt_sql).fetch_array_size(500).build().map_err(ora_err)?; + let rs = stmt.query(&[]).map_err(ora_err)?; + let infos = rs.column_info().to_vec(); + let columns: Vec = infos + .iter() + .map(|c| { + let mut ci = ColumnInfo::new(c.name().to_string(), type_label(c.oracle_type())); + ci.nullable = c.nullable(); + ci + }) + .collect(); + if tx.blocking_send(Msg::Columns(columns)).is_err() { + return Ok(()); + } + let mut batch = Vec::with_capacity(BATCH); + for row in rs { + let row = row.map_err(ora_err)?; + batch.push(row.sql_values().iter().zip(infos.iter()).map(|(v, c)| cell_to_json(v, c.oracle_type())).collect()); + if batch.len() >= BATCH && tx.blocking_send(Msg::Rows(std::mem::replace(&mut batch, Vec::with_capacity(BATCH)))).is_err() { + return Ok(()); + } + } + if !batch.is_empty() { + let _ = tx.blocking_send(Msg::Rows(batch)); + } + Ok(()) + }); + tokio::pin!(reader); + let t0 = Instant::now(); + let mut columns = Vec::new(); + let mut total = 0usize; + let mut sink_err: Option = None; + let mut reader_out: Option> = None; + let mut stopped = false; + loop { + tokio::select! { + msg = rx.recv() => match msg { + Some(Msg::Columns(c)) => columns = c, + Some(Msg::Rows(rows)) => { + if sink_err.is_some() || stopped { + continue; + } + for r in rows { + if total == 0 { + if let Err(e) = sink.start(columns.clone()) { sink_err = Some(e); break; } + } + if let Err(e) = sink.push(r) { sink_err = Some(e); break; } + total += 1; + } + if sink_err.is_some() { + rx.close(); + } + } + None => break, + }, + out = &mut reader, if reader_out.is_none() => reader_out = Some(out), + _ = stop.wait(), if !stopped => { + stopped = true; + rx.close(); + cancel(handle).await; + } + } + } + let out = match reader_out { + Some(o) => o, + None => reader.await, + }; + if stopped { + return Err(super::query::QUERY_CANCELLED.into()); + } + if let Some(e) = sink_err { + return Err(e); + } + out?; + if total > 0 { + sink.finish()?; + } + Ok(SqlResult { columns, rows: vec![], row_count: Some(total as i64), message: None, query_ms: t0.elapsed().as_millis() as u64, sql: sql.to_string() }) +} + /// `run_one`, committed when it wrote (a query has nothing to commit). fn run_commit(conn: &Connection, sql: &str, binds: &[Bind]) -> Result { let r = run_one(conn, sql, binds); diff --git a/src-tauri/src/db/posthog.rs b/src-tauri/src/db/posthog.rs index 02dd8844..c7609830 100644 --- a/src-tauri/src/db/posthog.rs +++ b/src-tauri/src/db/posthog.rs @@ -41,6 +41,50 @@ fn client() -> &'static reqwest::Client { /// POST one query object to the project's query endpoint. async fn post_query(config: &PosthogConfig, query: Value) -> Result { + post_query_as(config, query, None).await +} + +/// `query`, stoppable on PostHog's side: the request carries a +/// client_query_id, and Stop asks PostHog to cancel that query (DELETE on the +/// query endpoint) instead of leaving it running against the 3-at-once limit. +pub async fn query_cancellable(config: &PosthogConfig, sql: &str, stop: &super::stop::Stop) -> Result { + let trimmed = sql.trim().trim_end_matches(';'); + if !is_read_query(trimmed) { + return Err("PostHog is read-only here: HogQL runs SELECT (and WITH) queries.".into()); + } + static SEQ: std::sync::atomic::AtomicU64 = std::sync::atomic::AtomicU64::new(0); + let id = format!( + "stroke-{:x}-{:x}-{:x}", + std::process::id(), + std::time::SystemTime::now().duration_since(std::time::UNIX_EPOCH).map(|d| d.as_millis() as u64).unwrap_or(0), + SEQ.fetch_add(1, std::sync::atomic::Ordering::Relaxed) + ); + let t0 = Instant::now(); + let run = post_query_as(config, json!({ "kind": "HogQLQuery", "query": trimmed }), Some(&id)); + tokio::pin!(run); + tokio::select! { + r = &mut run => Ok(to_sql_result(&r?, sql, t0.elapsed().as_millis() as u64)), + _ = stop.wait() => { + drop(run); + let _ = tokio::time::timeout(Duration::from_secs(10), cancel_query(config, &id)).await; + Err(super::query::QUERY_CANCELLED.into()) + } + } +} + +/// DELETE /api/projects/{id}/query/{client_query_id}/: PostHog kills it in ClickHouse. +pub async fn cancel_query(config: &PosthogConfig, client_query_id: &str) -> Result<(), String> { + let url = format!( + "{}/api/projects/{}/query/{}/", + config.base_url(), + urlencoding::encode(config.project_id.trim()), + urlencoding::encode(client_query_id) + ); + let resp = client().delete(&url).bearer_auth(config.api_key.trim()).send().await.map_err(|e| format!("PostHog request failed: {e}"))?; + if resp.status().is_success() { Ok(()) } else { Err(format!("PostHog could not cancel the query (HTTP {})", resp.status().as_u16())) } +} + +async fn post_query_as(config: &PosthogConfig, query: Value, client_query_id: Option<&str>) -> Result { let url = format!( "{}/api/projects/{}/query/", config.base_url(), @@ -49,7 +93,10 @@ async fn post_query(config: &PosthogConfig, query: Value) -> Result json!({ "query": query, "name": "stroke", "client_query_id": id }), + None => json!({ "query": query, "name": "stroke" }), + }) .send() .await .map_err(|e| format!("PostHog request failed: {e}"))?; @@ -392,6 +439,42 @@ pub async fn get_ddl(config: &PosthogConfig, table: &str) -> Result(); + tokio::spawn(async move { + let mut held = Vec::new(); + loop { + let (mut s, _) = l.accept().await.unwrap(); + let mut buf = vec![0u8; 16384]; + let n = s.read(&mut buf).await.unwrap(); + let req = String::from_utf8_lossy(&buf[..n]).to_string(); + let _ = seen_tx.send(req.clone()); + if req.starts_with("DELETE") { + s.write_all(b"HTTP/1.1 204 No Content\r\nContent-Length: 0\r\n\r\n").await.unwrap(); + } else { + held.push(s); // never answered: a long query + } + } + }); + let cfg = PosthogConfig { name: "t".into(), host: format!("http://127.0.0.1:{port}"), project_id: "7".into(), api_key: "k".into() }; + let (tx, rx) = tokio::sync::oneshot::channel(); + let stop = crate::db::stop::Stop::from_rx(rx); + tokio::spawn(async move { tokio::time::sleep(Duration::from_millis(300)).await; let _ = tx.send(()); }); + let e = query_cancellable(&cfg, "SELECT count() FROM events", &stop).await.unwrap_err(); + assert_eq!(e, crate::db::query::QUERY_CANCELLED); + let post = seen_rx.recv().await.unwrap(); + assert!(post.starts_with("POST /api/projects/7/query/"), "{post}"); + let id = post.split("\"client_query_id\":\"").nth(1).and_then(|r| r.split('"').next()).expect("client_query_id sent").to_string(); + let del = seen_rx.recv().await.unwrap(); + assert!(del.starts_with(&format!("DELETE /api/projects/7/query/{id}/")), "{del}"); + } + #[test] fn hogql_response_maps_columns_types_and_rows() { let body = json!({ diff --git a/src-tauri/src/db/query.rs b/src-tauri/src/db/query.rs index fd399066..cb9ab777 100644 --- a/src-tauri/src/db/query.rs +++ b/src-tauri/src/db/query.rs @@ -4628,6 +4628,9 @@ pub(crate) async fn run_one_cancellable( } } } + // Cancel by client_query_id / X-Opaque-Id on the server. + ActiveConnection::Posthog(cfg) => super::posthog::query_cancellable(cfg, stmt, stop).await, + ActiveConnection::Doc(super::docdb::DocConn::Elastic(cfg)) => super::elastic::query_cancellable(cfg, stmt, stop).await, // No server-side stop exists: stop waiting, and say the server may not. other => tokio::select! { r = run_one_other(other, stmt) => r, @@ -4671,6 +4674,8 @@ async fn stream_one( } } } + ActiveConnection::Doc(super::docdb::DocConn::Mongo(m)) if super::mongo::streams(stmt) => super::mongo::stream_into(m, stmt, sink, stop).await, + ActiveConnection::Oracle(h) if super::oracle::reads(stmt) => super::oracle::stream_into(h, stmt, sink, stop).await, ActiveConnection::Clickhouse(cfg) if super::clickhouse::reads(stmt) => super::clickhouse::stream_into(cfg, stmt, sink, stop).await, _ => return None, }) @@ -6250,6 +6255,203 @@ mod stream_tests { check(&store, "ch-big", r, before, 120).await; } + // STROKE_TEST_CH=18123:password cargo test --lib stream_tests::clickhouse_live + fn live_clickhouse() -> Option { + let spec = std::env::var("STROKE_TEST_CH").ok()?; + let (port, pw) = spec.split_once(':')?; + let mut cfg = fake_clickhouse_cfg(port.parse().ok()?); + cfg.password = pw.into(); + Some(cfg) + } + + #[tokio::test] + async fn clickhouse_live_streams_and_stops() { + let Some(cfg) = live_clickhouse() else { return }; + let store = super::super::result_store::ResultStore::default(); + let mut sink = sink_into(&store, "ch-live"); + let before = rss_mb(); + let (_tx, rx) = tokio::sync::oneshot::channel(); + let stop = super::super::stop::Stop::from_rx(rx); + let sql = format!("SELECT number + 1 AS id, hex(MD5(toString(number))) AS v FROM numbers({N})"); + let r = super::super::clickhouse::stream_into(&cfg, &sql, &mut sink, &stop).await.unwrap(); + assert_eq!(r.columns[0].name, "id"); + assert_eq!(r.row_count, Some(N as i64)); + let stored = store.get("ch-live").unwrap(); + assert_eq!(stored.len(), N); + // UInt64 arrives quoted (output_format_json_quote_64bit_integers), as + // it did collected: values past 2^53 stay exact in the grid. + let deep: Vec = serde_json::from_str(&stored.read_window(N - 1, 1).unwrap()).unwrap(); + assert_eq!(deep[0][0], serde_json::json!(N.to_string())); + let grew = rss_mb().saturating_sub(before); + assert!(grew < 120, "memory grew {grew} MB while streaming"); + + // Stop: the server must not keep running the query. + let mut sink = sink_into(&store, "ch-stop"); + let (tx, rx) = tokio::sync::oneshot::channel(); + let stop = super::super::stop::Stop::from_rx(rx); + tokio::spawn(async move { tokio::time::sleep(std::time::Duration::from_millis(800)).await; let _ = tx.send(()); }); + let marker = "stroke_live_stop_marker"; + let long = format!("SELECT count() AS {marker} FROM numbers(100000000000) WHERE sipHash64(number) = 1"); + let t0 = std::time::Instant::now(); + let e = super::super::clickhouse::stream_into(&cfg, &long, &mut sink, &stop).await.unwrap_err(); + assert_eq!(e, QUERY_CANCELLED); + assert!(t0.elapsed() < std::time::Duration::from_secs(15)); + let left = super::super::clickhouse::query(&cfg, &format!("SELECT count() FROM system.processes WHERE query LIKE '%{marker}%' AND query NOT LIKE '%system.processes%'")).await.unwrap(); + assert_eq!(left.rows[0][0].as_str().map(|s| s.to_string()).unwrap_or(left.rows[0][0].to_string()), "0", "query still running on the server"); + } + + // STROKE_TEST_ORACLE=user/pw@//host:1521/FREEPDB1 STROKE_ORACLE_CLIENT_DIR=... cargo test --lib stream_tests::oracle_live + #[tokio::test] + async fn oracle_live_streams_and_stops() { + let Ok(url) = std::env::var("STROKE_TEST_ORACLE") else { return }; + let (creds, cs) = url.split_once('@').unwrap(); + let (user, password) = creds.split_once('/').unwrap(); + let cfg = super::super::connection::OracleConfig { + ssh: None, name: "live".into(), host: String::new(), port: 1521, service_name: String::new(), sid: String::new(), + connect_string: cs.into(), user: user.into(), password: password.into(), + client_lib_dir: std::env::var("STROKE_ORACLE_CLIENT_DIR").unwrap_or_default(), + }; + super::super::oracle::prepare_client(&cfg.client_lib_dir).await.unwrap(); + let h = super::super::oracle::connect(&cfg).await.unwrap(); + let store = super::super::result_store::ResultStore::default(); + let mut sink = sink_into(&store, "ora-big"); + let before = rss_mb(); + let (_tx, rx) = tokio::sync::oneshot::channel(); + let stop = super::super::stop::Stop::from_rx(rx); + let sql = format!("SELECT a.n * 1000 + b.n + 1 AS id, RPAD('v', 64, 'x') AS v FROM (SELECT level - 1 AS n FROM dual CONNECT BY level <= {}) a CROSS JOIN (SELECT level - 1 AS n FROM dual CONNECT BY level <= 1000) b ORDER BY 1", N / 1000); + let r = super::super::oracle::stream_into(&h, &sql, &mut sink, &stop).await.unwrap(); + assert_eq!(r.columns[0].name, "ID"); + assert_eq!(r.row_count, Some(N as i64)); + let stored = store.get("ora-big").unwrap(); + assert_eq!(stored.len(), N); + let grew = rss_mb().saturating_sub(before); + assert!(grew < 120, "memory grew {grew} MB while streaming"); + + // Stop mid-stream: the call breaks and the session stays usable. + let mut sink = sink_into(&store, "ora-stop"); + let (tx, rx) = tokio::sync::oneshot::channel(); + let stop = super::super::stop::Stop::from_rx(rx); + tokio::spawn(async move { tokio::time::sleep(std::time::Duration::from_millis(700)).await; let _ = tx.send(()); }); + let slow = "SELECT COUNT(*) FROM (SELECT level n FROM dual CONNECT BY level <= 3000) a, (SELECT level n FROM dual CONNECT BY level <= 3000) b, (SELECT level n FROM dual CONNECT BY level <= 3000) c"; + let t0 = std::time::Instant::now(); + let e = super::super::oracle::stream_into(&h, slow, &mut sink, &stop).await.unwrap_err(); + assert_eq!(e, QUERY_CANCELLED); + assert!(t0.elapsed() < std::time::Duration::from_secs(10)); + super::super::oracle::ping(&h).await.unwrap(); + } + + // STROKE_TEST_MONGO=mongodb://127.0.0.1:27018 cargo test --lib stream_tests::mongo_live + #[tokio::test] + async fn mongo_live_streams_past_the_console_cap() { + let Ok(uri) = std::env::var("STROKE_TEST_MONGO") else { return }; + use mongodb::bson::doc; + const M: usize = 400_000; + let m = super::super::mongo::connect(&super::super::mongo::MongoConfig { ssh: None, name: "t".into(), uri, database: Some("stroke_stream".into()) }).await.unwrap(); + let c = m.client.database("stroke_stream").collection::("big"); + let _ = c.drop().await; + let mut batch = Vec::with_capacity(10_000); + for i in 0..M { + let mut d = doc! { "_id": i as i64 + 1, "n": i as i64, "v": format!("{:064x}", i) }; + // A field the first 1,000 documents never had. + if i == 250_000 { d.insert("late", "here"); } + batch.push(d); + if batch.len() == 10_000 { c.insert_many(std::mem::take(&mut batch)).await.unwrap(); } + } + let store = super::super::result_store::ResultStore::default(); + let mut sink = sink_into(&store, "mongo-big"); + let before = rss_mb(); + let (_tx, rx) = tokio::sync::oneshot::channel(); + let stop = super::super::stop::Stop::from_rx(rx); + let stmt = "db.big.find({}).sort({_id: 1})"; + assert!(super::super::mongo::streams(stmt)); + assert!(!super::super::mongo::streams("db.big.find({}).limit(10)")); + let r = super::super::mongo::stream_into(&m, stmt, &mut sink, &stop).await.unwrap(); + assert_eq!(r.row_count, Some(M as i64), "past the 5,000-document cap"); + assert_eq!(r.columns.last().unwrap().name, super::super::mongo::OTHER_FIELDS); + let stored = store.get("mongo-big").unwrap(); + assert_eq!(stored.len(), M); + let late: Vec> = serde_json::from_str(&stored.read_window(250_000, 1).unwrap()).unwrap(); + assert_eq!(late[0].last().unwrap(), &serde_json::json!({ "late": "here" })); + let grew = rss_mb().saturating_sub(before); + assert!(grew < 120, "memory grew {grew} MB while streaming"); + // Stop while the server is still working: the tagged operation is killed. + let slow = "db.big.aggregate([{$group: {_id: {$mod: [\"$n\", 7]}, s: {$sum: {$function: {body: 'function(n){var x=0; for (var i=0;i<20000;i++){x+=i;} return x;}', args: [\"$n\"], lang: 'js'}}}}}])"; + assert!(super::super::mongo::streams(slow)); + let mut sink = sink_into(&store, "mongo-stop"); + let (tx, rx) = tokio::sync::oneshot::channel(); + let stop = super::super::stop::Stop::from_rx(rx); + tokio::spawn(async move { tokio::time::sleep(std::time::Duration::from_millis(1500)).await; let _ = tx.send(()); }); + let t0 = std::time::Instant::now(); + let e = super::super::mongo::stream_into(&m, slow, &mut sink, &stop).await.unwrap_err(); + assert_eq!(e, QUERY_CANCELLED); + assert!(t0.elapsed() < std::time::Duration::from_secs(15)); + tokio::time::sleep(std::time::Duration::from_millis(1000)).await; + let admin = m.client.database("admin"); + let left: Vec = futures::TryStreamExt::try_collect(admin.aggregate(vec![ + doc! { "$currentOp": {} }, + doc! { "$match": { "command.aggregate": "big" } }, + ]).await.unwrap()).await.unwrap(); + assert!(left.is_empty(), "the aggregate still runs on the server: {left:?}"); + let _ = c.drop().await; + } + + /// Explain on the document engines, against real servers: + /// STROKE_TEST_MONGO=mongodb://127.0.0.1:27018 STROKE_TEST_ES=http://127.0.0.1:19200 + /// STROKE_TEST_COUCH=http://admin:admin@127.0.0.1:15984 cargo test --lib live_doc_plans + #[tokio::test] + async fn live_doc_plans() { + fn types(n: &Value, out: &mut Vec) { + out.push(n["Node Type"].as_str().unwrap_or("").to_string()); + for k in n["Plans"].as_array().into_iter().flatten() { + types(k, out); + } + } + if let Ok(uri) = std::env::var("STROKE_TEST_MONGO") { + use mongodb::bson::doc; + let m = super::super::mongo::connect(&super::super::mongo::MongoConfig { ssh: None, name: "t".into(), uri, database: Some("stroke_plan".into()) }).await.unwrap(); + let c = m.client.database("stroke_plan").collection::("t"); + let _ = c.drop().await; + c.insert_many((0..2000).map(|i| doc! { "n": i, "k": i % 10 })).await.unwrap(); + c.create_index(mongodb::IndexModel::builder().keys(doc! { "n": 1 }).build()).await.unwrap(); + let p = super::super::mongo::explain_plan(&m, "db.t.find({n: {$gt: 1500}}).sort({n: 1})").await.unwrap(); + let mut t = vec![]; + types(&p, &mut t); + assert!(t.iter().any(|s| s == "IXSCAN"), "{p}"); + assert!(p.to_string().contains("n_1"), "{p}"); + let p = super::super::mongo::explain_plan(&m, "db.t.find({k: 3})").await.unwrap(); + assert!(p.to_string().contains("COLLSCAN"), "{p}"); + assert!(p.to_string().contains("Rows Removed by Filter"), "{p}"); + let p = super::super::mongo::explain_plan(&m, "db.t.aggregate([{$match: {k: 3}}, {$group: {_id: null, s: {$sum: \"$n\"}}}])").await.unwrap(); + assert!(p["Node Type"].is_string(), "{p}"); + println!("mongo aggregate plan: {p}"); + let _ = c.drop().await; + } + if let Ok(url) = std::env::var("STROKE_TEST_ES") { + let cfg = super::super::elastic::EsConfig { ssh: None, via: None, name: "t".into(), url, user: None, password: None, api_key: None, flavor: None, insecure: true }; + let cfg = super::super::elastic::connect(cfg).await.unwrap(); + super::super::elastic::request(&cfg, "PUT", "stroke_plan/_doc/1?refresh=wait_for", Some(&serde_json::json!({"level": "error", "ms": 12}))).await.unwrap(); + let p = super::super::elastic::explain_plan(&cfg, r#"stroke_plan.search({"query": {"bool": {"must": [{"term": {"level.keyword": "error"}}], "filter": [{"range": {"ms": {"gte": 5}}}]}}})"#).await.unwrap(); + let mut t = vec![]; + types(&p, &mut t); + assert_eq!(t[0], "Search"); + assert!(t.iter().any(|s| s == "Shard"), "{p}"); + assert!(t.iter().any(|s| s == "BooleanQuery"), "{p}"); + assert_eq!(p["Actual Rows"], serde_json::json!(1)); + let p = super::super::elastic::explain_plan(&cfg, "GET stroke_plan/_search\n{\"query\": {\"match_all\": {}}}").await.unwrap(); + assert!(p.to_string().contains("MatchAllDocsQuery"), "{p}"); + let _ = super::super::elastic::request(&cfg, "DELETE", "stroke_plan", None).await; + } + if let Ok(url) = std::env::var("STROKE_TEST_COUCH") { + let cfg = super::super::couchdb::CouchConfig { ssh: None, via: None, name: "t".into(), url, user: Some("admin".into()), password: Some("admin".into()) }; + // Set up outside (creating databases is refused here): stroke_plan + // with a "by-level" index on level. + let p = super::super::couchdb::explain_plan(&cfg, r#"stroke_plan.find({"other": 1})"#).await.unwrap(); + assert!(p.to_string().contains("All documents scan"), "{p}"); + let p = super::super::couchdb::explain_plan(&cfg, r#"stroke_plan.find({"level": "error"})"#).await.unwrap(); + assert!(p.to_string().contains("by-level"), "{p}"); + } + } + #[tokio::test] async fn clickhouse_error_after_the_first_rows_is_reported() { let port = fake_clickhouse(500, "Code: 241. DB::Exception: Memory limit exceeded\n").await; @@ -6271,12 +6473,26 @@ mod stream_tests { let sql = format!("SELECT range + 1 AS id, md5(range::varchar) || md5((range * 7)::varchar) AS v FROM range({N})"); let (_tx, rx) = tokio::sync::oneshot::channel(); let r = super::super::duckdb::stream_into(&h, &sql, &mut sink, rx).await.unwrap(); - // duckdb-rs reads rows only from a materialized result (its streaming - // cursor is Arrow-only, and panics on a mid-stream error), so DuckDB - // holds the result natively: measured 214 MB here. This side holds one - // 1,000-row batch. Collected, the same result grew memory 931 MB - // (duckdb_collected_for_comparison), before the window's copy. - check(&store, "duck-big", r, before, 350).await; + // DuckDB executes in streaming mode (duckdb::for_each_row_streaming), + // so neither side holds the result: this side keeps one 1,000-row + // batch. Materialized, DuckDB alone held 214 MB of it; collected, the + // same result grew memory 931 MB (duckdb_collected_for_comparison). + check(&store, "duck-big", r, before, 120).await; + } + + /// An error raised after rows have streamed comes back as an error. The + /// driver's own streaming iterator panics there, which with panic=abort + /// would close the app. + #[tokio::test] + async fn duckdb_mid_stream_error_is_an_error() { + let cfg = super::super::connection::DuckdbConfig { name: "t".into(), file_path: ":memory:".into() }; + let h = super::super::connection::open_duckdb(&cfg).await.unwrap(); + let store = super::super::result_store::ResultStore::default(); + let mut sink = sink_into(&store, "duck-err"); + let (_tx, rx) = tokio::sync::oneshot::channel(); + let sql = "SELECT CASE WHEN range = 250000 THEN error('boom at 250000') ELSE range END AS v FROM range(400000)"; + let e = super::super::duckdb::stream_into(&h, sql, &mut sink, rx).await.unwrap_err(); + assert!(e.contains("boom at 250000"), "{e}"); } // For the record: what the same DuckDB result costs collected. diff --git a/src-tauri/src/db/tx.rs b/src-tauri/src/db/tx.rs index 2deced08..005ab560 100644 --- a/src-tauri/src/db/tx.rs +++ b/src-tauri/src/db/tx.rs @@ -77,7 +77,10 @@ impl TxConn { // Their own connection: dropping it closes it, and the server // rolls back what it held. TxConn::Duckdb(h) => drop(h), - TxConn::Mssql(h) => drop(h), + TxConn::Mssql(h) => { + super::mssql::forget(&h); + drop(h) + } TxConn::Oracle(h) => drop(h), } } @@ -686,6 +689,111 @@ async fn fetch_rows(conn: &mut TxConn, stmt: &str) -> Result mod tests { use super::leading_keyword; + fn live_session(conn: super::TxConn) -> super::TxSession { + super::TxSession { + conn, + epoch: 0, + blocker: None, + statements: 0, + rows_affected: 0, + started: std::time::Instant::now(), + last_used: std::time::Instant::now(), + } + } + + fn count_of(r: &super::SqlResult) -> i64 { + let v = &r.rows[0][0]; + v.as_i64().or_else(|| v.as_str().and_then(|s| s.parse().ok())).or_else(|| v.as_f64().map(|f| f as i64)).unwrap_or(-1) + } + + /// One engine's transaction, end to end through tx.rs: BEGIN, a write + /// that only the transaction sees, ROLLBACK; then BEGIN, write, COMMIT; + /// then a transaction abandoned mid-way, which must roll back on release. + /// `outside` reads through the shared handle. + async fn tx_round_trip(open: impl Fn() -> Fut, outside: F, table: &str) + where + Fut: std::future::Future, + F: Fn(String) -> std::pin::Pin>>, + { + use super::{execute_in, release, run_statement}; + let count = format!("SELECT COUNT(*) FROM {table}"); + let insert = format!("INSERT INTO {table} (v) VALUES (1)"); + + let mut c = open().await; + run_statement(&mut c, "BEGIN").await.unwrap(); + let w = execute_in(&mut c, &insert).await.unwrap(); + assert!(w.message.unwrap_or_default().contains("not committed")); + assert_eq!(count_of(&execute_in(&mut c, &count).await.unwrap()), 1, "the transaction sees its own row"); + run_statement(&mut c, "ROLLBACK").await.unwrap(); + assert_eq!(outside(count.clone()).await, 0, "rolled back"); + + run_statement(&mut c, "BEGIN").await.unwrap(); + execute_in(&mut c, &insert).await.unwrap(); + run_statement(&mut c, "COMMIT").await.unwrap(); + assert_eq!(outside(count.clone()).await, 1, "committed"); + + let mut c2 = open().await; + run_statement(&mut c2, "BEGIN").await.unwrap(); + execute_in(&mut c2, &insert).await.unwrap(); + release(live_session(c2)).await; + drop(c); + assert_eq!(outside(count.clone()).await, 1, "abandoned transaction rolled back"); + } + + // STROKE_TEST_MSSQL=127.0.0.1:11433:sa:password cargo test --lib tx::tests::mssql_live + #[tokio::test] + async fn mssql_live_transaction() { + let Ok(spec) = std::env::var("STROKE_TEST_MSSQL") else { return }; + let p: Vec<&str> = spec.splitn(4, ':').collect(); + let cfg = super::super::connection::MssqlConfig { + name: "live".into(), host: p[0].into(), port: p[1].parse().unwrap(), database: "master".into(), + user: p[2].into(), password: p[3].into(), encrypt: false, trust_cert: true, ssh: None, via: None, + }; + let shared = super::super::connection::open_mssql(&cfg).await.unwrap(); + super::super::mssql::batch(&shared, "IF OBJECT_ID('stroke_tx_live') IS NOT NULL DROP TABLE stroke_tx_live; CREATE TABLE stroke_tx_live (v int)").await.unwrap(); + let s2 = shared.clone(); + tx_round_trip( + || { let s = shared.clone(); async move { super::TxConn::Mssql(super::super::mssql::open_tx_client(&s).await.unwrap()) } }, + move |sql| { let s = s2.clone(); Box::pin(async move { count_of(&super::super::mssql::execute_sql(&s, &sql).await.unwrap()) }) }, + "stroke_tx_live", + ).await; + } + + // STROKE_TEST_ORACLE=app/pw@//127.0.0.1:11521/FREEPDB1 STROKE_ORACLE_CLIENT_DIR=... cargo test --lib tx::tests::oracle_live + #[tokio::test] + async fn oracle_live_transaction() { + let Ok(url) = std::env::var("STROKE_TEST_ORACLE") else { return }; + let (creds, cs) = url.split_once('@').unwrap(); + let (user, password) = creds.split_once('/').unwrap(); + let cfg = super::super::connection::OracleConfig { + ssh: None, name: "live".into(), host: String::new(), port: 1521, service_name: String::new(), sid: String::new(), + connect_string: cs.into(), user: user.into(), password: password.into(), + client_lib_dir: std::env::var("STROKE_ORACLE_CLIENT_DIR").unwrap_or_default(), + }; + super::super::oracle::prepare_client(&cfg.client_lib_dir).await.unwrap(); + let shared = super::super::oracle::connect(&cfg).await.unwrap(); + let _ = super::super::oracle::execute_sql(&shared, "DROP TABLE stroke_tx_live").await; + super::super::oracle::execute_sql(&shared, "CREATE TABLE stroke_tx_live (v NUMBER)").await.unwrap(); + let s2 = shared.clone(); + tx_round_trip( + || { let s = shared.clone(); async move { super::TxConn::Oracle(super::super::oracle::open_tx_session(&s).await.unwrap()) } }, + move |sql| { let s = s2.clone(); Box::pin(async move { count_of(&super::super::oracle::execute_sql(&s, &sql).await.unwrap()) }) }, + "stroke_tx_live", + ).await; + } + + #[tokio::test] + async fn duckdb_transaction_round_trip() { + let shared: super::super::connection::DuckdbHandle = std::sync::Arc::new(std::sync::Mutex::new(::duckdb::Connection::open_in_memory().unwrap())); + super::super::duckdb::batch(&shared, "CREATE TABLE stroke_tx_live (v INTEGER)").await.unwrap(); + let s2 = shared.clone(); + tx_round_trip( + || { let s = shared.clone(); async move { super::TxConn::Duckdb(super::super::duckdb::open_tx_conn(&s).await.unwrap()) } }, + move |sql| { let s = s2.clone(); Box::pin(async move { count_of(&super::super::duckdb::execute_sql(&s, &sql).await.unwrap()) }) }, + "stroke_tx_live", + ).await; + } + // Live: a released session must give back a connection that is out of its // transaction. Before `release`, the idle sweep dropped the PoolConnection, // sqlx returned it to the pool still inside the BEGIN, and the next user of diff --git a/src/lib/components/AiModelPicker.svelte b/src/lib/components/AiModelPicker.svelte index 3ff93fee..64d7c1bd 100644 --- a/src/lib/components/AiModelPicker.svelte +++ b/src/lib/components/AiModelPicker.svelte @@ -320,7 +320,7 @@
-