diff --git a/Cargo.lock b/Cargo.lock index 7c7355d..6344379 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -409,8 +409,8 @@ checksum = "88904434abc2901f197fe8cc55f0445e7ded921dba5911dad2e2b39b48e663c4" [[package]] name = "minigraf" -version = "2.0.3" -source = "git+https://github.com/project-minigraf/minigraf?rev=97cd4a2a85301ea3478250dd654139d45a1f7cd0#97cd4a2a85301ea3478250dd654139d45a1f7cd0" +version = "2.0.5" +source = "git+https://github.com/project-minigraf/minigraf?rev=df7282f1f3ad17628d33fa5488730201313ac268#df7282f1f3ad17628d33fa5488730201313ac268" dependencies = [ "anyhow", "chrono", @@ -921,7 +921,6 @@ version = "1.23.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bf80a72845275afea99e7f2b434723d3bc7e38470fcd1c7ed39a599c73319a53" dependencies = [ - "getrandom", "js-sys", "serde_core", "sha1_smol", diff --git a/Cargo.toml b/Cargo.toml index 363584d..6b71176 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -15,7 +15,7 @@ path = "src/uniffi_bindgen.rs" [dependencies] # Unreleased v3 core (format v8). Switches to minigraf = "=3.0.0" at the v3.0.0 release. -minigraf = { git = "https://github.com/project-minigraf/minigraf", rev = "97cd4a2a85301ea3478250dd654139d45a1f7cd0" } +minigraf = { git = "https://github.com/project-minigraf/minigraf", rev = "df7282f1f3ad17628d33fa5488730201313ac268" } uniffi = { version = "0.31.1", features = ["cli"] } thiserror = "2.0.18" serde_json = "1.0.149" diff --git a/README.md b/README.md index 1a3cd58..75255d2 100644 --- a/README.md +++ b/README.md @@ -43,10 +43,17 @@ import minigraf src = minigraf.open("app.graph", read_only=True, page_cache_size=4096) # Every option: read_only, page_cache_size, allow_unlocked, wal_checkpoint_threshold, -# max_derived_facts, max_results, synchronous (minigraf.SyncMode.FULL / NORMAL). -db = minigraf.MiniGrafDb.open_with_options("app.graph", minigraf.OpenOptions(max_results=10_000)) +# max_derived_facts, max_results, synchronous (minigraf.SyncMode.FULL / NORMAL), +# query_memory_limit. +db = minigraf.MiniGrafDb.open_with_options("app.graph", minigraf.OpenOptions(max_derived_facts=10_000)) ``` +`query_memory_limit` is the most bytes one query may hold, counted per entry by the +operators that keep state (join build sides, negation and `or` tables, distinct sets, +aggregate groups, window rows, rule-derived facts). A query that would hold more raises +`MiniGrafError` with QRY-012 and frees what it held. The default, and `2**63 - 1`, is no +limit. Rows a cursor returns are not counted, so read a large answer with `db.query()`. + `wal_checkpoint_threshold=minigraf.WAL_CHECKPOINT_NEVER` turns off automatic checkpoints and the checkpoint when the handle closes, for applications that call `checkpoint()` on their own schedule. diff --git a/minigraf/__init__.py b/minigraf/__init__.py index 2cd2e81..9b7bc75 100644 --- a/minigraf/__init__.py +++ b/minigraf/__init__.py @@ -38,12 +38,14 @@ def open( max_derived_facts=None, max_results=None, synchronous=None, + query_memory_limit=None, ): """Open a file-backed database. An option left as ``None`` keeps its default. ``minigraf.open("app.graph", read_only=True)`` opens the file read-only: any number of read-only handles can share it, and writes raise - ``MiniGrafError`` with code API-014. + ``MiniGrafError`` with code API-014. ``query_memory_limit`` caps the bytes + one query may hold; a query that would hold more raises QRY-012. """ return MiniGrafDb.open_with_options( path, @@ -55,6 +57,7 @@ def open( max_derived_facts=max_derived_facts, max_results=max_results, synchronous=synchronous, + query_memory_limit=query_memory_limit, ), ) diff --git a/src/lib.rs b/src/lib.rs index 8e557dd..944f09d 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -126,12 +126,17 @@ pub struct OpenOptions { /// Facts a recursive rule may derive per iteration. Default 1,000,000. #[uniffi(default)] pub max_derived_facts: Option, - /// Rows a query may return. Default 1,000,000. + /// Deprecated alias of `max_derived_facts`; the lower of the two applies. #[uniffi(default)] pub max_results: Option, /// WAL durability. Default `Full`. #[uniffi(default)] pub synchronous: Option, + /// The most bytes one query may hold; a query that would hold more fails + /// with QRY-012. The maximum (i64::MAX, 9223372036854775807) means no + /// limit, the default. + #[uniffi(default)] + pub query_memory_limit: Option, } impl OpenOptions { @@ -160,7 +165,10 @@ impl OpenOptions { o = o.max_derived_facts(to_usize(v, "max_derived_facts")?); } if let Some(v) = self.max_results { - o = o.max_results(to_usize(v, "max_results")?); + #[allow(deprecated)] + { + o = o.max_results(to_usize(v, "max_results")?); + } } if let Some(v) = self.synchronous { o = o.synchronous(match v { @@ -168,6 +176,16 @@ impl OpenOptions { SyncMode::Normal => minigraf::SyncMode::Normal, }); } + if let Some(v) = self.query_memory_limit { + // i64::MAX stands for core's usize::MAX (no limit), as for + // wal_checkpoint_threshold. + let n = if v == i64::MAX { + usize::MAX + } else { + to_usize(v, "query_memory_limit")? + }; + o = o.query_memory_limit(n); + } Ok(o) } } @@ -717,7 +735,7 @@ mod tests { #[test] fn value_to_json_ref() { - let id = minigraf::EntityId::new_v4(); + let id = minigraf::EntityId::from_u128(0x6f1c_2a3b_4d5e_4f60_8172_839a_b4c5_d6e7); let j = value_to_json(&Value::Ref(id)); assert_eq!(j, serde_json::Value::String(id.to_string())); } @@ -832,7 +850,7 @@ mod tests { #[test] fn value_round_trips_through_core() { - let id = minigraf::EntityId::new_v4(); + let id = minigraf::EntityId::from_u128(0x6f1c_2a3b_4d5e_4f60_8172_839a_b4c5_d6e7); for v in [ Value::String("s".into()), Value::Integer(-3), @@ -914,6 +932,40 @@ mod tests { let _ = std::fs::remove_dir_all(&dir); } + #[test] + fn query_memory_limit_is_qry_012() { + let options = |limit| OpenOptions { + query_memory_limit: Some(limit), + ..OpenOptions::default() + }; + assert!(msg(err(options(-1).to_core())).starts_with("[API-017]")); + + let dir = std::env::temp_dir().join(format!("minigraf_ffi_qml_{}", std::process::id())); + let _ = std::fs::remove_dir_all(&dir); + std::fs::create_dir_all(&dir).expect("dir"); + // A cartesian product holds one side in full. + let product = "(query [:find ?a ?b :where [?a :n _] [?b :m _]])"; + for (name, limit, fails) in [("small", 64, true), ("max", i64::MAX, false)] { + let path = dir.join(format!("{name}.graph")); + let db = + MiniGrafDb::open_with_options(path.to_str().expect("utf8").into(), options(limit)) + .expect("open"); + db.execute("(transact [[:a :n 1] [:b :m 2]])".into()) + .expect("transact"); + let r = db.execute(product.into()); + if fails { + let e = err(r); + assert!(matches!(e, MiniGrafError::Query { .. }), "a query error"); + assert!(msg(e).starts_with("[QRY-012]")); + let cursor = db.query(product.into()).expect("query"); + assert!(msg(err(cursor.next_batch(10))).starts_with("[QRY-012]")); + } else { + r.expect("no limit"); + } + } + let _ = std::fs::remove_dir_all(&dir); + } + #[test] fn cursor_batches_and_closes() { let db = MiniGrafDb::open_in_memory().expect("open"); diff --git a/tests/test_basic.py b/tests/test_basic.py index 1991079..dcdb680 100644 --- a/tests/test_basic.py +++ b/tests/test_basic.py @@ -1,6 +1,7 @@ import json import os import pytest +import minigraf from minigraf import MiniGrafDb, MiniGrafError @@ -36,3 +37,18 @@ def test_file_backed_roundtrip(tmp_path): db2 = MiniGrafDb.open(path) result = json.loads(db2.execute("(query [:find ?n :where [?e :name ?n]])")) assert result["results"][0][0] == "Bob" + + +def test_query_memory_limit_raises_qry_012(tmp_path): + """#549: a query that would hold more than query_memory_limit bytes fails.""" + product = "(query [:find ?a ?b :where [?a :n _] [?b :m _]])" + + db = minigraf.open(str(tmp_path / "small.graph"), query_memory_limit=64) + db.execute("(transact [[:a :n 1] [:b :m 2]])") + with pytest.raises(MiniGrafError.Query) as e: + db.execute(product) + assert e.value.msg.startswith("[QRY-012]") + + db = minigraf.open(str(tmp_path / "max.graph"), query_memory_limit=2**63 - 1) + db.execute("(transact [[:a :n 1] [:b :m 2]])") + assert len(json.loads(db.execute(product))["results"]) == 1