feat: query_memory_limit and QRY-012 (minigraf#549)

Cargo.toml

@@ -15,7 +15,7 @@ path = "src/uniffi_bindgen.rs"
 
 [dependencies]
 # Unreleased v3 core (format v8). Switches to minigraf = "=3.0.0" at the v3.0.0 release.
-minigraf = { git = "https://github.com/project-minigraf/minigraf", rev = "97cd4a2a85301ea3478250dd654139d45a1f7cd0" }
+minigraf = { git = "https://github.com/project-minigraf/minigraf", rev = "df7282f1f3ad17628d33fa5488730201313ac268" }
 uniffi = { version = "0.31.1", features = ["cli"] }
 thiserror = "2.0.18"
 serde_json = "1.0.149"

README.md

@@ -43,10 +43,17 @@ import minigraf
 src = minigraf.open("app.graph", read_only=True, page_cache_size=4096)
 
 # Every option: read_only, page_cache_size, allow_unlocked, wal_checkpoint_threshold,
-# max_derived_facts, max_results, synchronous (minigraf.SyncMode.FULL / NORMAL).
-db = minigraf.MiniGrafDb.open_with_options("app.graph", minigraf.OpenOptions(max_results=10_000))
+# max_derived_facts, max_results, synchronous (minigraf.SyncMode.FULL / NORMAL),
+# query_memory_limit.
+db = minigraf.MiniGrafDb.open_with_options("app.graph", minigraf.OpenOptions(max_derived_facts=10_000))
 ```
 
+`query_memory_limit` is the most bytes one query may hold, counted per entry by the
+operators that keep state (join build sides, negation and `or` tables, distinct sets,
+aggregate groups, window rows, rule-derived facts). A query that would hold more raises
+`MiniGrafError` with QRY-012 and frees what it held. The default, and `2**63 - 1`, is no
+limit. Rows a cursor returns are not counted, so read a large answer with `db.query()`.
+
 `wal_checkpoint_threshold=minigraf.WAL_CHECKPOINT_NEVER` turns off automatic checkpoints
 and the checkpoint when the handle closes, for applications that call `checkpoint()` on
 their own schedule.

minigraf/__init__.py

@@ -38,12 +38,14 @@ def open(
     max_derived_facts=None,
     max_results=None,
     synchronous=None,
+    query_memory_limit=None,
 ):
     """Open a file-backed database. An option left as ``None`` keeps its default.
 
     ``minigraf.open("app.graph", read_only=True)`` opens the file read-only:
     any number of read-only handles can share it, and writes raise
-    ``MiniGrafError`` with code API-014.
+    ``MiniGrafError`` with code API-014. ``query_memory_limit`` caps the bytes
+    one query may hold; a query that would hold more raises QRY-012.
     """
     return MiniGrafDb.open_with_options(
         path,
@@ -55,6 +57,7 @@ def open(
             max_derived_facts=max_derived_facts,
             max_results=max_results,
             synchronous=synchronous,
+            query_memory_limit=query_memory_limit,
         ),
     )
 

src/lib.rs

@@ -126,12 +126,17 @@ pub struct OpenOptions {
     /// Facts a recursive rule may derive per iteration. Default 1,000,000.
     #[uniffi(default)]
     pub max_derived_facts: Option<i64>,
-    /// Rows a query may return. Default 1,000,000.
+    /// Deprecated alias of `max_derived_facts`; the lower of the two applies.
     #[uniffi(default)]
     pub max_results: Option<i64>,
     /// WAL durability. Default `Full`.
     #[uniffi(default)]
     pub synchronous: Option<SyncMode>,
+    /// The most bytes one query may hold; a query that would hold more fails
+    /// with QRY-012. The maximum (i64::MAX, 9223372036854775807) means no
+    /// limit, the default.
+    #[uniffi(default)]
+    pub query_memory_limit: Option<i64>,
 }
 
 impl OpenOptions {
@@ -160,14 +165,27 @@ impl OpenOptions {
             o = o.max_derived_facts(to_usize(v, "max_derived_facts")?);
         }
         if let Some(v) = self.max_results {
-            o = o.max_results(to_usize(v, "max_results")?);
+            #[allow(deprecated)]
+            {
+                o = o.max_results(to_usize(v, "max_results")?);
+            }
         }
         if let Some(v) = self.synchronous {
             o = o.synchronous(match v {
                 SyncMode::Full => minigraf::SyncMode::Full,
                 SyncMode::Normal => minigraf::SyncMode::Normal,
             });
         }
+        if let Some(v) = self.query_memory_limit {
+            // i64::MAX stands for core's usize::MAX (no limit), as for
+            // wal_checkpoint_threshold.
+            let n = if v == i64::MAX {
+                usize::MAX
+            } else {
+                to_usize(v, "query_memory_limit")?
+            };
+            o = o.query_memory_limit(n);
+        }
         Ok(o)
     }
 }
@@ -717,7 +735,7 @@ mod tests {
 
     #[test]
     fn value_to_json_ref() {
-        let id = minigraf::EntityId::new_v4();
+        let id = minigraf::EntityId::from_u128(0x6f1c_2a3b_4d5e_4f60_8172_839a_b4c5_d6e7);
         let j = value_to_json(&Value::Ref(id));
         assert_eq!(j, serde_json::Value::String(id.to_string()));
     }
@@ -832,7 +850,7 @@ mod tests {
 
     #[test]
     fn value_round_trips_through_core() {
-        let id = minigraf::EntityId::new_v4();
+        let id = minigraf::EntityId::from_u128(0x6f1c_2a3b_4d5e_4f60_8172_839a_b4c5_d6e7);
         for v in [
             Value::String("s".into()),
             Value::Integer(-3),
@@ -914,6 +932,40 @@ mod tests {
         let _ = std::fs::remove_dir_all(&dir);
     }
 
+    #[test]
+    fn query_memory_limit_is_qry_012() {
+        let options = |limit| OpenOptions {
+            query_memory_limit: Some(limit),
+            ..OpenOptions::default()
+        };
+        assert!(msg(err(options(-1).to_core())).starts_with("[API-017]"));
+
+        let dir = std::env::temp_dir().join(format!("minigraf_ffi_qml_{}", std::process::id()));
+        let _ = std::fs::remove_dir_all(&dir);
+        std::fs::create_dir_all(&dir).expect("dir");
+        // A cartesian product holds one side in full.
+        let product = "(query [:find ?a ?b :where [?a :n _] [?b :m _]])";
+        for (name, limit, fails) in [("small", 64, true), ("max", i64::MAX, false)] {
+            let path = dir.join(format!("{name}.graph"));
+            let db =
+                MiniGrafDb::open_with_options(path.to_str().expect("utf8").into(), options(limit))
+                    .expect("open");
+            db.execute("(transact [[:a :n 1] [:b :m 2]])".into())
+                .expect("transact");
+            let r = db.execute(product.into());
+            if fails {
+                let e = err(r);
+                assert!(matches!(e, MiniGrafError::Query { .. }), "a query error");
+                assert!(msg(e).starts_with("[QRY-012]"));
+                let cursor = db.query(product.into()).expect("query");
+                assert!(msg(err(cursor.next_batch(10))).starts_with("[QRY-012]"));
+            } else {
+                r.expect("no limit");
+            }
+        }
+        let _ = std::fs::remove_dir_all(&dir);
+    }
+
     #[test]
     fn cursor_batches_and_closes() {
         let db = MiniGrafDb::open_in_memory().expect("open");

tests/test_basic.py

@@ -1,6 +1,7 @@
 import json
 import os
 import pytest
+import minigraf
 from minigraf import MiniGrafDb, MiniGrafError
 
 
@@ -36,3 +37,18 @@ def test_file_backed_roundtrip(tmp_path):
     db2 = MiniGrafDb.open(path)
     result = json.loads(db2.execute("(query [:find ?n :where [?e :name ?n]])"))
     assert result["results"][0][0] == "Bob"
+
+
+def test_query_memory_limit_raises_qry_012(tmp_path):
+    """#549: a query that would hold more than query_memory_limit bytes fails."""
+    product = "(query [:find ?a ?b :where [?a :n _] [?b :m _]])"
+
+    db = minigraf.open(str(tmp_path / "small.graph"), query_memory_limit=64)
+    db.execute("(transact [[:a :n 1] [:b :m 2]])")
+    with pytest.raises(MiniGrafError.Query) as e:
+        db.execute(product)
+    assert e.value.msg.startswith("[QRY-012]")
+
+    db = minigraf.open(str(tmp_path / "max.graph"), query_memory_limit=2**63 - 1)
+    db.execute("(transact [[:a :n 1] [:b :m 2]])")
+    assert len(json.loads(db.execute(product))["results"]) == 1