From cb20d730e4be4a66c02e33272a702ff26da438f7 Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Thu, 5 Nov 2026 13:12:26 -0800 Subject: [PATCH 01/71] fix(evals): bound HTTP decision response memory Refs #4535. Read at most 1 MiB plus one byte from the selected endpoint. Refuse oversized advertised responses and streaming responses without a length before parsing untrusted JSON, as well as malformed Content-Length. Avoid allocating an unbounded remote response. Cover all three refusal paths against a real loopback HTTP server. The isolated transport probe passed; hosted Bazel tests are still needed. --- evals/intent_engine/run_http.py | 17 +++++++++- evals/intent_engine/test_run_http.py | 49 ++++++++++++++++++++++++++++ 2 files changed, 65 insertions(+), 1 deletion(-) diff --git a/evals/intent_engine/run_http.py b/evals/intent_engine/run_http.py index 07f9340f7f8..c849a1aedf4 100644 --- a/evals/intent_engine/run_http.py +++ b/evals/intent_engine/run_http.py @@ -30,6 +30,10 @@ "B2_noul_text_promptfirst": lambda c: text_window(c, prompt_first=True), } +# Each server response contains only a bounded set of typed decision answers. +# Refuse an oversized or streaming response before parsing untrusted JSON. +MAX_DECISION_RESPONSE_BYTES = 1024 * 1024 + def digest(value): return hashlib.sha256( @@ -170,7 +174,18 @@ def post(url, body, timeout=1800, bearer_token=None): ) started = time.perf_counter() with opener.open(req, timeout=timeout) as response: - out = json.loads(response.read()) + declared = response.headers.get("Content-Length") + if declared is not None: + try: + declared = int(declared) + except ValueError as error: + raise ValueError("decision response has an invalid Content-Length") from error + if declared < 0 or declared > MAX_DECISION_RESPONSE_BYTES: + raise ValueError("decision response exceeds the 1 MiB size limit") + payload = response.read(MAX_DECISION_RESPONSE_BYTES + 1) + if len(payload) > MAX_DECISION_RESPONSE_BYTES: + raise ValueError("decision response exceeds the 1 MiB size limit") + out = json.loads(payload) return out, (time.perf_counter() - started) * 1000 diff --git a/evals/intent_engine/test_run_http.py b/evals/intent_engine/test_run_http.py index 6d5e710698c..d7ee40985e1 100644 --- a/evals/intent_engine/test_run_http.py +++ b/evals/intent_engine/test_run_http.py @@ -1050,5 +1050,54 @@ def log_message(self, *_): server.server_close() + def test_decision_response_rejects_oversized_or_invalid_http_bodies(self): + class Answer(BaseHTTPRequestHandler): + def do_POST(self): + self.rfile.read(int(self.headers["Content-Length"])) + valid = b'{"answers":{}}' + self.send_response(200) + if self.path == "/declared": + # The server claims a huge body but sends valid, small JSON. + # A check after an unrestricted read would accept this. + self.send_header("Content-Length", str(run_http.MAX_DECISION_RESPONSE_BYTES + 1)) + self.end_headers() + self.wfile.write(valid) + elif self.path == "/invalid": + self.send_header("Content-Length", "not-a-number") + self.end_headers() + self.wfile.write(valid) + else: + # No Content-Length: a streaming server must also be bounded. + payload = (b'{"answers":{},"padding":"' + + b"x" * run_http.MAX_DECISION_RESPONSE_BYTES + b'"}') + self.end_headers() + try: + self.wfile.write(payload) + except BrokenPipeError: + pass + + def log_message(self, *_): + pass + + server = HTTPServer(("127.0.0.1", 0), Answer) + worker = Thread(target=server.serve_forever, daemon=True) + worker.start() + try: + base = f"http://127.0.0.1:{server.server_port}" + body = {"state": "private transcript", "questions": {}} + for path, message in ( + ("/declared", "size limit"), + ("/streamed", "size limit"), + ("/invalid", "invalid Content-Length"), + ): + with self.subTest(path=path): + with self.assertRaisesRegex(ValueError, message): + run_http.post(base + path, body, timeout=2) + finally: + server.shutdown() + worker.join(timeout=2) + server.server_close() + + if __name__ == "__main__": unittest.main() From 76630b9c488d61be720479b23805baaeb9758649 Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Thu, 5 Nov 2026 14:15:14 -0800 Subject: [PATCH 02/71] fix(evals): require typed Noul responses for HTTP evaluation Refs #4535. The active /v1/systemone answer contract carries a type tag. Previously an untyped answer with a noul field was accepted as a successful prediction, obscuring protocol drift during model qualification. Refuse missing or mismatched tags before persisting rows or raw samples. Update test server doubles and cover missing-tag denial with partial-log preservation. --- evals/intent_engine/run_http.py | 8 ++++---- evals/intent_engine/test_run_http.py | 5 ++++- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/evals/intent_engine/run_http.py b/evals/intent_engine/run_http.py index c849a1aedf4..1c47f575b7f 100644 --- a/evals/intent_engine/run_http.py +++ b/evals/intent_engine/run_http.py @@ -227,11 +227,11 @@ def noul_probabilities(response, questions): raise ValueError("noul answers must cover exactly the requested questions") values = {} for key, answer in answers.items(): - # Older decision servers omit the kind, but a present tag must agree - # with the requested question. A Score response with an incidental - # noul field is not evidence of a successful Noul prediction. + # The current decision contract tags every answer. An untyped + # response could be an unrelated decision with an incidental noul + # value, so it cannot qualify as Noul model evidence. if (not isinstance(answer, dict) - or ("type" in answer and answer["type"] != "noul") + or answer.get("type") != "noul" or "noul" not in answer): raise ValueError(f"invalid noul answer for {key}") values[key] = answer["noul"] diff --git a/evals/intent_engine/test_run_http.py b/evals/intent_engine/test_run_http.py index d7ee40985e1..750eb25838e 100644 --- a/evals/intent_engine/test_run_http.py +++ b/evals/intent_engine/test_run_http.py @@ -48,7 +48,7 @@ def write(self, name, value): (self.root / name).write_text(json.dumps(value), encoding="utf-8") def response(self, url, body): - return {"answers": {key: {"noul": 0.25} for key in body["questions"]}, + return {"answers": {key: {"type": "noul", "noul": 0.25} for key in body["questions"]}, "usage": {"input_tokens": 12}, "latency_ms": 0}, 5.0 def run_main(self, extra=()): @@ -762,6 +762,9 @@ def test_invalid_answer_kinds_and_shapes_preserve_resumable_rows(self): sample = self.root / "rejected-response.json" response, elapsed = self.response("", {"questions": run_http.noul_questions()}) invalid = [] + untyped = copy.deepcopy(response) + del untyped["answers"]["impact"]["type"] + invalid.append(untyped) for declared_type in ("score", None, False, [], {"kind": "noul"}): malformed = copy.deepcopy(response) malformed["answers"]["impact"]["type"] = declared_type From e4988338f2778102567b2b219fbeacb3331297db Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Thu, 5 Nov 2026 15:09:14 -0800 Subject: [PATCH 03/71] fix(evals): reject ambiguous JSON decision responses Refs #4535. Reject duplicate object keys at every JSON depth before an inference result can enter the resumable evidence log or sample. The loopback regression covers both duplicate answer maps and duplicate Noul values, followed by a successful valid resume. --- evals/intent_engine/run_http.py | 12 +++++- evals/intent_engine/test_run_http.py | 55 ++++++++++++++++++++++++++++ 2 files changed, 66 insertions(+), 1 deletion(-) diff --git a/evals/intent_engine/run_http.py b/evals/intent_engine/run_http.py index 1c47f575b7f..1fd720711af 100644 --- a/evals/intent_engine/run_http.py +++ b/evals/intent_engine/run_http.py @@ -161,6 +161,16 @@ def load_bearer_token(name, url): return token +def reject_duplicate_json_fields(pairs): + """Reject ambiguous nested JSON objects before recording model evidence.""" + obj = {} + for key, value in pairs: + if key in obj: + raise ValueError(f"decision response contains duplicate JSON field: {key}") + obj[key] = value + return obj + + def post(url, body, timeout=1800, bearer_token=None): require_http_endpoint(url) headers = {"content-type": "application/json"} @@ -185,7 +195,7 @@ def post(url, body, timeout=1800, bearer_token=None): payload = response.read(MAX_DECISION_RESPONSE_BYTES + 1) if len(payload) > MAX_DECISION_RESPONSE_BYTES: raise ValueError("decision response exceeds the 1 MiB size limit") - out = json.loads(payload) + out = json.loads(payload, object_pairs_hook=reject_duplicate_json_fields) return out, (time.perf_counter() - started) * 1000 diff --git a/evals/intent_engine/test_run_http.py b/evals/intent_engine/test_run_http.py index 750eb25838e..d564164dbdf 100644 --- a/evals/intent_engine/test_run_http.py +++ b/evals/intent_engine/test_run_http.py @@ -1053,6 +1053,61 @@ def log_message(self, *_): server.server_close() + def test_duplicate_json_response_fields_cannot_publish_evaluation_evidence(self): + response, _ = self.response("", {"questions": run_http.noul_questions()}) + valid = json.dumps(response).encode("utf-8") + duplicate_answers = ( + b'{"answers":{},"answers":' + + json.dumps(response["answers"]).encode("utf-8") + b'}' + ) + duplicate_noul = valid.replace( + b'"noul": 0.25', b'"noul": 0.1, "noul": 0.25', 1 + ) + self.assertNotEqual(duplicate_noul, valid) + + class Answer(BaseHTTPRequestHandler): + payload = valid + + def do_POST(self): + self.rfile.read(int(self.headers["Content-Length"])) + payload = type(self).payload + self.send_response(200) + self.send_header("Content-Length", str(len(payload))) + self.end_headers() + self.wfile.write(payload) + + def log_message(self, *_): + pass + + server = HTTPServer(("127.0.0.1", 0), Answer) + worker = Thread(target=server.serve_forever, daemon=True) + worker.start() + sample = self.root / "first-response.json" + extra = [ + "--url", f"http://127.0.0.1:{server.server_port}/v1/systemone", + "--sample-out", str(sample), + ] + try: + for ambiguous in (duplicate_answers, duplicate_noul): + Answer.payload = ambiguous + with self.subTest(payload=ambiguous[:100]): + with self.assertRaisesRegex(ValueError, "duplicate JSON field"): + self.run_main(extra) + self.assertEqual(len(self.partial.read_text().splitlines()), 1) + self.assertFalse(sample.exists()) + self.assertFalse(self.output.exists()) + Answer.payload = valid + self.run_main(extra) + result = json.loads(self.output.read_text(encoding="utf-8")) + self.assertEqual([row["id"] for row in result["rows"]], ["first", "second"]) + self.assertTrue(sample.exists()) + self.assertFalse(self.partial.exists()) + finally: + server.shutdown() + worker.join(timeout=2) + server.server_close() + + def test_decision_response_rejects_oversized_or_invalid_http_bodies(self): class Answer(BaseHTTPRequestHandler): def do_POST(self): From 2bcc5c5b9baa6a00983ce89859b6226078746e70 Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Thu, 5 Nov 2026 16:15:07 -0800 Subject: [PATCH 04/71] fix(evals): reject ambiguous JSON inputs and scoring evidence Refs #4535. Reject duplicate object fields and non-JSON numeric constants in dataset and server manifests, partial resumes, and scored results. Preserve completed summaries and resumable evidence on refusal. --- evals/intent_engine/common.py | 19 +++++++- evals/intent_engine/run_http.py | 24 +++------- evals/intent_engine/score.py | 11 +++-- evals/intent_engine/test_run_http.py | 68 ++++++++++++++++++++++++++++ evals/intent_engine/test_score.py | 35 ++++++++++++++ 5 files changed, 136 insertions(+), 21 deletions(-) diff --git a/evals/intent_engine/common.py b/evals/intent_engine/common.py index 1678498ee6a..489dbae7fcb 100644 --- a/evals/intent_engine/common.py +++ b/evals/intent_engine/common.py @@ -2,6 +2,22 @@ import json import math + +def strict_json_loads(data): + """Read evaluation inputs and evidence without ambiguous JSON fields.""" + def unique_fields(pairs): + fields = {} + for key, value in pairs: + if key in fields: + raise ValueError(f"duplicate JSON field: {key}") + fields[key] = value + return fields + + def refuse_constant(value): + raise ValueError(f"invalid JSON numeric constant: {value}") + + return json.loads(data, object_pairs_hook=unique_fields, parse_constant=refuse_constant) + CATS = ["session_recall", "memory_store", "call_graph", "impact", "refactor_edit", "review_changes", "type_orientation", "project_context", "unexpected_changes"] @@ -68,4 +84,5 @@ def json_window(c, turns=None): "recent_tools": [t["text"] for t in turns if t["role"] == "tool"]} def load(path="dataset.json"): - return json.load(open(path)) + with open(path, encoding="utf-8") as source: + return strict_json_loads(source.read()) diff --git a/evals/intent_engine/run_http.py b/evals/intent_engine/run_http.py index 1fd720711af..36e1652c1ed 100644 --- a/evals/intent_engine/run_http.py +++ b/evals/intent_engine/run_http.py @@ -22,7 +22,7 @@ HERE = os.path.dirname(os.path.abspath(__file__)) sys.path.insert(0, HERE) -from common import calibrated, noul_questions, prompt_only, text_window +from common import calibrated, noul_questions, prompt_only, strict_json_loads, text_window STATES = { "A_noul_prompt": prompt_only, @@ -161,16 +161,6 @@ def load_bearer_token(name, url): return token -def reject_duplicate_json_fields(pairs): - """Reject ambiguous nested JSON objects before recording model evidence.""" - obj = {} - for key, value in pairs: - if key in obj: - raise ValueError(f"decision response contains duplicate JSON field: {key}") - obj[key] = value - return obj - - def post(url, body, timeout=1800, bearer_token=None): require_http_endpoint(url) headers = {"content-type": "application/json"} @@ -195,13 +185,13 @@ def post(url, body, timeout=1800, bearer_token=None): payload = response.read(MAX_DECISION_RESPONSE_BYTES + 1) if len(payload) > MAX_DECISION_RESPONSE_BYTES: raise ValueError("decision response exceeds the 1 MiB size limit") - out = json.loads(payload, object_pairs_hook=reject_duplicate_json_fields) + out = strict_json_loads(payload) return out, (time.perf_counter() - started) * 1000 def load_server_config(path): with open(path, encoding="utf-8") as source: - config = json.load(source) + config = strict_json_loads(source.read()) if not isinstance(config, dict): raise ValueError("--server-config must contain a JSON object") for field in ("checkpoint", "server_revision", "backend", "dtype"): @@ -331,13 +321,13 @@ def open_partial_for_resume(path): def resume_rows(partial, header, cases, requests, questions, temperature): """Validate the whole existing log before appending or issuing HTTP work.""" first = partial.readline() - if not first.endswith("\n") or json.loads(first) != header: + if not first.endswith("\n") or strict_json_loads(first) != header: raise ValueError("partial output belongs to another or unrecorded experiment; use a new tag") done = {} for line_number, line in enumerate(partial, 2): if not line.endswith("\n"): raise ValueError(f"partial output has an incomplete row at line {line_number}") - row = json.loads(line) + row = strict_json_loads(line) case_id = row.get("id") if isinstance(row, dict) else None if not isinstance(case_id, str) or case_id not in cases or case_id in done: raise ValueError(f"partial output has an unknown or duplicate case at line {line_number}") @@ -410,13 +400,13 @@ def main(argv=None): raise ValueError(f"sample path exists but is not a regular file: {args.sample_out}") with open(os.path.join(HERE, "dataset.json"), "rb") as source: dataset_bytes = source.read() - dataset = json.loads(dataset_bytes) + dataset = strict_json_loads(dataset_bytes) dataset_sha = hashlib.sha256(dataset_bytes).hexdigest() questions = noul_questions() case_ids = validate_dataset(dataset, questions) if args.subset: with open(args.subset, encoding="utf-8") as source: - ids = json.load(source)["ids"] + ids = strict_json_loads(source.read())["ids"] if (not isinstance(ids, list) or not ids or any(not isinstance(case_id, str) for case_id in ids) or len(set(ids)) != len(ids) or not set(ids).issubset(case_ids)): diff --git a/evals/intent_engine/score.py b/evals/intent_engine/score.py index da9375d9624..65299dbf89f 100644 --- a/evals/intent_engine/score.py +++ b/evals/intent_engine/score.py @@ -6,7 +6,7 @@ """ import bisect, hashlib, json, glob, math, os, re, statistics, sys, tempfile sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) -from common import CATS, calibrated +from common import CATS, calibrated, strict_json_loads TH = [0.05, 0.1, 0.15, 0.2, 0.25, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9] @@ -304,10 +304,15 @@ def ch(rs): if __name__ == "__main__": here = os.path.dirname(os.path.abspath(__file__)) - fixed = json.load(open(os.path.join(here, "fixed_thresholds.json"))) if os.path.exists(os.path.join(here, "fixed_thresholds.json")) else None + fixed = None + thresholds_path = os.path.join(here, "fixed_thresholds.json") + if os.path.exists(thresholds_path): + with open(thresholds_path, encoding="utf-8") as source: + fixed = strict_json_loads(source.read()) summary = {}; allres = {} for f in sorted(glob.glob(os.path.join(here, "results", "*.json"))): - res = json.load(open(f)) + with open(f, encoding="utf-8") as source: + res = strict_json_loads(source.read()) if res["config"] in allres: raise ValueError("duplicate configuration %r in result files" % res["config"]) allres[res["config"]] = res diff --git a/evals/intent_engine/test_run_http.py b/evals/intent_engine/test_run_http.py index d564164dbdf..447d1516d2d 100644 --- a/evals/intent_engine/test_run_http.py +++ b/evals/intent_engine/test_run_http.py @@ -111,6 +111,74 @@ def test_invalid_dataset_is_refused_before_network_or_partial_output(self): finally: self.write("dataset.json", self.dataset) + def test_ambiguous_json_inputs_refuse_before_inference(self): + dataset_path = self.root / "dataset.json" + server_path = self.root / "server.json" + subset_path = self.root / "subset.json" + original_dataset = dataset_path.read_text(encoding="utf-8") + original_server = server_path.read_text(encoding="utf-8") + variants = [ + (dataset_path, original_dataset.replace( + '"labels": ["impact"]', '"labels": [], "labels": ["impact"]', 1 + ), (), "duplicate JSON field"), + (server_path, original_server.replace( + '"checkpoint": "weights@immutable-revision"', + '"checkpoint": "other", "checkpoint": "weights@immutable-revision"', 1 + ), (), "duplicate JSON field"), + (subset_path, '{"ids":["first"],"ids":["second"]}', + ("--subset", str(subset_path)), "duplicate JSON field"), + (dataset_path, original_dataset.replace( + '"current_prompt": "What breaks?"', + '"current_prompt": "What breaks?", "debug": NaN', 1 + ), (), "invalid JSON numeric constant"), + ] + for path, contents, extra, diagnostic in variants: + original = path.read_bytes() if path.exists() else None + try: + if original is not None: + self.assertNotEqual(contents, original.decode("utf-8")) + path.write_text(contents, encoding="utf-8") + with self.subTest(path=path.name, diagnostic=diagnostic): + with patch.object(run_http, "post") as post: + with self.assertRaisesRegex(ValueError, diagnostic): + self.run_main(extra) + post.assert_not_called() + self.assertFalse(self.partial.exists()) + self.assertFalse(self.output.exists()) + finally: + if original is None: + path.unlink() + else: + path.write_bytes(original) + + def test_ambiguous_partial_json_refuses_without_mutating_resume(self): + self.interrupt() + original = self.partial.read_text(encoding="utf-8") + header, row = original.splitlines() + variants = [ + header.replace( + '"experiment_sha256":', '"experiment_sha256": "fake", "experiment_sha256":', 1 + ) + "\n" + row + "\n", + header + "\n" + row.replace( + '"labels":', '"labels": [], "labels":', 1 + ) + "\n", + header + "\n" + row.replace( + '"ms":', '"unrelated": Infinity, "ms":', 1 + ) + "\n", + ] + try: + for ambiguous in variants: + with self.subTest(evidence=ambiguous[-120:]): + self.assertNotEqual(ambiguous, original) + self.partial.write_text(ambiguous, encoding="utf-8") + self.assert_refused_unchanged() + finally: + self.partial.write_text(original, encoding="utf-8") + with patch.object(run_http, "post", side_effect=self.response) as post: + self.run_main() + post.assert_called_once() + self.assertEqual(len(json.loads(self.output.read_text())["rows"]), 2) + def test_exact_resume_only_requests_the_missing_case(self): self.interrupt() with patch.object(run_http, "post", side_effect=self.response) as post: diff --git a/evals/intent_engine/test_score.py b/evals/intent_engine/test_score.py index c5f7100b5f5..6953419a60e 100644 --- a/evals/intent_engine/test_score.py +++ b/evals/intent_engine/test_score.py @@ -228,6 +228,41 @@ def test_cli_reports_the_scored_subset_without_loading_the_full_dataset(self): self.assertIn("AUC eligible categories", compared.stdout) self.assertFalse((root / "dataset.json").exists()) + def test_ambiguous_completed_result_refuses_to_replace_valid_summary(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + self.stage_scripts(root) + results = root / "results" + results.mkdir() + original = json.dumps({"config": "sample", "rows": [row("a")]}) + variants = [ + (original.replace( + '"config": "sample"', '"config": "other", "config": "sample"', 1 + ), "duplicate JSON field"), + (original.replace( + '"labels": []', '"labels": ["impact"], "labels": []', 1 + ), "duplicate JSON field"), + (original.replace( + '"ms": 100', '"ignored": NaN, "ms": 100', 1 + ), "invalid JSON numeric constant"), + ] + summary = root / "summary.json" + for contents, diagnostic in variants: + self.assertNotEqual(contents, original) + (results / "sample.json").write_text(contents, encoding="utf-8") + summary.write_bytes(b"last verified summary\\n") + with self.subTest(diagnostic=diagnostic): + attempt = self.run_script(root, "score.py") + self.assertNotEqual(attempt.returncode, 0) + self.assertIn(diagnostic, attempt.stderr) + self.assertEqual(summary.read_bytes(), b"last verified summary\\n") + (results / "sample.json").write_text(original, encoding="utf-8") + scored = self.run_script(root, "score.py") + self.assertEqual(scored.returncode, 0, scored.stderr) + self.assertEqual( + json.loads(summary.read_text())["sample"]["latency"]["total_cases"], 1 + ) + def test_empty_cli_result_does_not_overwrite_an_existing_summary(self): with tempfile.TemporaryDirectory() as directory: root = Path(directory) From 203d0ddf2b4fcf3445ae15186e27138695d42328 Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Thu, 5 Nov 2026 17:14:17 -0800 Subject: [PATCH 05/71] fix(evals): bind sampled HTTP responses to durable experiment evidence Refuse stale or swapped raw samples on new runs and resumed partials. Record the raw response digest in completed rows; verify sampled experiment, case, request and response before further inference. Keep one capture for the first configuration and reject competing publication without replacing foreign evidence. Refs #4535 --- evals/intent_engine/run_http.py | 74 ++++++++++++++++++++----- evals/intent_engine/test_run_http.py | 82 +++++++++++++++++++++++++++- 2 files changed, 140 insertions(+), 16 deletions(-) diff --git a/evals/intent_engine/run_http.py b/evals/intent_engine/run_http.py index 36e1652c1ed..b0a85f865a1 100644 --- a/evals/intent_engine/run_http.py +++ b/evals/intent_engine/run_http.py @@ -46,6 +46,38 @@ def request_bytes(body): return json.dumps(body, allow_nan=False).encode("utf-8") +def verify_existing_sample(path, header, requests, done, questions): + """A resumed capture must describe a durable response from this experiment. + + Never silently reuse an unrelated file as the model's first raw answer. + Opening a regular file without following links also avoids blocking on + hostile sample paths such as named pipes. + """ + flags = os.O_RDONLY | getattr(os, "O_NONBLOCK", 0) | getattr(os, "O_NOFOLLOW", 0) + descriptor = os.open(path, flags) + try: + if not stat.S_ISREG(os.fstat(descriptor).st_mode): + raise ValueError(f"sample path is not a regular file: {path}") + with os.fdopen(descriptor, "rb") as source: + descriptor = None + sample = strict_json_loads(source.read()) + finally: + if descriptor is not None: + os.close(descriptor) + case_id = sample.get("case_id") if isinstance(sample, dict) else None + if (not isinstance(case_id, str) or case_id not in requests + or sample.get("experiment_sha256") != header["experiment_sha256"] + or sample.get("request_sha256") != requests[case_id][1] + or sample.get("request") != requests[case_id][0]): + raise ValueError("sample capture does not belong to this experiment") + if case_id not in done: + raise ValueError("sample capture has no durable matching response row") + response = sample.get("response") + noul_probabilities(response, questions) + if done[case_id].get("response_sha256") != digest(response): + raise ValueError("sample capture disagrees with durable response evidence") + + def publish_sample(path, serialized): """Publish a completely rendered sample without clobbering an existing one. @@ -62,11 +94,12 @@ def publish_sample(path, serialized): output.write(serialized) try: os.link(staged, path) - except FileExistsError: - # A dangling symlink occupies the pathname but exists() says - # false. Do not treat that unusable capture as already published. - if not os.path.isfile(path): - raise ValueError(f"sample path exists but is not a regular file: {path}") + except FileExistsError as error: + # Another run may have published a different capture after + # preflight. Never silently use it as evidence for this experiment. + if not os.path.isfile(path) or os.path.islink(path): + raise ValueError(f"sample path exists but is not a regular file: {path}") from error + raise ValueError(f"sample capture was concurrently published: {path}") from error finally: os.unlink(staged) @@ -396,7 +429,8 @@ def main(argv=None): server_config = load_server_config(args.server_config) bearer_token = (load_bearer_token(args.bearer_token_env, args.url) if args.bearer_token_env is not None else None) - if args.sample_out and os.path.lexists(args.sample_out) and not os.path.isfile(args.sample_out): + if (args.sample_out and os.path.lexists(args.sample_out) + and (os.path.islink(args.sample_out) or not os.path.isfile(args.sample_out))): raise ValueError(f"sample path exists but is not a regular file: {args.sample_out}") with open(os.path.join(HERE, "dataset.json"), "rb") as source: dataset_bytes = source.read() @@ -418,13 +452,19 @@ def main(argv=None): cases = {case["id"]: case for case in dataset} os.makedirs(os.path.join(HERE, "results"), exist_ok=True) for cfg in args.configs: + # One --sample-out captures one response from the first configuration. + # Later configurations must not misattribute that capture to themselves. + sample_path = args.sample_out if cfg == args.configs[0] else None name = f"{args.tag}_{cfg}" path = os.path.join(HERE, "results", name + ".json") if os.path.lexists(path): raise ValueError(f"completed output already exists: {path}; use a new tag") + if (sample_path and os.path.lexists(sample_path) + and not os.path.lexists(path + ".partial")): + raise ValueError(f"sample capture already exists for a new run: {sample_path}") require_exclusive_publication(os.path.dirname(os.path.abspath(path))) - if args.sample_out and not os.path.isfile(args.sample_out): - require_exclusive_publication(os.path.dirname(os.path.abspath(args.sample_out))) + if sample_path and not os.path.lexists(sample_path): + require_exclusive_publication(os.path.dirname(os.path.abspath(sample_path))) requests = {} for case in dataset: body = {"state": STATES[cfg](case), "questions": questions} @@ -467,6 +507,10 @@ def main(argv=None): done = {} else: done = resume_rows(partial, header, cases, requests, questions, args.noul_temperature) + if sample_path and os.path.lexists(sample_path): + if created: + raise ValueError(f"sample capture already exists for a new run: {sample_path}") + verify_existing_sample(sample_path, header, requests, done, questions) for index, case in enumerate(dataset): if case["id"] in done: continue @@ -491,6 +535,7 @@ def main(argv=None): "input_tokens": usage.get("input_tokens"), "state_tokens": usage.get("state_tokens"), "dropped": usage.get("state_tokens_dropped"), "probs": probs, + "response_sha256": digest(response), } if args.noul_temperature is not None: row["probs_raw"] = probs @@ -499,11 +544,14 @@ def main(argv=None): # either. A non-finite response must not leave a sample file # that a later valid retry mistakes for a completed capture. serialized_row = serialize_result(row, questions, args.noul_temperature) - if args.sample_out and not os.path.exists(args.sample_out): - serialized_sample = json.dumps( - {"request": body, "response": response}, indent=1, allow_nan=False - ) - publish_sample(args.sample_out, serialized_sample) + if sample_path and not os.path.lexists(sample_path): + serialized_sample = json.dumps({ + "case_id": case["id"], + "experiment_sha256": header["experiment_sha256"], + "request_sha256": request_sha, + "request": body, "response": response, + }, indent=1, allow_nan=False) + publish_sample(sample_path, serialized_sample) partial.write(serialized_row + "\n") partial.flush() done[case["id"]] = row diff --git a/evals/intent_engine/test_run_http.py b/evals/intent_engine/test_run_http.py index 447d1516d2d..1e863065998 100644 --- a/evals/intent_engine/test_run_http.py +++ b/evals/intent_engine/test_run_http.py @@ -903,14 +903,90 @@ def test_nonfinite_raw_response_does_not_publish_sample_or_result(self): self.assertFalse(sample.exists()) self.assertEqual(len(self.partial.read_text().splitlines()), 1) - def test_existing_sample_is_never_replaced(self): + def test_existing_sample_from_other_run_refuses_without_inference(self): sample = self.root / "first-response.json" sample.write_text('{"previous": true}') - with patch.object(run_http, "post", side_effect=self.response): - self.run_main(["--sample-out", str(sample)]) + with patch.object(run_http, "post") as post: + with self.assertRaisesRegex(ValueError, "sample capture already exists"): + self.run_main(["--sample-out", str(sample)]) + post.assert_not_called() self.assertEqual(sample.read_text(), '{"previous": true}') + self.assertFalse(self.partial.exists()) + self.assertFalse(self.output.exists()) + + def test_sample_identity_and_raw_response_must_match_durable_resume(self): + sample = self.root / "first-response.json" + extra = ["--sample-out", str(sample)] + self.interrupt(extra) + original = json.loads(sample.read_text()) + header = json.loads(self.partial.read_text().splitlines()[0]) + first_row = json.loads(self.partial.read_text().splitlines()[1]) + self.assertEqual(original["case_id"], "first") + self.assertEqual(original["experiment_sha256"], header["experiment_sha256"]) + self.assertEqual(run_http.digest(original["response"]), + first_row["response_sha256"]) + before = self.partial.read_bytes() + changes = ( + ("experiment", lambda capture: capture.__setitem__("experiment_sha256", "0" * 64)), + ("request", lambda capture: capture["request"].__setitem__("state", "unrelated")), + ("response", lambda capture: capture["response"]["answers"]["impact"].__setitem__( + "noul", 0.9)), + ) + for reason, change in changes: + with self.subTest(reason=reason): + altered = copy.deepcopy(original) + change(altered) + sample.write_text(json.dumps(altered)) + with patch.object(run_http, "post") as post: + with self.assertRaisesRegex(ValueError, "sample capture"): + self.run_main(extra) + post.assert_not_called() + self.assertEqual(self.partial.read_bytes(), before) + self.assertFalse(self.output.exists()) + sample.write_text(json.dumps(original)) + with patch.object(run_http, "post", side_effect=self.response) as post: + self.run_main(extra) + post.assert_called_once() + self.assertFalse(self.partial.exists()) self.assertEqual(len(json.loads(self.output.read_text())["rows"]), 2) + def test_multiple_configs_capture_only_first_configuration(self): + sample = self.root / "first-response.json" + first_config = "A_noul_prompt" + with patch.object(run_http, "post", side_effect=self.response) as post: + with contextlib.redirect_stdout(io.StringIO()): + run_http.main(self.base + [ + "--sample-out", str(sample), first_config, self.config, + ]) + self.assertEqual(post.call_count, len(self.dataset) * 2) + captured = json.loads(sample.read_text()) + first = json.loads((self.root / "results" / + f"probe_{first_config}.json").read_text()) + second = json.loads(self.output.read_text()) + self.assertEqual(captured["experiment_sha256"], first["experiment_sha256"]) + self.assertNotEqual(captured["experiment_sha256"], second["experiment_sha256"]) + self.assertEqual(run_http.digest(captured["response"]), + first["rows"][0]["response_sha256"]) + + def test_sample_link_race_cannot_adopt_foreign_capture(self): + sample = self.root / "first-response.json" + original_link = run_http.os.link + + def competing_sample(source, destination): + if destination == str(sample): + sample.write_text('{"foreign": true}') + raise FileExistsError(destination) + return original_link(source, destination) + + with patch.object(run_http, "post", side_effect=self.response): + with patch.object(run_http.os, "link", side_effect=competing_sample): + with self.assertRaisesRegex(ValueError, "concurrently published"): + self.run_main(["--sample-out", str(sample)]) + self.assertEqual(sample.read_text(), '{"foreign": true}') + self.assertFalse(self.output.exists()) + self.assertEqual(len(self.partial.read_text().splitlines()), 1) + self.assertEqual(list(self.root.glob(".noul-sample-*")), []) + @unittest.skipUnless(os.name == "posix", "dangling links require portable POSIX support") def test_dangling_sample_link_refuses_without_inference_or_partial_output(self): sample = self.root / "first-response.json" From 115c410ccc936c53be8f6019ccd55b927f1d0048 Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Thu, 5 Nov 2026 18:23:04 -0800 Subject: [PATCH 06/71] fix(evals): bound untrusted resume and sample file reads Refuse oversized partial headers, rows and raw sample captures before unbounded JSON reads, and preserve resumable evidence unchanged on denial. Add behavioral regressions for oversized partial and sample files followed by successful exact resumption. Refs #4535 --- evals/intent_engine/run_http.py | 32 ++++++++++++++++++-- evals/intent_engine/test_run_http.py | 44 ++++++++++++++++++++++++++++ 2 files changed, 73 insertions(+), 3 deletions(-) diff --git a/evals/intent_engine/run_http.py b/evals/intent_engine/run_http.py index b0a85f865a1..e4a2b3ada0d 100644 --- a/evals/intent_engine/run_http.py +++ b/evals/intent_engine/run_http.py @@ -46,6 +46,14 @@ def request_bytes(body): return json.dumps(body, allow_nan=False).encode("utf-8") +def sample_read_limit(requests): + """Bound saved raw captures relative to the transmitted request and HTTP cap.""" + largest_request = max(len(request_bytes(body)) for body, _ in requests.values()) + # Indentation and JSON escaping can expand the parsed HTTP body. Keep + # a conservative allowance without accepting arbitrarily large files. + return 64 * 1024 + 8 * (MAX_DECISION_RESPONSE_BYTES + largest_request) + + def verify_existing_sample(path, header, requests, done, questions): """A resumed capture must describe a durable response from this experiment. @@ -58,9 +66,15 @@ def verify_existing_sample(path, header, requests, done, questions): try: if not stat.S_ISREG(os.fstat(descriptor).st_mode): raise ValueError(f"sample path is not a regular file: {path}") + limit = sample_read_limit(requests) + if os.fstat(descriptor).st_size > limit: + raise ValueError("sample capture exceeds bounded evidence size") with os.fdopen(descriptor, "rb") as source: descriptor = None - sample = strict_json_loads(source.read()) + contents = source.read(limit + 1) + if len(contents) > limit: + raise ValueError("sample capture exceeds bounded evidence size") + sample = strict_json_loads(contents) finally: if descriptor is not None: os.close(descriptor) @@ -353,11 +367,21 @@ def open_partial_for_resume(path): def resume_rows(partial, header, cases, requests, questions, temperature): """Validate the whole existing log before appending or issuing HTTP work.""" - first = partial.readline() + header_limit = len(json.dumps(header, allow_nan=False)) + 1 + first = partial.readline(header_limit + 1) + if len(first) > header_limit: + raise ValueError("partial output has an oversized header") if not first.endswith("\n") or strict_json_loads(first) != header: raise ValueError("partial output belongs to another or unrecorded experiment; use a new tag") + # Fresh rows contain only bounded probabilities and telemetry plus the + # known case ID. Do not load an arbitrary-sized untrusted log line. + row_limit = 64 * 1024 + max(len(json.dumps(case_id)) for case_id in cases) done = {} - for line_number, line in enumerate(partial, 2): + line_number = 1 + while line := partial.readline(row_limit + 1): + line_number += 1 + if len(line) > row_limit: + raise ValueError(f"partial output has an oversized row at line {line_number}") if not line.endswith("\n"): raise ValueError(f"partial output has an incomplete row at line {line_number}") row = strict_json_loads(line) @@ -551,6 +575,8 @@ def main(argv=None): "request_sha256": request_sha, "request": body, "response": response, }, indent=1, allow_nan=False) + if len(serialized_sample.encode("utf-8")) > sample_read_limit(requests): + raise ValueError("sample capture exceeds bounded evidence size") publish_sample(sample_path, serialized_sample) partial.write(serialized_row + "\n") partial.flush() diff --git a/evals/intent_engine/test_run_http.py b/evals/intent_engine/test_run_http.py index 1e863065998..0f1411c2b94 100644 --- a/evals/intent_engine/test_run_http.py +++ b/evals/intent_engine/test_run_http.py @@ -179,6 +179,50 @@ def test_ambiguous_partial_json_refuses_without_mutating_resume(self): post.assert_called_once() self.assertEqual(len(json.loads(self.output.read_text())["rows"]), 2) + def test_oversized_resume_lines_refuse_before_network_without_damage(self): + self.interrupt() + original = self.partial.read_bytes() + header = original.splitlines(keepends=True)[0] + invalid = ( + (b"x" * 200_000 + b"\n", "oversized header"), + (header + b"x" * 200_000 + b"\n", "oversized row"), + ) + for contents, diagnostic in invalid: + with self.subTest(diagnostic=diagnostic): + self.partial.write_bytes(contents) + with patch.object(run_http, "post") as post: + with self.assertRaisesRegex(ValueError, diagnostic): + self.run_main() + post.assert_not_called() + self.assertEqual(self.partial.read_bytes(), contents) + self.assertFalse(self.output.exists()) + self.partial.write_bytes(original) + with patch.object(run_http, "post", side_effect=self.response) as post: + self.run_main() + post.assert_called_once() + self.assertEqual(len(json.loads(self.output.read_text())["rows"]), 2) + + def test_oversized_sample_capture_refuses_before_resume_inference(self): + sample = self.root / "first-response.json" + extra = ["--sample-out", str(sample)] + self.interrupt(extra) + original_sample = sample.read_bytes() + original_partial = self.partial.read_bytes() + oversized = b"x" * (9 * run_http.MAX_DECISION_RESPONSE_BYTES + 65_536) + sample.write_bytes(oversized) + with patch.object(run_http, "post") as post: + with self.assertRaisesRegex(ValueError, "sample capture exceeds"): + self.run_main(extra) + post.assert_not_called() + self.assertEqual(sample.read_bytes(), oversized) + self.assertEqual(self.partial.read_bytes(), original_partial) + self.assertFalse(self.output.exists()) + sample.write_bytes(original_sample) + with patch.object(run_http, "post", side_effect=self.response) as post: + self.run_main(extra) + post.assert_called_once() + self.assertEqual(len(json.loads(self.output.read_text())["rows"]), 2) + def test_exact_resume_only_requests_the_missing_case(self): self.interrupt() with patch.object(run_http, "post", side_effect=self.response) as post: From 71183f388b6e426cd0b1d5574deb50b5ebf083ed Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Thu, 5 Nov 2026 19:12:31 -0800 Subject: [PATCH 07/71] fix(evals): report measured p90 and p95 tail latency Use nearest-rank selection so sparse runs cannot understate their slowest observed decision at p90/p95. Include direct metric and CLI regressions without extrapolating unmeasured latencies. Refs #4535 --- evals/intent_engine/score.py | 9 ++++++--- evals/intent_engine/test_score.py | 18 ++++++++++++++++++ 2 files changed, 24 insertions(+), 3 deletions(-) diff --git a/evals/intent_engine/score.py b/evals/intent_engine/score.py index 65299dbf89f..0217210e86a 100644 --- a/evals/intent_engine/score.py +++ b/evals/intent_engine/score.py @@ -210,7 +210,10 @@ def lat(rows): observed_truncated = sum(value > 0 for value in dropped) result = { "median_ms": round(statistics.median(ms)), - "p90_ms": round(ms[int(0.9 * (len(ms) - 1))]), + # Nearest-rank tails cannot understate short runs by taking the + # first value of a two-sample run. + "p90_ms": round(ms[math.ceil(0.90 * len(ms)) - 1]), + "p95_ms": round(ms[math.ceil(0.95 * len(ms)) - 1]), "total_cases": len(rows), "cases_truncated": observed_truncated if len(dropped) == len(rows) else None, "observed_cases_truncated": observed_truncated, @@ -322,12 +325,12 @@ def ch(rs): summary[k] = score_config(res, fixed) publish_summary(os.path.join(here, "summary.json"), summary) f2 = lambda x: " - " if x is None else "%.2f" % x - print("config macroAUC ctxAUC | t0.5: mF1 top1 top1ctx noneFH | t0.6: mF1 top1 top1ctx noneFH | best: mF1 top1 top1ctx noneFH | ECE | med/p90 ms") + print("config macroAUC ctxAUC | t0.5: mF1 top1 top1ctx noneFH | t0.6: mF1 top1 top1ctx noneFH | best: mF1 top1 top1ctx noneFH | ECE | med/p90/p95 ms") for k, s in summary.items(): line = "%-26s %s %s |" % (k, f2(s["macro_auc"]), f2(s["macro_auc_ctx"])) for nm in ("t0.5", "t0.6", "best_t_in_sample"): m = s[nm]; line += " %s %s %s %s |" % (f2(m["macro_f1"]), f2(m["all"]["top1_acc"]), f2(m["ctx"]["top1_acc"]), f2(m["all"]["none_false_hint_rate"])) - line += " LOOmF1 %s | %.3f | %d/%d" % (f2(s["macro_loo_f1"]), s["reliability"]["ece"], s["latency"]["median_ms"], s["latency"]["p90_ms"]) + line += " LOOmF1 %s | %.3f | %d/%d/%d" % (f2(s["macro_loo_f1"]), s["reliability"]["ece"], s["latency"]["median_ms"], s["latency"]["p90_ms"], s["latency"]["p95_ms"]) line += " AUC categories %d/%d ctx %d/%d" % (s["macro_auc_n_categories"], len(CATS), s["macro_auc_ctx_n_categories"], len(CATS)) if "choice_top1_acc" in s: line += " argmax top1 all/ctx %s/%s noneFH %s" % (f2(s["choice_top1_acc"]["all"]), f2(s["choice_top1_acc"]["ctx"]), f2(s["choice_none_false_hint_rate"])) diff --git a/evals/intent_engine/test_score.py b/evals/intent_engine/test_score.py index 6953419a60e..f3acf540633 100644 --- a/evals/intent_engine/test_score.py +++ b/evals/intent_engine/test_score.py @@ -121,6 +121,22 @@ def test_singleton_has_no_leave_one_out_training_evidence(self): self.assertIsNone(result["per_cat"][category][metric]) self.assertEqual(result["t0.5"]["all"]["top1_acc"], 1.0) + def test_tail_latency_reports_nearest_rank_for_small_and_full_cohorts(self): + # Old floor((n - 1) * p) selected the fastest of two requests. + # Tail percentiles must come from observed samples. + for samples, expected in ( + ([10], (10, 10, 10)), + ([10, 20], (15, 20, 20)), + (list(range(1, 21)), (10, 18, 19)), + ): + with self.subTest(samples=samples): + rows = [{**row(str(i)), "ms": elapsed} + for i, elapsed in enumerate(samples)] + timing = score.score_config({"config": "tails", "rows": rows})["latency"] + actual = (timing["median_ms"], timing["p90_ms"], timing["p95_ms"]) + self.assertEqual(actual, expected) + self.assertEqual(timing["total_cases"], len(samples)) + def test_empty_results_and_partial_choice_coverage_are_refused(self): with self.assertRaisesRegex(ValueError, "empty.*no rows to score"): score.score_config({"config": "empty", "rows": []}) @@ -218,6 +234,8 @@ def test_cli_reports_the_scored_subset_without_loading_the_full_dataset(self): self.assertEqual(scored.returncode, 0, scored.stderr) summary = json.loads((root / "summary.json").read_text(encoding="utf-8")) self.assertIsNone(summary["sample"]["choice_top1_acc"]["no_ctx"]) + self.assertEqual(summary["sample"]["latency"]["p95_ms"], 100) + self.assertIn("med/p90/p95 ms", scored.stdout) self.assertIn( f"AUC categories 1/{len(score.CATS)} ctx 1/{len(score.CATS)}", scored.stdout, ) From 50c54996987a073973c6155b6903b6ffe86ba275 Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Thu, 5 Nov 2026 20:11:03 -0800 Subject: [PATCH 08/71] fix(evals): refuse malformed HTTP timing and subset documents Preserve resumable evidence and first-response captures when a model replies with non-object timing metadata; reject malformed subset containers before network inference or partial-file creation. Add behavioral recovery tests. Refs #4535 --- evals/intent_engine/run_http.py | 10 +++++++-- evals/intent_engine/test_run_http.py | 33 ++++++++++++++++++++++++++++ 2 files changed, 41 insertions(+), 2 deletions(-) diff --git a/evals/intent_engine/run_http.py b/evals/intent_engine/run_http.py index e4a2b3ada0d..b01c2a8254c 100644 --- a/evals/intent_engine/run_http.py +++ b/evals/intent_engine/run_http.py @@ -464,7 +464,10 @@ def main(argv=None): case_ids = validate_dataset(dataset, questions) if args.subset: with open(args.subset, encoding="utf-8") as source: - ids = strict_json_loads(source.read())["ids"] + subset = strict_json_loads(source.read()) + if not isinstance(subset, dict): + raise ValueError("subset must be a JSON object with ids") + ids = subset.get("ids") if (not isinstance(ids, list) or not ids or any(not isinstance(case_id, str) for case_id in ids) or len(set(ids)) != len(ids) or not set(ids).issubset(case_ids)): @@ -546,7 +549,10 @@ def main(argv=None): probs = noul_probabilities(response, questions) server_ms = response.get("latency_ms") if server_ms is None: - server_ms = (response.get("timing") or {}).get("inference_ms") + timing = response.get("timing") + if timing is not None and not isinstance(timing, dict): + raise ValueError("response timing must be an object or null") + server_ms = (timing or {}).get("inference_ms") usage = response.get("usage") if usage is None: usage = {} diff --git a/evals/intent_engine/test_run_http.py b/evals/intent_engine/test_run_http.py index 0f1411c2b94..33211b8ae8b 100644 --- a/evals/intent_engine/test_run_http.py +++ b/evals/intent_engine/test_run_http.py @@ -648,6 +648,18 @@ def test_invalid_settings_are_refused_without_inference(self): self.run_main() post.assert_not_called() + def test_malformed_subset_document_refuses_before_network_or_evidence(self): + subset_path = self.root / "subset.json" + for malformed in ([], None, "first", False, {"case_ids": ["first"]}): + with self.subTest(subset=malformed): + self.write("subset.json", malformed) + with patch.object(run_http, "post") as post: + with self.assertRaisesRegex(ValueError, "subset must"): + self.run_main(["--subset", str(subset_path)]) + post.assert_not_called() + self.assertFalse(self.partial.exists()) + self.assertFalse(self.output.exists()) + def test_invalid_subsets_cannot_silently_change_the_cohort(self): for ids in ([], ["absent"], ["first", "first"], "first"): with self.subTest(ids=ids), patch.object(run_http, "post") as post: @@ -918,6 +930,27 @@ def test_explicit_noul_answer_kind_still_records_valid_predictions(self): self.assertEqual(len(rows), len(self.dataset)) self.assertTrue(all(row["probs"]["impact"] == 0.25 for row in rows)) + def test_malformed_server_timing_refuses_without_publishing_evidence(self): + sample = self.root / "first-response.json" + response, elapsed = self.response("", {"questions": run_http.noul_questions()}) + response.pop("latency_ms") + for malformed in ([], "unknown", True, 23): + with self.subTest(timing=malformed): + with patch.object(run_http, "post", + return_value=({**response, "timing": malformed}, elapsed)): + with self.assertRaisesRegex(ValueError, "response timing must be an object"): + self.run_main(["--sample-out", str(sample)]) + self.assertEqual(len(self.partial.read_text().splitlines()), 1) + self.assertFalse(sample.exists()) + self.assertFalse(self.output.exists()) + + # Refusal must not poison a later valid run or publish misleading samples. + with patch.object(run_http, "post", side_effect=self.response) as post: + self.run_main(["--sample-out", str(sample)]) + self.assertEqual(post.call_count, len(self.dataset)) + self.assertEqual(len(json.loads(self.output.read_text())["rows"]), len(self.dataset)) + self.assertTrue(sample.exists()) + def test_malformed_response_does_not_poison_first_sample_on_resume(self): sample = self.root / "first-response.json" response, elapsed = self.response("", {"questions": run_http.noul_questions()}) From 43c9c838ff34365cd5ef08c3dbd651aa64382890 Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Thu, 5 Nov 2026 20:22:52 -0800 Subject: [PATCH 09/71] fix(evals): refuse late first-response captures on resume A partial run can already contain the first response when --sample-out is added on retry. Refuse rather than label a later response as the experiment's first raw evidence. Preserve durable partial rows and permit an ordinary resume without a capture. Add a behavioral interruption regression. Refs #4535 --- evals/intent_engine/run_http.py | 7 +++++++ evals/intent_engine/test_run_http.py | 17 +++++++++++++++++ 2 files changed, 24 insertions(+) diff --git a/evals/intent_engine/run_http.py b/evals/intent_engine/run_http.py index b01c2a8254c..55b4a31db1e 100644 --- a/evals/intent_engine/run_http.py +++ b/evals/intent_engine/run_http.py @@ -538,6 +538,13 @@ def main(argv=None): if created: raise ValueError(f"sample capture already exists for a new run: {sample_path}") verify_existing_sample(sample_path, header, requests, done, questions) + elif sample_path and done: + # A new capture would be the next response, not the first + # response of this experiment. Never mislabel it as raw evidence. + raise ValueError( + "cannot capture the first response after rows were recorded; " + "resume without --sample-out or use a new --tag" + ) for index, case in enumerate(dataset): if case["id"] in done: continue diff --git a/evals/intent_engine/test_run_http.py b/evals/intent_engine/test_run_http.py index 33211b8ae8b..2a41070140e 100644 --- a/evals/intent_engine/test_run_http.py +++ b/evals/intent_engine/test_run_http.py @@ -240,6 +240,23 @@ def test_exact_resume_only_requests_the_missing_case(self): self.assertEqual(result["rows"][0]["server_ms"], 0) self.assertFalse(self.partial.exists()) + def test_late_first_response_capture_refuses_without_changing_resume(self): + self.interrupt() + before = self.partial.read_bytes() + sample = self.root / "late-first-response.json" + with patch.object(run_http, "post") as post: + with self.assertRaisesRegex(ValueError, "cannot capture the first response"): + self.run_main(["--sample-out", str(sample)]) + post.assert_not_called() + self.assertEqual(self.partial.read_bytes(), before) + self.assertFalse(sample.exists()) + self.assertFalse(self.output.exists()) + + # Omitting the unavailable raw capture still permits an exact resume. + with patch.object(run_http, "post", side_effect=self.response) as post: + self.run_main() + post.assert_called_once() + self.assertEqual(len(json.loads(self.output.read_text())["rows"]), 2) def test_completed_rows_cannot_be_relabelled_for_scoring(self): with patch.object(run_http, "post", side_effect=self.response): From ee8b42b891ec2cc94fb99d631d0cc179ec5a5983 Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Thu, 5 Nov 2026 21:07:49 -0800 Subject: [PATCH 10/71] fix(evals): keep raw samples out of scored results Refs #4535 Reject sample destinations under results before inference. Otherwise a raw sample may be misread as a completed run or occupy its final path. Exercise direct, normalized, nested collisions and a valid external sample. --- evals/intent_engine/HTTP_RUNNER.md | 5 +++++ evals/intent_engine/run_http.py | 12 ++++++++++++ evals/intent_engine/test_run_http.py | 25 +++++++++++++++++++++++++ 3 files changed, 42 insertions(+) diff --git a/evals/intent_engine/HTTP_RUNNER.md b/evals/intent_engine/HTTP_RUNNER.md index e8f355dab66..b03f705ee17 100644 --- a/evals/intent_engine/HTTP_RUNNER.md +++ b/evals/intent_engine/HTTP_RUNNER.md @@ -47,6 +47,11 @@ duplicate/foreign cases, and malformed probability maps also refuse; no migratio or silent salvage is performed. Choose a new tag for another experiment or an old unidentifiable log. Completed output files are never overwritten. +A raw `--sample-out` capture must be outside the `results/` tree. The scorer +treats `results/*.json` as completed experiments, and a sample at the final +result pathname can block publication of a correctly completed run. The runner +refuses such a destination before creating the partial log or issuing inference. + An exclusive lock on the partial log rejects concurrent writers for the same tag/configuration. Rows are flushed after each response; this is not a crash-durable database or a concurrent-writer protocol. The runner writes and syncs the final result to a temporary file in the results directory, diff --git a/evals/intent_engine/run_http.py b/evals/intent_engine/run_http.py index 55b4a31db1e..08a9073e2a2 100644 --- a/evals/intent_engine/run_http.py +++ b/evals/intent_engine/run_http.py @@ -456,6 +456,18 @@ def main(argv=None): if (args.sample_out and os.path.lexists(args.sample_out) and (os.path.islink(args.sample_out) or not os.path.isfile(args.sample_out))): raise ValueError(f"sample path exists but is not a regular file: {args.sample_out}") + if args.sample_out: + # The scorer treats results/*.json as completed experiments. Publishing + # a raw sample under that tree can masquerade as a result, or occupy + # the final result path and permanently prevent exact resume. + result_root = os.path.normcase(os.path.realpath(os.path.join(HERE, "results"))) + sample_target = os.path.normcase(os.path.realpath(args.sample_out)) + try: + inside_results = os.path.commonpath([result_root, sample_target]) == result_root + except ValueError: + inside_results = False # Paths on different Windows drives. + if inside_results: + raise ValueError("--sample-out must be outside the results directory") with open(os.path.join(HERE, "dataset.json"), "rb") as source: dataset_bytes = source.read() dataset = strict_json_loads(dataset_bytes) diff --git a/evals/intent_engine/test_run_http.py b/evals/intent_engine/test_run_http.py index 2a41070140e..1ef81568deb 100644 --- a/evals/intent_engine/test_run_http.py +++ b/evals/intent_engine/test_run_http.py @@ -202,6 +202,31 @@ def test_oversized_resume_lines_refuse_before_network_without_damage(self): post.assert_called_once() self.assertEqual(len(json.loads(self.output.read_text())["rows"]), 2) + def test_sample_capture_cannot_impersonate_scored_results(self): + result_root = self.output.parent + for sample in ( + self.output, + result_root / "foreign.json", + result_root / "nested" / "another.json", + result_root / ".." / "results" / "alias.json", + ): + with self.subTest(sample=str(sample)): + with patch.object(run_http, "post") as post: + with self.assertRaisesRegex(ValueError, "--sample-out must be outside"): + self.run_main(["--sample-out", str(sample)]) + post.assert_not_called() + self.assertFalse(self.partial.exists()) + self.assertFalse(self.output.exists()) + self.assertFalse(result_root.exists()) + + # Capturing raw evidence elsewhere still publishes a valid result. + sample = self.root / "first-response.json" + with patch.object(run_http, "post", side_effect=self.response) as post: + self.run_main(["--sample-out", str(sample)]) + self.assertEqual(post.call_count, len(self.dataset)) + self.assertEqual(json.loads(sample.read_text())["case_id"], "first") + self.assertEqual(len(json.loads(self.output.read_text())["rows"]), 2) + def test_oversized_sample_capture_refuses_before_resume_inference(self): sample = self.root / "first-response.json" extra = ["--sample-out", str(sample)] From 5c94df35c324986ab67ab8606876fc3b60b11c7c Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Thu, 5 Nov 2026 21:20:42 -0800 Subject: [PATCH 11/71] fix(evals): require first-case identity for resumed raw samples --- evals/intent_engine/run_http.py | 4 +++ evals/intent_engine/test_run_http.py | 45 ++++++++++++++++++++++++++++ 2 files changed, 49 insertions(+) diff --git a/evals/intent_engine/run_http.py b/evals/intent_engine/run_http.py index 08a9073e2a2..b193784ca36 100644 --- a/evals/intent_engine/run_http.py +++ b/evals/intent_engine/run_http.py @@ -84,6 +84,10 @@ def verify_existing_sample(path, header, requests, done, questions): or sample.get("request_sha256") != requests[case_id][1] or sample.get("request") != requests[case_id][0]): raise ValueError("sample capture does not belong to this experiment") + # --sample-out records the first response, not merely any completed row. + # A later response with matching request/digest is still mislabelled evidence. + if case_id != next(iter(requests)): + raise ValueError("sample capture must correspond to the first case") if case_id not in done: raise ValueError("sample capture has no durable matching response row") response = sample.get("response") diff --git a/evals/intent_engine/test_run_http.py b/evals/intent_engine/test_run_http.py index 1ef81568deb..cf31360056c 100644 --- a/evals/intent_engine/test_run_http.py +++ b/evals/intent_engine/test_run_http.py @@ -1069,6 +1069,51 @@ def test_sample_identity_and_raw_response_must_match_durable_resume(self): self.assertFalse(self.partial.exists()) self.assertEqual(len(json.loads(self.output.read_text())["rows"]), 2) + def test_resumed_sample_cannot_impersonate_a_later_durable_case(self): + sample = self.root / "first-response.json" + extra = ["--sample-out", str(sample)] + self.interrupt(extra) + # Leave both rows durable but simulate a failure to publish the result. + with patch.object(run_http, "post", side_effect=self.response): + with patch.object(run_http, "publish_final_result", + side_effect=OSError("publication interrupted")): + with self.assertRaisesRegex(OSError, "publication interrupted"): + self.run_main(extra) + self.assertEqual(len(self.partial.read_text().splitlines()), 3) + self.assertFalse(self.output.exists()) + + genuine = sample.read_bytes() + forged = json.loads(genuine) + second_request = { + "state": run_http.STATES[self.config](self.dataset[1]), + "questions": run_http.noul_questions(), + } + forged["case_id"] = "second" + forged["request"] = second_request + forged["request_sha256"] = hashlib.sha256( + run_http.request_bytes(second_request) + ).hexdigest() + second_row = json.loads(self.partial.read_text().splitlines()[2]) + self.assertEqual(forged["request_sha256"], second_row["request_sha256"]) + self.assertEqual(run_http.digest(forged["response"]), + second_row["response_sha256"]) + sample.write_text(json.dumps(forged), encoding="utf-8") + before = self.partial.read_bytes() + with patch.object(run_http, "post") as post: + with self.assertRaisesRegex(ValueError, "first case"): + self.run_main(extra) + post.assert_not_called() + self.assertEqual(self.partial.read_bytes(), before) + self.assertFalse(self.output.exists()) + + # The genuine first-case capture still allows exact completion. + sample.write_bytes(genuine) + with patch.object(run_http, "post") as post: + self.run_main(extra) + post.assert_not_called() + self.assertFalse(self.partial.exists()) + self.assertEqual(len(json.loads(self.output.read_text())["rows"]), 2) + def test_multiple_configs_capture_only_first_configuration(self): sample = self.root / "first-response.json" first_config = "A_noul_prompt" From 36233acfa08c881d72a3a5f6d4bed617c200c79c Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Thu, 5 Nov 2026 22:07:45 -0800 Subject: [PATCH 12/71] fix(evals): reject overflowing JSON numbers in evidence Refuse finite-looking JSON exponents that Python decodes to infinity in untrusted datasets, resumable rows and scored result files. Preserve the last durable evidence/summary instead of admitting nonfinite extra data. Regression coverage includes pre-network dataset rejection, read-only resume rejection and preservation of a previously scored summary. Refs #4535 --- evals/intent_engine/common.py | 11 ++++++++++- evals/intent_engine/test_run_http.py | 7 +++++++ evals/intent_engine/test_score.py | 3 +++ 3 files changed, 20 insertions(+), 1 deletion(-) diff --git a/evals/intent_engine/common.py b/evals/intent_engine/common.py index 489dbae7fcb..72670c2794f 100644 --- a/evals/intent_engine/common.py +++ b/evals/intent_engine/common.py @@ -16,7 +16,16 @@ def unique_fields(pairs): def refuse_constant(value): raise ValueError(f"invalid JSON numeric constant: {value}") - return json.loads(data, object_pairs_hook=unique_fields, parse_constant=refuse_constant) + def finite_float(token): + value = float(token) + if not math.isfinite(value): + raise ValueError(f"nonfinite JSON number: {token}") + return value + + return json.loads( + data, object_pairs_hook=unique_fields, + parse_constant=refuse_constant, parse_float=finite_float, + ) CATS = ["session_recall", "memory_store", "call_graph", "impact", "refactor_edit", "review_changes", "type_orientation", "project_context", "unexpected_changes"] diff --git a/evals/intent_engine/test_run_http.py b/evals/intent_engine/test_run_http.py index cf31360056c..d86d230431a 100644 --- a/evals/intent_engine/test_run_http.py +++ b/evals/intent_engine/test_run_http.py @@ -131,6 +131,10 @@ def test_ambiguous_json_inputs_refuse_before_inference(self): '"current_prompt": "What breaks?"', '"current_prompt": "What breaks?", "debug": NaN', 1 ), (), "invalid JSON numeric constant"), + (dataset_path, original_dataset.replace( + '"current_prompt": "What breaks?"', + '"current_prompt": "What breaks?", "debug": 1e999', 1 + ), (), "nonfinite JSON number"), ] for path, contents, extra, diagnostic in variants: original = path.read_bytes() if path.exists() else None @@ -165,6 +169,9 @@ def test_ambiguous_partial_json_refuses_without_mutating_resume(self): header + "\n" + row.replace( '"ms":', '"unrelated": Infinity, "ms":', 1 ) + "\n", + header + "\n" + row.replace( + '"ms":', '"unrelated": 1e999, "ms":', 1 + ) + "\n", ] try: for ambiguous in variants: diff --git a/evals/intent_engine/test_score.py b/evals/intent_engine/test_score.py index f3acf540633..52e734a20a4 100644 --- a/evals/intent_engine/test_score.py +++ b/evals/intent_engine/test_score.py @@ -263,6 +263,9 @@ def test_ambiguous_completed_result_refuses_to_replace_valid_summary(self): (original.replace( '"ms": 100', '"ignored": NaN, "ms": 100', 1 ), "invalid JSON numeric constant"), + (original.replace( + '"ms": 100', '"ignored": -1e999, "ms": 100', 1 + ), "nonfinite JSON number"), ] summary = root / "summary.json" for contents, diagnostic in variants: From 49307822be1f52310f7f219a355761624996aa24 Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Thu, 5 Nov 2026 22:19:14 -0800 Subject: [PATCH 13/71] docs: remove stale code storage design passages --- docs/LEAN-CTX-COMPARE.md | 2 +- docs/V2-OPERATING-MODEL.md | 2 +- docs/dashboard.md | 4 +- docs/graph-at-rest/README.md | 151 +--- docs/plans/storage/README.md | 70 +- docs/plans/storage/evidence/why/inv-3-docs.md | 4 +- docs/plans/tracedecay-v2/00-plan-set-index.md | 27 +- docs/plans/tracedecay-v2/05-query-crate.md | 7 +- ...ality-evaluation-and-retrieval-research.md | 2 +- ...cross-project-repository-worktree-scope.md | 4 +- .../25-code-intelligence-indexing-crate.md | 2 +- .../26-observability-accounting-and-usage.md | 2 +- ...1-native-fastembed-semantic-code-search.md | 708 ------------------ ...8-storage-retention-size-and-efficiency.md | 11 +- .../39-embedded-grafeo-graph-database.md | 127 +--- ...generation-storage-and-zero-copy-replay.md | 60 -- 16 files changed, 61 insertions(+), 1122 deletions(-) delete mode 100644 docs/plans/tracedecay-v2/31-native-fastembed-semantic-code-search.md diff --git a/docs/LEAN-CTX-COMPARE.md b/docs/LEAN-CTX-COMPARE.md index 80d39b1052d..5ea251f4ca3 100644 --- a/docs/LEAN-CTX-COMPARE.md +++ b/docs/LEAN-CTX-COMPARE.md @@ -12,7 +12,7 @@ through its daemon and product contract. | Concern | TraceDecay | lean-ctx | |---|---|---| | Code and evidence | A daemon-owned authority serves code, project-memory, and session evidence with exact provenance, generation, coverage, and receipts. | Describes a local context layer for read modes, shell output, request-context compression, and context-use visibility. | -| Graph and durable state | Grafeo owns admitted graph/vector data; SQLite owns relational/content records. Clients do not open either store directly. | Describes a property graph and portable context packages alongside its context-management features. | +| Graph and durable state | Grafeo owns admitted graph data; SQLite owns relational/content records. Clients do not open either store directly. | Describes a property graph and portable context packages alongside its context-management features. | | Session context | Lossless Context Memory retains raw-message evidence and summary nodes, with bounded replay and drill-down through daemon operations. | Describes persistent session memory and recovery across chats. | | Workflow and multi-agent work | The V2 product includes the daemon-owned work graph and typed workflow runtime: task evidence, ownership, authorization, execution, cancellation, recovery, collaboration, and receipts are product data, not host scripts. | Describes agent handoff and shared-context features. | | Operating model | Hosts submit bounded hints and typed operations; daemon convergence, freshness, and maintenance remain explicit. | Describes agent wrappers, shell hooks, and an optional request proxy that can mediate context on the read and model-request paths. | diff --git a/docs/V2-OPERATING-MODEL.md b/docs/V2-OPERATING-MODEL.md index 62d812a2f8a..d11149e1472 100644 --- a/docs/V2-OPERATING-MODEL.md +++ b/docs/V2-OPERATING-MODEL.md @@ -10,7 +10,7 @@ capabilities are available. ## Authorities - `tracedecay-graph-db` is the only TraceDecay boundary for the workspace-pinned - embedded Grafeo `=0.5.42` runtime. It exposes typed TraceDecay identities, + embedded Grafeo runtime. It exposes typed TraceDecay identities, operations, snapshots, generations, and outcomes; Grafeo handles and types do not escape it. - Grafeo is in-process and embedded. It has no graph server, sidecar, network diff --git a/docs/dashboard.md b/docs/dashboard.md index 9ebc37ea9b8..b84cfb6d71b 100644 --- a/docs/dashboard.md +++ b/docs/dashboard.md @@ -25,8 +25,8 @@ repository and generation provenance, coverage, and receipts. Warming, partial, unavailable, denied, cancelled, retained, and recreation-required outcomes are distinct UI states, never a blank successful panel. -Structure views present Grafeo-backed graph and vector results supplied by the -daemon. Conversation and memory views hydrate authorized content from its owning +Structure views present Grafeo-backed graph results supplied by the daemon. +Conversation and memory views hydrate authorized content from its owning authority. Work controls separate proposal, authorization, execution, and repository effects. Diagnostics are read-only; maintenance actions are explicit daemon operations with authorization and receipts. diff --git a/docs/graph-at-rest/README.md b/docs/graph-at-rest/README.md index 5a17164514e..d0ad8837a24 100644 --- a/docs/graph-at-rest/README.md +++ b/docs/graph-at-rest/README.md @@ -1,9 +1,5 @@ # An at-rest form for the graph: measurement and verdict -Bench-lane investigation. Nothing here is wired into production; the two -harnesses are `#[ignore]`d and the one new crate method is gated on -`test-helpers` so it cannot compile into a shipped binary. - ## The question Activating a sealed generation replays every node and edge into grafeo's in-RAM @@ -31,17 +27,8 @@ This investigation measured what calling it would buy. > and served read-only. That dissolves blocker 2 (nothing ever writes to a > sealed artifact), the pinned fork's compact-store property hash index > dissolves blocker 1 (sealed reopens register `INDEXED_PROPERTIES` exactly -> like staging opens), and vector search deliberately never routes to sealed -> stores. Bytes round-trip the pinned dictionary codec losslessly now, but -> `COMPACT_ROUND_TRIPS_BYTES` stays **off**: a real 431-file code generation -> (~45k entities carrying serialized-record Bytes payloads) fails its -> post-reopen recovered-digest proof in compact form with "relation scalar -> endpoints do not match native topology" and stays permanently unseated -> behind activation retries, while the same corpus seals and serves in replay -> form; the toy-scale sealed-store contract compacts and reads exactly, so -> the defect only shows at generation scale. This document remains the -> measurement record for the whole-database compaction that was measured and -> rejected. +> like staging opens). This document remains the measurement record for the +> whole-database compaction that was measured and rejected. > > **Refreshes seal as layers (graph format 7):** a small refresh no longer > re-encodes its whole generation. Its sealed store is a delta container @@ -56,18 +43,6 @@ This investigation measured what calling it would buy. > becomes the next base. Stores sealed under format 6 are superseded and > rebuild cold; no migration exists. -## Verdict - -**`compact()` cannot be applied to TraceDecay's graph as it is currently -shaped.** Not "does not pay off", it returns an error. - -The blocker is TraceDecay's own schema, not grafeo. And under a schema that -does not trip it, the columnar form is a large, real win: **10.9x faster -activation at 1M rows, 2.4x lower resident memory, 185x faster adjacency.** - -So the at-rest work is worth doing, but the first ticket is a schema change, -not a storage change. - ## Numbers All figures from a single box, `--test-threads=1`, one scenario per process @@ -308,11 +283,6 @@ close/reopen verification. Small: the crate already holds `compact_snapshot_for_bench` does today. Everything downstream (section write, section reload, layered rewiring) is already implemented in grafeo. -**Phase 3, restore vector-index planning.** Either override -`has_vector_index` on the layered store so it consults the overlay and the -vector section, or hoist the check in `src/runtime.rs` off `graph_store()`. -Small, but do not skip it: gap 3 fails silently. - **Phase 4, optional fork work.** Streaming section serialize (gap 5) and mmap-backed container open (gap 6). Both are pure wins on top of a working path, and neither is needed for the activation number. @@ -424,104 +394,6 @@ contract failures, a sealed generation reopened twice came back with its format marker gone, which is the same defect reached through TraceDecay's own lifecycle rather than the minimal probe. -### The vector index is durable now, fork branch ready, pin not moved - -The claim above that the HNSW index "was never durable across a reopen -anyway" was true, and it turned out to be a wiring gap rather than a -missing format. - -**Where durability was lost.** grafeo has always *written* the index. -`build_sections` emits a `SectionType::VectorStore` carrying each index's -full HNSW topology, entry point, max level, per-node per-level neighbour -lists, and `CatalogSection::collect_indexes` records every index's -label, property, dimensions, metric, M and ef_construction beside it. The -restore side dropped both: - -- `CatalogSection::deserialize` handed the loader its `property_indexes` - and nothing else. The `vector_indexes` it had just decoded were - discarded (`catalog_section.rs`). -- `load_from_sections` guarded the topology restore on - `store.vector_index_entries()` being non-empty - (`database/mod.rs`, the `SectionType::VectorStore` block). A cold open - builds its `LpgStore` from nothing, so that map is empty at exactly the - moment the guard runs. The section bytes were read off disk and - dropped on the floor. - -So the index was durable on disk and absent in memory, and every reopen -re-indexed the whole corpus. - -**Fork, not sidecar.** The section format round-trips HNSW faithfully -already: topology is all it needs to carry, because the vectors live in -LPG node properties (persisted anyway) and are read through a -`VectorAccessor` at query time. Nothing was missing from the format, only -the restore wiring, a tracedecay-side sidecar would have duplicated a -working serializer to re-derive an artifact grafeo already had on disk, -and left every other consumer broken. - -**Branch:** `tracedecay/0.5.42-vector-index-durable`, stacked on -`tracedecay/0.5.42-close-and-overlay`, at -`f38218653dfc69fda67f9c669371036fde1ed5fe`. Three commits: - -1. re-register catalog definitions as empty indexes and let the section - fill their topology. Two incomplete-restore cases are deliberately - left absent for rebuild rather than half-restored, a definition with - no section, and a definition the section carried no topology for, - because an index that exists and covers nothing reports - `has_vector_index() == true` and answers every search with nothing. - Quantized indexes are left to a rebuild too: their codebook lives - inside the index and the section carries only topology, so restoring - one would silently downgrade it to full precision. The catalog - snapshot is v2 to record the mode; v1 files still load. -2. mark index creation, removal, and binding changes as unlogged state. - `close` may skip its flush when the WAL has not moved, and nothing - about an index reaches the WAL, so an index built after open never - reached the file, and the next open rebuilt it, and the next. Harmless - while indexes were always rebuilt; load-bearing the moment they are - restored. -3. price it: 5,000 x 64-dim vectors, release profile, **8.7ms** to - restore the index as part of opening the whole database against - **359.6ms** to rebuild the index alone on a database already open. - ~41x, and identical neighbour sets across 24 probes. - -The branch also adds an opaque per-index **binding token** that rides the -catalog section, so a caller can stamp the generation digest its index -was built over and compare after a reopen instead of trusting that a -restored topology still describes the rows. - -**The pin has moved.** The vector-durability commits were reintegrated -onto the fork's `fix/catalog-version-guard` branch (ported over the -out-of-place checkpoint and close-skip storage rewrite that landed in -the meantime) and the five `grafeo-*` revs in `[patch.crates-io]` now -point at that lineage. The reopen tests in `tests/graph_db_suite/runtime_contract.rs` -assert `Available`, restored, within the same admission bound that -proves no rebuild ran, and the fork gates the semantics with -`vector_index_reopen`, `vector_index_restore_cost`, and -`torn_vector_checkpoint` (a checkpoint killed at every injection point -over a populated HNSW index must reopen and serve search). - -### `has_vector_index`, resolved, and worse than gap 3 described - -Gap 3 said `CompactStore` inherits the trait default `false`. The layered store -is more specific than that: `LayeredStore::has_vector_index` delegates to the -*overlay* (`compact/layered.rs:1081`), which `compact()` leaves empty, so it -answers `false` for indexes the store had a moment earlier. That part is at -least truthful, the HNSW index really is gone, and it was never durable across -a reopen anyway. - -The silent failure is one layer along. `create_vector_index` builds by scanning -`nodes_by_label(label)`, and TraceDecay's vector label is -`entity_projection_label`, one of several labels on an entity node. On a -compacted base that scan matches nothing, so the build succeeds over zero -vectors, `has_vector_index` then answers `true`, and every vector search -returns empty while reporting an index exists. - -The resolution, when phase 2 becomes unblocked, is the routed one: refuse to -build an HNSW index over a store serving from a columnar base, with a typed -error rather than an empty index, and keep a generation that carries vector -indexes in its replay form. Both halves were implemented against the seal-time -compaction and reverted with it; neither is reachable while nothing calls -`compact()` outside the bench lane. - ### Why "never write after compact" is not a guarantee TraceDecay can make The obvious containment for blocker 2 is to promise the store is never mutated @@ -563,19 +435,10 @@ the heap-copy path and are a floor, not a ceiling. every reopened store ran without them. That was invisible while identity resolved through the intrinsic label index, and cost 23.7s for 64 point reads at 500k once it did not. -- **Phase 2, seal implies compact. Landed as per-generation stores; compact - form still gated.** The whole-database `compact()` this plan scoped stays - rejected; the sealed-store lane (`src/sealed_store.rs`) seals each - generation into its own single-generation database, exactly the - "per-generation stores" design blocker 2 demanded, and compacts it when - the rows carry no Bytes or Vector properties. The fork's compact property - index settles blocker 1's economics and the pinned dictionary codec - round-trips Bytes losslessly, but Bytes-carrying generations stay in replay - form behind `COMPACT_ROUND_TRIPS_BYTES` until a generation-scale compact - seal passes its post-reopen proof (see the constant's doc for the measured - scale failure). -- **Phase 3, vector planning. Not applicable to sealed stores:** vector - search never routes to a sealed artifact; HNSW indexes are rebuilt against - the staging database (`apply_sealed_copy_batch` doc). +- **Phase 2, seal implies compact. Landed as per-generation stores.** The + whole-database `compact()` this plan scoped stays rejected; the sealed-store + lane (`src/sealed_store.rs`) seals each generation into its own + single-generation database, exactly the "per-generation stores" design + blocker 2 demanded. - **Phase 4, streaming section writer, mmap container open.** Unchanged, and now clearly behind a property index for the columnar base in priority. diff --git a/docs/plans/storage/README.md b/docs/plans/storage/README.md index 67e644e9a0f..e1ad1596656 100644 --- a/docs/plans/storage/README.md +++ b/docs/plans/storage/README.md @@ -158,14 +158,10 @@ this redesign does not need to settle; the last cutover step measures it. initialized subdirectory of a worktree (a monorepo subproject, which today's `discover_project_root` supports), the pair of root commit and relative path. On this host 7 of the 14 tracedecay clones have a local-path origin, 5 https and - 2 ssh, so only the root commit converges all of them; the 14 clones and 32 linked - worktrees resolve to one project. + 2 ssh; the 14 clones and 32 linked worktrees resolve to one project. - A project never re-keys. An unshallowed clone, a first commit in an unborn repository, or a new root after a fetch adds an alias to the existing project. So durable memory is never stranded under an old id. -- `tracedecay project link ` and `tracedecay project split` record an - explicit alias or refusal in `catalog.db`, scoped to this profile, for forks that - should not share and unrelated repositories that should. - A cwd outside any repository and not explicitly initialized belongs to `projects/_profile/`. It never mints a project of its own. - Code indexes stay per worktree in `worktrees//`. Segments are @@ -275,9 +271,9 @@ in. reconnect. Batched writes take one savepoint per job, so a job's ordinary error rolls back only that job. A suspect-connection error rolls back the whole batch; each job of the batch then runs alone on a fresh connection. -- One fixed pool, `tracedecay-store-N` (default 6, clamp 2 to 16), serves every - file. A lane is per-file state: a write queue (the existing deficit-round-robin - `FairQueue`), a read queue, one write connection, at most two read connections, +- One fixed pool, `tracedecay-store-N`, serves every file. A lane is per-file + state: a write queue (the existing deficit-round-robin `FairQueue`), a read + queue, one write connection, at most two read connections, and fence state. Any pool thread can run any lane; at most one thread holds a lane's write connection at a time. One thread is reserved for foreground classes (catalog, journal, memory) so bulk derivation cannot starve Work. Open lanes are @@ -368,24 +364,10 @@ Each growth source from #4099: Each file carries `user_version` and a schema digest. The rule follows the class: - authoritative and durable classes (catalog, journal, memory, evidence): - forward-only, additive migrations in the crate, applied by the executor at open. - A non-additive change copies into `.next` and renames over the file; no - copy is kept. No user-authored state is reset by a schema change. + admit only the exact final shape. Any other shape returns typed + `ResetRequired`; there is no migration. - rebuildable classes (derived, code): a mismatch means unlink and rebuild. -Upgrade path for beta installs: `tracedecay wipe --stale` plus `tracedecay init` -remains the reset for anything this design does not carry. Durable classes are -carried, not reset. Because the store key stays `ProjectId` and the directory stays -`projects//`, each project converts in place the first time the new -binary opens it: `memory_v2_*` rows and the anchors they cite move from -`tracedecay.db` to `memory.db`, workflow definitions and native-integration -approvals move from `sessions.db` to `journal.db`, configuration revisions move from -`global.db` to `catalog.db`, and the old files are deleted after the copy commits. -Evidence is re-ingested from host files rather than copied. The conversion runs once -per file, is deleted two releases later, and is not a compatibility reader: no code -reads the old files after the conversion. The operator host was wiped on 2026-10-16 -and needs none of it. - ### 8. Public port surface | Type | Public methods | @@ -512,7 +494,8 @@ Act on, and the change made: `forget` deletes durable data. - Evidence and journal were reset or unlinked on a schema change, destroying LLM summaries, history past host retention, workflow definitions and approvals (A, B, - C). All authoritative and durable classes now migrate forward; corruption salvages. + C). Authoritative and durable classes now admit only the exact final shape, and + corruption salvages. - Facts cite retrieval anchors through a foreign key, so anchors could not live in evictable evidence (B). Cited anchors live in `memory.db`. - `FnOnce` jobs made the reconnect-and-rerun ladder and fence reruns impossible, @@ -549,15 +532,11 @@ blocking pool; that hop is intentional and the executor step removes it. byte on disk. - We accept losing cross-page snapshot consistency, because every read is a short job, in exchange for a total checkpoint fence. Pagination uses keyset cursors. -- We accept that forks sharing a root commit share a project until `project - split`, in exchange for clones converging with no configuration. - We accept partial derived search during a derived rebuild, in exchange for corruption and schema changes that heal without an operator. - We accept a 30 s ceiling per write job; bulk work pages. - We accept Grafeo as an immutable file format. This is not a second engine: it has no WAL and no mutation, like the code-index generation files. -- We accept a one-shot in-place conversion of durable rows for two releases, - because "durable memory survives" forbids a reset of that class. ## Alternatives considered @@ -588,10 +567,8 @@ changes a file that holds memory. it, and no read parks a native worker. Proof: a native task awaiting a typed read leaves the native reader-wait counter unchanged (a mutation that reads inline fails the test). -2. Memory durability. The stores that hold memory run at FULL; the memory tables - get additive migrations instead of `ResetRequired` with their first schema - change. Proof: those stores' writers report `synchronous=FULL` and the others - `NORMAL`. +2. Memory durability. The stores that hold memory run at FULL. Proof: those + stores' writers report `synchronous=FULL` and the others `NORMAL`. 3. Growth stops now. Worktree liveness by root instead of the shared `.git`, a collector for unregistered directories that never deletes durable data, response-handle sweeps every tick, `hook_analytics.jsonl` rotation, age-pruned @@ -618,14 +595,12 @@ changes a file that holds memory. `ProfileLost` within one tick with the descriptors closed. 8. Partition the sessions file and identity by repository. Split each project into journal, memory, evidence and derived; telemetry leaves the catalog; identity - adopts through aliases. The in-place conversion of durable rows ships here. - Per-stream cursors and dead letters replace the global queue: row poison - dead-letters per stream, and corrupt derived authority (ghost or divergent - provenance) rebuilds `derived.db` instead of being skipped. Delete + adopts through aliases. Per-stream cursors and dead letters replace the global + queue: row poison dead-letters per stream, and corrupt derived authority (ghost + or divergent provenance) rebuilds `derived.db` instead of being skipped. Delete `projection_queue`, `lcm-payloads/`, the string classifier, the store locator resolver and standalone registries. Proof: two full clones and a linked worktree - resolve to one project with three worktree directories; converted facts, - workflow definitions and approvals read back identically; a poisoned row + resolve to one project with three worktree directories; a poisoned row dead-letters while other streams project; corrupt provenance triggers a derived rebuild and the stream resumes. 9. Graph narrowing. The memory graph and session-relation graphs move to SQLite @@ -647,14 +622,11 @@ changes a file that holds memory. - Plan 02's "No asynchronous database API" and its interactive repository transactions: every storage API is async and every transaction is one job. -- Plan 38's "Store soft budgets are owner-configured and inert by default": budgets - have defaults and are enforced by `Storage::maintain`. +- Store soft budgets have defaults and are enforced by `Storage::maintain`. - Plan 39's move of graph-shaped data off SQLite: mutable graph data (memory and session relations) lives in SQLite; Grafeo remains an immutable sealed format. - Plan 40's monolithic per-worktree generation segments: segments are content-addressed per project. -- The roadmap's reset-only rule for released stores, for the durable and - authoritative classes: they migrate forward; derived and code still rebuild. - The 2026-07-15 fold of sessions into the project-wide store, which had no recorded reason: ADR-003's failure-domain split is restored by class. @@ -675,9 +647,7 @@ changes a file that holds memory. profile memory store (`user-memory.db`), commit at `synchronous=FULL`; every other store keeps `NORMAL`. `global.db` stays `NORMAL` although the catalog class is FULL, because it still takes per-hook analytics writes until telemetry splits out - in step 8. The additive-migration rule for memory ships with the first change to a - memory table (the conversion in step 8), not as machinery with no migration to - run. The class contract is unchanged. + in step 8. The class contract is unchanged. - Step 3 (2026-10-16, done). Four PRs stopped the growth this step names: - #4120. Orphan-store liveness no longer counts the shared git common directory, so a deleted worktree's store becomes collectable while its @@ -1021,11 +991,3 @@ changes a file that holds memory. get it; it is deferred until a silent-corruption incident is on record. - Are 64 GiB per profile and 8 GiB of evidence per project the right defaults? Step 10 calibrates them from the first weeks of persisted usage. - -## Next implementation step - -The rest of step 5: the native writer-wait counter becomes a refusal of any -synchronous writer wait on a native thread and both counters are deleted; then -the interactive exact-SQL transactions (56 direct sites and about 190 behind the -runtime-core async `Transaction`) become jobs. Step 7's first slice, writers and -readers as tasks on one fixed pool, is in progress in parallel. diff --git a/docs/plans/storage/evidence/why/inv-3-docs.md b/docs/plans/storage/evidence/why/inv-3-docs.md index 822c5db1444..ce9449411a1 100644 --- a/docs/plans/storage/evidence/why/inv-3-docs.md +++ b/docs/plans/storage/evidence/why/inv-3-docs.md @@ -107,13 +107,13 @@ cross-references, "page history" is `git log`/pickaxe on `docs/`. 9. **Stated goal: move graph-shaped data off SQLite and custom adjacency structures.** - "**Goal:** Replace custom adjacency structures and graph-shaped SQLite storage with one embedded Grafeo runtime boundary while retaining SQLite only for genuinely relational, transactional, and content-bearing records." - - Where: `docs/plans/tracedecay-v2/39-embedded-grafeo-graph-database.md` line 12, unchanged since the first version 955552d44b (ScriptedAlchemy, 2026-08-03, "docs(v2): plan embedded Grafeo graph cutover"). The code commit 0e3284dd2f ("feat(query): embed Grafeo graph projection") landed earlier the same day. All of these commit messages have empty bodies. + - Where: `docs/plans/tracedecay-v2/39-embedded-grafeo-graph-database.md` goal, unchanged since the first version 955552d44b (ScriptedAlchemy, 2026-08-03, "docs(v2): plan embedded Grafeo graph cutover"). The code commit 0e3284dd2f ("feat(query): embed Grafeo graph projection") landed earlier the same day. All of these commit messages have empty bodies. - Plan index (00, lines ~1003-1013): "graph-shaped data moves off custom adjacency structures and SQLite. Relational journals, receipts, raw content, configuration, and execution fencing remain in SQLite." - Relevance: the only stated "why". No doc compares Grafeo against SQLite recursive CTEs or adjacency tables on a measured workload before adoption. Task 12 ("Compare against the pre-Grafeo baseline") remained unchecked, and the 2026-08-14 audit lists "E18 POST-RC Grafeo aggregate/performance provenance ... pre-Grafeo performance comparison" as outstanding (deleted `audits/v2-implementation-audit-2026-08-14.md`). 10. **Plan 40 states what the pre-V2 SQLite graph did well and what V2 adds.** - "Master (pre-V2) proved the opposite trade-offs work. SQLite adjacency graph, per-file incremental sync, no stored body text, open-and-serve startup, but lacked what V2 genuinely adds: immutable digest-sealed generations, verify-then-publish with a watermark, exact branch/revision identity. Doc 39 already assigns the durable graph to Grafeo; the cutover never completed, leaving the sealed JSON as both canonical authority **and** de facto serving store." - - Where: `docs/plans/tracedecay-v2/40-partitioned-generation-storage-and-zero-copy-replay.md` §Problem (status "ACTIVE (decided 2026-08-22)"). + - Where: `docs/plans/tracedecay-v2/40-partitioned-generation-storage-and-zero-copy-replay.md` §Problem. - Measured trigger in the same doc: "One sealed code generation for this repository is a single `generation-.json` of **1.39 GB**"; "Warmup on this repository read tens of GB, pinned the 8 GiB cgroup boundary, and starved queries." - Relevance: the doc credits immutable generations and verify-then-publish to V2's design. It credits the earlier SQLite graph with good operational economics. diff --git a/docs/plans/tracedecay-v2/00-plan-set-index.md b/docs/plans/tracedecay-v2/00-plan-set-index.md index 72e8ce80b24..20bc503d1d3 100644 --- a/docs/plans/tracedecay-v2/00-plan-set-index.md +++ b/docs/plans/tracedecay-v2/00-plan-set-index.md @@ -106,13 +106,9 @@ invariants and do not by themselves keep a journey open. receipt-owned paths and a drift-converges acceptance test. Operational closure still requires the measured doctor journey and a clean Cursor install/upgrade/doctor pass on an operator machine.) -- Semantic search is disabled by an invalid configuration snapshot. Plan 20 - owns final-snapshot validity and explicit reset/recreation; Plan 31 owns - semantic activation. - Exact, lexical, and graph retrieval remain the required available fallback. - This snapshot does not mean semantic search is product-disabled. - (Superseded 2026-09-13: dense semantic retrieval is retired by rejected - decision 11; there is no activation to repair.) +- Dense semantic retrieval is retired by rejected decision 11. Exact, lexical, + and graph retrieval remain the required available authorities; there is no + semantic activation to repair. - A live profile was observed 237 minutes stale (the index later reported 285 minutes stale while this reconciliation was being checked). Plan 25 and the active incremental-indexing slice own cadence/freshness diagnosis; the @@ -275,8 +271,8 @@ records the rejected mechanism, the reason, and the retained replacement: installation downloads no model, loads no inference runtime, builds no vectors, and exposes no activation workflow. Released relational stores retain their data under the admission exception above; dense retrieval - is never reactivated. Plan 31 is retired as an authority; its lexical-fallback - and exact-tier obligations already belong to Plans 05, 15, and 25. The + is never reactivated. The lexical-fallback and exact-tier obligations + already belong to Plans 05, 15, and 25. The retained replacement is exactly three code-intelligence authorities and no fourth "semantic" one: lexical search (exact, fielded, phrase, proximity, bounded identifier/path splitting, and named query-time aliases that stay @@ -932,8 +928,7 @@ Unmeasured speculative optimizations and placeholder benchmarks do not ship. [15](15-search-quality-evaluation-and-retrieval-research.md), [23](23-session-lcm-temporal-retrieval-and-evaluation.md), and [25](25-code-intelligence-indexing-crate.md). - [Plan 31](31-native-fastembed-semantic-code-search.md) is retired by - rejected decision 11 and is archival. + Dense retrieval is retired by rejected decision 11. Indexing runtime is `tracedecay-code-index-runtime`; generation retention is `tracedecay-code-index-retention`. - Policy and delivery surfaces: [Plans 06](06-policy-crate.md), [08](08-tool-catalog-crate.md), @@ -999,8 +994,7 @@ Unmeasured speculative optimizations and placeholder benchmarks do not ship. [Plan 39](39-embedded-grafeo-graph-database.md)'s embedded graph storage cutover and [Plan 40](40-partitioned-generation-storage-and-zero-copy-replay.md)'s - partitioned generation storage, which supersedes the monolithic sealed - generation envelope as the long-term storage design. + partitioned generation storage. Embedded graph storage ([Plan 39](39-embedded-grafeo-graph-database.md)) threads through the active @@ -1031,10 +1025,9 @@ debris quarantine, and one-content-copy enforcement. Session retention keeps raw and projected evidence deduplicated while LCM summary nodes retain losslessly expandable source references. Plan 09 exposes the corresponding storage findings through read-only Doctor composition over Plan 26 telemetry; -Doctor never invokes cleanup, GC, retention, relink, or recovery. Store soft -budgets are owner-configured and inert by default. Direct validation uses an -isolated profile and proves size reduction without reading or mutating the -operator profile. +Doctor never invokes cleanup, GC, retention, relink, or recovery. Direct +validation uses an isolated profile and proves size reduction without reading +or mutating the operator profile. The remaining delivery proceeds by complete product journeys under this index. Branch, pull-request, merge, worktree, and SHA choreography is not roadmap diff --git a/docs/plans/tracedecay-v2/05-query-crate.md b/docs/plans/tracedecay-v2/05-query-crate.md index a563a0af211..3c4a0fcf868 100644 --- a/docs/plans/tracedecay-v2/05-query-crate.md +++ b/docs/plans/tracedecay-v2/05-query-crate.md @@ -22,15 +22,14 @@ same-host measurement showing a smaller frequently touched compile graph. - The memory delivery adds facts and provenance, the temporal-kernel delivery adds LCM/session retrieval, the code-intelligence delivery adds - lexical code search, code navigation, and current diagnostics, and the - semantic delivery adds semantic search. + lexical code search, code navigation, and current diagnostics. - The application delivery composes query use cases in application and policy. The surface delivery exposes them through CLI, MCP, HTTP, LSP, export, and live non-dashboard adapters. The dashboard delivery first ships dashboard binding and dashboard parity. - Plan 05 owns the shared execution semantics. Plan 23 remains the current - owner of temporal retrieval semantics; Plans 25 and 31 own active - lexical/code and semantic acceptance. Plans 09/10/11/12/14 and Plan 24 + owner of temporal retrieval semantics; Plan 25 owns active lexical/code + acceptance. Plans 09/10/11/12/14 and Plan 24 consume the accepted temporal-kernel behavior, not a superseded implementation shape. - The task/work delivery journey reuses these shared scope, budget, cancellation, cursor, watermark, merge, and coverage primitives to execute diff --git a/docs/plans/tracedecay-v2/15-search-quality-evaluation-and-retrieval-research.md b/docs/plans/tracedecay-v2/15-search-quality-evaluation-and-retrieval-research.md index 58ab928292b..dcfdb4df2c5 100644 --- a/docs/plans/tracedecay-v2/15-search-quality-evaluation-and-retrieval-research.md +++ b/docs/plans/tracedecay-v2/15-search-quality-evaluation-and-retrieval-research.md @@ -20,7 +20,7 @@ graph, Plan 13 diagnostic anchors, or their authorization rules. Plan 15 owns retrieval quality, composition, evaluation, and profile-selection rules. Plan 25 owns code retrieval and shared-code facts. The application, dashboard, task/work, and public surface plans consume tested behavior and -evaluation results. Plan 31 is archival. +evaluation results. ## Outcome diff --git a/docs/plans/tracedecay-v2/16-cross-project-repository-worktree-scope.md b/docs/plans/tracedecay-v2/16-cross-project-repository-worktree-scope.md index 12ffb77a405..de39953e69f 100644 --- a/docs/plans/tracedecay-v2/16-cross-project-repository-worktree-scope.md +++ b/docs/plans/tracedecay-v2/16-cross-project-repository-worktree-scope.md @@ -49,8 +49,8 @@ right data. ## Worktree index identity and reuse -Plan 16 composes the incremental index delivered by Plans 25/31; it does not -create a second watcher, parser, chunker, vector store, or scheduler. +Plan 16 composes the incremental index delivered by Plan 25; it does not +create a second watcher, parser, chunker, or scheduler. - Every linked worktree has an exact typed identity, fenced snapshot frontier, immutable code generation, freshness, and coverage. A shared Git common diff --git a/docs/plans/tracedecay-v2/25-code-intelligence-indexing-crate.md b/docs/plans/tracedecay-v2/25-code-intelligence-indexing-crate.md index 0fc40d61ea2..d4f988dfacf 100644 --- a/docs/plans/tracedecay-v2/25-code-intelligence-indexing-crate.md +++ b/docs/plans/tracedecay-v2/25-code-intelligence-indexing-crate.md @@ -75,7 +75,7 @@ regresses.) Plan 25 owns code-generation, chunking, graph, and generation-bound evidence semantics, including source-bound shared-code facts sealed into each generation. Plan 15 owns quality evaluation. Application, transport, and -dashboard plans consume the tested operations later. Plan 31 is archival. +dashboard plans consume the tested operations later. ## Outcome diff --git a/docs/plans/tracedecay-v2/26-observability-accounting-and-usage.md b/docs/plans/tracedecay-v2/26-observability-accounting-and-usage.md index 42424860fca..7acdf3d377b 100644 --- a/docs/plans/tracedecay-v2/26-observability-accounting-and-usage.md +++ b/docs/plans/tracedecay-v2/26-observability-accounting-and-usage.md @@ -145,7 +145,7 @@ vocabulary as current ownership. cancellation; analyzer startup, restart, and indexing/degraded state; cache reuse and overlay freshness; diagnostic add and clear; provider conflicts; host delivery path; partial coverage and drops; and bridge reconnect. -- Plans 25/31 emit bounded incremental-index observations for exact worktree +- Plan 25 emits bounded incremental-index observations for exact worktree identity classes without paths: event-to-reconcile and event-to-ready latency; debounced/raw event counts; overflow/rescan reason; candidate, hashed, parsed, and no-op file counts; Tree-sitter changed-range bytes; diff --git a/docs/plans/tracedecay-v2/31-native-fastembed-semantic-code-search.md b/docs/plans/tracedecay-v2/31-native-fastembed-semantic-code-search.md deleted file mode 100644 index f4e0c99c151..00000000000 --- a/docs/plans/tracedecay-v2/31-native-fastembed-semantic-code-search.md +++ /dev/null @@ -1,708 +0,0 @@ -# Native FastEmbed semantic code search (archival) - -**Status:** superseded on 2026-09-13 by -[rejected decision 11](00-plan-set-index.md#rejected-approaches). This file -records the retired dense-retrieval design. It is not current implementation, -product, storage, or test authority. - -TraceDecay no longer downloads code-search models, loads an inference runtime, -builds vector generations, exposes vector search, or runs semantic activation, -qualification, calibration, reranking, or rollback. Plans 05, 15, and 25 own -the retained exact, lexical, graph, and source-bound shared-code behavior. - -The content below is historical. It must not be used to restore a fourth -semantic code-intelligence authority. - -## Outcome - -TraceDecay augments exact lexical and graph search with local code embeddings. -Exact results remain authoritative without a model; similarity alone never proves impact, lineage, or equivalence. - -## Ownership - -- Plan 25 builds deterministic, storage-neutral `CodeSearchDocumentV1`, - `CodeSearchChunkV1`, `ChangedCodeChunkSetV1`, and - `CodeIndexCapabilityManifestV1` values from an exact code snapshot. -- Plan 04 defines deterministic changed-input projection, checkpoint, retry, - and publication semantics. Daemon/service orchestration decides when that - resumable work is scheduled. -- Plan 02 stores immutable vector generations, manifests, checkpoints, the - atomic active-generation pointer, and chunk projection receipts through - daemon-owned writer authority. -- Plans 05/15 own the common `CompactCandidate`, independent lane, deterministic - fusion, contribution, diversity, rerank, hydration, cursor, fallback, - explanation, and redundancy contracts shipped by federated retrieval. This plan adds one semantic - adapter; it does not create a parallel code-search kernel. -- Plan 20 owns versioned model/reranker profile selection, configuration - precedence, and atomic active/rollback profile pointers. This plan owns the - root-private versioned artifact manifest and SHA-256 integrity verifier, - FastEmbed runtime adapter, - bounded sessions, semantic generation service, and their typed ports. The - Plan 09 application layer later composes those accepted ports; it does not - retroactively own this plan's model implementation. -- Daemon/application orchestration schedules changed eligible chunks and - rebuilds. Plan 04 owns deterministic projection, checkpoint, retry, and - publication semantics, not scheduling policy. -- Plan 15 exclusively owns retrieval-research design, versioned corpora and - labels, metrics and strata, candidate profile comparison, thresholds, and - activation recommendations. This plan implements measured profiles and emits - results; it cannot tune or activate itself from aggregate or public benchmark - rank. -- Plans 10/11/20/21 expose the same application operations through API, - dashboard, configuration, CLI, and MCP. - -Only one root-private adapter depends on `fastembed`. Crates for indexing, store, -query, API, and UI depend on ports and stable domain values, never -FastEmbed runtime types. - -## Behavioral contracts and historical layout - -The semantic boundary must preserve the ownership and fields below. The listed -paths and Rust names are a historical design sketch, not artifact-name parity -requirements. Current owner-approved names and locations are valid when direct -contract regressions prove the same lane isolation, generation compatibility, -authorization, coverage, fallback, and receipt behavior. - -The original semantic-delivery design proposed these files: - -- `crates/tracedecay-domain/src/code_intelligence/search.rs`: - `EmbeddingProjectionKeyV1`, `SemanticSearchIndexKeyV1`, - `SemanticCapabilityManifestV1` and - code-specific semantic evidence values. Generic ranked-list, compact- - candidate, contribution, fusion, rerank, hydration, and cursor values remain - in federated retrieval's `crates/tracedecay-domain/src/retrieval.rs`. - Semantic projection uses Plan 25's `CodeChunkProjectionReceiptV1` and - `ProjectionBatchReceiptV1`; no semantic-specific receipt authority exists. -- `src/semantic_code/artifacts.rs`: versioned manifest parsing, SHA-256 member - integrity verification, import staging, atomic install, inventory, retention, - and quarantine. -- `src/semantic_code/manifest.rs`: model/runtime/projection and capability - manifest validation. -- `src/semantic_code/projector.rs`: changed-chunk batching, receipt production, - resumable checkpoints, and publication handoff. -- `src/semantic_code/fastembed_adapter.rs`: the only `fastembed` import and the - only model inference implementation. -- `src/semantic_code/session_pool.rs`: bounded sessions keyed by the complete - projection/privacy identity. -- `src/query/retrieval/semantic.rs`: the code-semantic implementation of the - federated `Retriever` port. It emits generic `CompactCandidate` values carrying a - typed code anchor and semantic evidence; it consumes neither lexical nor - graph candidates. -- `src/semantic_code/service.rs`: temporary root-private semantic orchestration for - artifact/runtime admission, projection generation, shadow execution, and - activation/rollback. It exposes typed ports consumed by the application - layer and contains no transport binding. -- `src/config/retrieval.rs`: Plan-20-owned versioned profile definitions and atomic - active/rollback profile pointers. -- `src/store/vector_generations.rs`: Plan-02-owned implementation of the - semantic projection read/write ports. Query and semantic modules do not - depend on its physical schema. - -The semantic adapter extends the federated retrieval contract rather than replacing -it: - -```rust -pub struct SemanticRetrievalRequestV1 { - pub query_digest: QueryDigest, - pub query_view: EphemeralSanitizedQueryViewV1, - pub authorized_scope: AuthorizedScopeReceiptV1, - pub authorization_epoch: AuthorizationEpoch, - pub scope_digest: ScopeDigest, - pub code_generation: CodeGenerationId, - pub projection_key: EmbeddingProjectionKeyV1, - pub capability_manifest_digest: ManifestDigest, - pub budget: RetrievalBudget, - pub continuation: Option, -} - -pub struct SemanticCodeRetriever { - // Root-private FastEmbed session and semantic projection read ports. -} - -// SemanticCodeRetriever implements -// Retriever. - -pub struct CodeSemanticEvidenceV1 { - pub projection_key: EmbeddingProjectionKeyV1, - pub vector_generation: VectorGenerationId, - pub chunk_id: CodeSearchChunkId, - pub distance: CanonicalSemanticDistanceV1, - pub search_kind: SemanticSearchKindV1, -} -``` - -The adapter implements the one federated `Retriever` port and receives one -frozen authorized scope, code generation, semantic -projection, and bounded ephemeral sanitized query view. It returns the generic -Plan 15 batch with one `CompactCandidate` and one occurrence-keyed -`CodeSemanticEvidenceV1` per result. It cannot mint exact -admission, read another lane's candidates, fuse, diversify, rerank, hydrate, or -retain raw query text/query vectors. Store conformance uses separate lexical, -graph, vector, receipt, and hydration representations; no table, vector index, -or materialized join becomes authority. -Deadline and cancellation limits are fields of the shared `RetrievalBudget`; -this plan does not introduce semantic-only budget or continuation types. - -## Deterministic documents and generations - -Each chunk records repository/project/worktree/ref/snapshot identity, immutable -code generation, file and symbol identity, source span, language/extractor and -chunker versions, sensitivity decision, content digest, stable ordinal, and -bounded sanitized text. Symbol boundaries are preferred; oversized symbols use -versioned structural splits. Generated, vendor, binary, ignored, fixture, and -unsupported content has an explicit classification. - -A vector-generation identity includes the ordered eligible-document manifest, -model/tokenizer/runtime manifest, dimension, metric, normalization, chunker, -privacy domain/key epoch, and source watermark. The manifest also pins query -and document instructions/prefixes, pooling, truncation side and length, -precision/quantization, runtime/backend/thread/device identity, and search or -ANN parameters when present. Builds checkpoint in bounded batches. Partial or -mixed generations are never queryable. Publication verifies membership, -dimensions, finite values, digests, and watermark before one atomic pointer -swap; deletion creates a tombstone and unchanged inputs do no embedding work. - -`EmbeddingProjectionKeyV1` pins every vector-affecting input: model artifact, -tokenizer/config and instruction digests, pooling, truncation side/length, -runtime/backend/build revision, deterministic device class, dimension, metric, -normalization, precision/quantization, chunk schema/chunker revision, privacy -domain, and key epoch. Its canonical digest is Plan 25's -`ProjectionKeyV1.profile_digest`. Search/ANN structure parameters use a -separate `SemanticSearchIndexKeyV1`; changing them rebuilds only derived search -structures and query caches. Thread/batch settings remain execution-manifest -evidence and must produce the same vector digest or fail publication. Vectors -and projection rows are replayable derived data; canonical chunks and exact -code-generation evidence remain authority. - -For each `ChangedCodeChunkSetV1`, the projector returns one ordered -`CodeChunkProjectionReceiptV1` per added/changed/deleted chunk and an explicit -aggregate reused count in `ProjectionBatchReceiptV1`. A receipt records source -generation and manifest, projection key, chunk ID, prior/current digest, -`embed | tombstone`, completed/skipped/failed outcome, and output digest. -Deleted receipts bind `prior_generation`/`prior_chunk_digest`, the batch -`request_digest`, and have no current digest. -Reused chunks produce no per-chunk receipt. A no-op request with an unchanged -target projection key invokes no inference; a key-only replay expands reused -eligible chunks into explicit embed operations before dispatch. A one- -symbol edit embeds only its changed chunks and affected file-level chunks. A -model, tokenizer, instructions, runtime-compatibility, dimension, metric, -normalization, precision, privacy-domain, or key-epoch change creates a new -projection generation by replaying retained canonical chunks without reparsing -unchanged source. Old retained projections remain immutable and addressable by -their exact key; partial receipt sets never activate. - -### Worktree-aware projection scheduling - -- Consume Plan 25's reconciled `ChangedCodeChunkSetV1`; do not watch files, - rescan repositories, invoke Git status, or infer changes inside the semantic - runtime. -- Batch only added/changed eligible chunks through FastEmbed - `TextEmbedding::embed`. Construct the model through - `try_new_from_user_defined` from explicitly installed local ONNX/tokenizer/ - config bytes. Query and projection paths never invoke the hub, discover an - ambient model cache, or download artifacts. -- A no-op changed set performs zero model calls. Deletions produce tombstones - without inference. A projection-key change replays retained eligible chunks - without parser/extractor work; a search-index-key-only change rebuilds only - the derived exact-flat/search structure. -- Reuse vector bytes across worktrees only when canonical chunk content, - projection key, privacy domain, and key epoch match. Every projection row, - receipt, and immutable vector generation remains bound to its exact source - worktree snapshot and code generation. The committed retrieval profile names - that vector generation, while each admitted query separately pins the exact - source code generation; neither identity is inferred from a process cache. -- Coalesce superseded projection batches by exact worktree and source - generation. Bound queued bytes, batch size, session count, model memory, and - publication concurrency. Interactive exact/lexical/graph queries have - priority; semantic query admission is fail-fast and never enters the - projection waiter queue. -- Keep a prior semantic generation queryable only when its complete projection - key and source compatibility still match the frozen code request. Otherwise - omit semantics until the new complete generation atomically publishes. - -## Model and offline lifecycle - -Configuration selects a versioned embedding profile and, independently, an -optional reranker profile. With automatic acquisition enabled, the daemon -queues the selected catalog model's immutable Hugging Face repository revision -in the background. Manifests pin actual model/tokenizer/config bytes, licenses, -runtime/build identity, dimensions, normalization, metric, device, threads, -batching, and resource ceilings. Implementation selects maintained -crate/runtime versions during the semantic delivery; activated model and reranker profiles still -require a passing Plan 15 evaluation. This plan contains no stale crate or -model-version pin. - -Every acquisition/import path verifies artifacts before installation or -activation. Queries never download a model or open an ambient cache. Offline -startup remains healthy and baseline-complete. Compatible warmed sessions -are pooled under bounded memory, concurrency, idle, and cancellation policy. -Load failure, OOM, corruption, missing bytes, or incompatible pins disables the -affected semantic stage without silently selecting another model. - -Artifact handling is explicit: - -- A versioned canonical manifest lists profile kind, model/tokenizer/config file digests, - byte lengths, licenses, upstream source metadata, runtime compatibility, - projection inputs, and the complete resource ceiling. The manifest and each - declared member are identified by SHA-256 solely to detect drift, corruption, - or incomplete installation. These versioned manifests and integrity checks - are the complete artifact-verification contract. -- Daemon-owned background acquisition resolves only the cataloged repository - and immutable revision into an explicit cache under the lifecycle root. It - stages each member, independently checks its declared length and SHA-256, - and atomically publishes the complete verified install. Query/runtime paths - perform no download, import, extraction, source discovery, or network - inference. -- Explicit local-path and configured-HTTPS import APIs remain available as - manual operations, but no production surface currently wires them. They are - not the delivered acquisition journey. -- Interrupted imports are resumable only when the source supplies immutable - length, digest, and range identity; otherwise staging is discarded. Archive - traversal, symlink/hardlink entries, absolute paths, duplicate members, - undeclared members, size expansion beyond the manifest, and digest mismatch - quarantine the import without exposing it to runtime discovery. -- Installed artifacts live in one Plan-02-owned user store keyed by versioned - manifest digest, never an ambient Hugging Face/ORT/FastEmbed cache. The - daemon's lifecycle-root Hugging Face cache is an acquisition source only, - not runtime or installation authority. Inventory records distinguish - `staged | verified | installed | quarantined | retained_for_rollback`. - Garbage collection may remove only unreferenced non-active/non-rollback - artifacts after a daemon lease and append-only receipt. -- Activation is an authenticated Plan 20 compare-and-swap from the expected - active/rollback profile digests. It verifies artifact inventory, accepted - Plan 15 report, runtime/platform compatibility, complete projection - generation, resource ceiling, and executable rollback before publishing. - Rollback uses the same checks and must pass a cold-start/offline drill. - -The mandatory Plan 25 `CodeIndexCapabilityManifestV1` admits lexical/graph -retrieval. Optional `SemanticCapabilityManifestV1` pins the authorized scope -digest, code and vector generations, projection and search-index keys, -supported chunk grains/languages, coverage, partial states, privacy domain/key -epoch, and manifest digest. Fusion, candidate budgets, diversity, reranking, -and hydration are validated separately from the active Plan 15/20 profile. The semantic service validates the base -manifest before any channel and validates the semantic augmentation only before -semantic/rerank work. Missing semantic capability yields lexical/graph mode; an -explicit strict-semantic request yields the typed unavailable result. - -## Query and redundancy - -Search resolves exact scope and frozen generation first, reproduces the -versioned exact+lexical+graph baseline, then adds a compatible semantic -candidate list. Fusion is stable and explainable; Plan 15's exact-admission -authority keeps exact identifiers, paths, quoted phrases, errors, tool names, -and configuration keys in a non-demotable tier. The first production semantic -baseline is deterministic exact flat-vector search unless measured -current/10x evidence shows it violates a reviewed resource budget. Optional -reranking is bounded to a configured top-N candidate set, is admitted only -after candidate-controlled gain with no protected-stratum regression, and -preserves the pre-rerank list byte-for-byte when unavailable. Raw similarity, -logits, margins, or fused scores are not confidence; calibrated abstention -requires a versioned cohort/generation-bound profile and reports invalid or -shifted calibration explicitly. Strict semantic requests return a typed -unavailable result. - -Semantic projection and indexing run asynchronously. Existing exact, lexical, -and graph operations remain callable and return without joining, polling, or -waiting for semantic work. The semantic lane is omitted until one complete -compatible immutable vector generation becomes current through a single atomic -publication. Staged, partial, indexing, stale, failed, cancelled, or -incompatible generations contribute no candidates, score, cap pressure, -cursor bytes, or rank effects. A request for strict semantic alone may return -typed unavailable; it may not delay or replace the normal federated path. - -`code.redundancy` reuses the same active generation. It canonicalizes pairs, -removes self/overlapping chunks, and reports `exact_clone`, -`structural_near_duplicate`, `semantic_analogue`, or `insufficient_evidence`. -Semantic-only matches remain review candidates, never automatic rewrites or CI -violations. Disabled semantics preserves the structural baseline and ordering. - -The Plan-05 query pipeline executes these phases without combining them: - -1. Resolve and authorize project/repository/worktree/ref scope; freeze code, - exact, lexical, graph, and compatible semantic generations; validate the - base and semantic capability manifests, authorization epochs, and budgets. -2. Reuse federated retrieval's separately inspectable `RetrieverOutcome` values for - `ExactLiteral`, `Lexical`, and `Graph`; produce one independent - `Semantic` outcome. No semantic code calls, wraps, filters, or mutates - another lane. -3. Validate/canonicalize the semantic outcome under the generic Plan 15 lane - contract: finite canonical scores, candidate identity dedupe, generation and - authorization equality, stable lane-local ordering, complete coverage, and - a bounded continuation. -4. Pass all compact candidates to the one federated fusion implementation. The - existing exact tier remains lexicographically first and cannot be demoted by - graph/semantic scores, diversity, or reranking. This plan evaluates semantic - weighting/fusion candidates, including exact rational RRF if proposed, on - the same recorded candidate inputs; no algorithm, constant, or weight - activates without a passing Plan 15 evaluation. -5. Use the complete comparator and diversity policy from the tested generic - profile. The cursor binds emitted identities, ordered overflow, every lane - continuation, profile/generation/projection digests, and authorization - epoch. Input insertion order, hash order, and completion order cannot affect - IDs, order, contributions, explanations, coverage, or cursor bytes. -6. Retain every generic `CandidateContribution`, including lane-local raw score - domain/rank, calibrated fixed-point feature when valid, weighted - contribution, profile/projection identity, evidence class, and exclusion - reason. Graph path coverage/authority and semantic projection/distance - provenance remain typed lane evidence. A fused score is not probability or - confidence. -7. Optionally rerank only the profile-bounded non-protected candidate set while - preserving the pre-rerank list and contributions. Absence/failure returns - those bytes unchanged. -8. After fusion, diversity, reranking, and page selection, reauthorize and - hydrate only selected hits through the federated hydration port. Hydration may add bounded - symbol text, file context, declarations, and graph neighbors from the same - generation; it cannot add candidates or change rank. - -Plan 15's generic `FusionProfile` pins the selected algorithm, lane features/ -weights, candidate budgets, diversity caps, protected exact classes, rerank -budget, hydration budget, and total comparator. Plan 15 recommends that profile -from direct evaluation; Plan 05 implements it, and Plan 31 supplies -only the semantic lane, artifact/runtime/projection implementation, and -measurements. - -## Authorization and local/private boundary - -- Owning source stores authorize exact scope, operation, privacy domain, key - epoch, and code generation. Plan 20's accepted profile and the semantic service - separately authorize semantic projection generation and runtime profile. The - semantic query composition validates both receipts before invoking the semantic - lane; source denial invokes zero channels, while - semantic denial invokes no semantic/rerank port and preserves authorized - lexical/graph execution. -- Candidate references carry authorization receipt ID and epoch without private - payload. Immediately before hydration, the shared query pipeline rechecks current - authorization, scope, privacy domain/key epoch, and frozen generation. - The owning store performs atomic receipt validation plus bounded payload/ - neighbor read; every neighbor must be inside the authorized scope. Revocation - before or during a multi-read discards the complete hydrated hit and returns - a typed denial with no source text or neighbor payload. -- All caches are local and privacy-domain/key-epoch separated. Model artifacts - key by versioned manifest digest; sessions by embedding projection key; vectors - and checkpoints by projection key plus source generation/chunk digest. - Result caches additionally key by authorization receipt/epoch, authorized - scope and query digests, fusion/rerank/request revisions, and pagination. - Hydration caches additionally key by candidate identity, requested fields, - and neighbor budget. Domain/epoch changes produce zero hits in every cache; - authorization changes produce zero result/hydration hits. No ambient model - cache, network inference, browser runtime, external process, or cross-domain - cache is permitted. -- Raw queries, source text, vectors, private paths/symbols, and hydrated - explanations never enter telemetry or checked-in benchmark artifacts. - A raw query is sanitized into a bounded ephemeral query view before model - inference. `EphemeralSanitizedQueryViewV1` contains bounded sanitized bytes - plus sanitizer and normalization revisions; it is non-serializable, - non-cacheable, and valid only for the authorized request. `QueryDigest` is a - privacy-domain/key-epoch keyed MAC over that view, never an unkeyed content - hash. `QueryDigest` is owned by - `crates/tracedecay-domain/src/retrieval.rs` and may appear only in in-process - request state, authenticated cursor identity, and privacy-separated local - cache keys. `EphemeralSanitizedQueryViewV1` is owned by - `code_intelligence/search.rs`. Neither value may enter receipts, telemetry, - durable stores, or checked-in artifacts. Query text and vectors remain in - memory only for that request. - Operational receipts contain opaque identities, revisions, outcomes, counts, - and digests only. Only sanitized fixtures and aggregate Plan 15 reports may - enter Git. -- Semantic errors, timeout, cancellation, OOM, corruption, or missing artifacts cannot - broaden scope. When the selected profile permits fallback, the lexical/graph - lane outcomes and Plan 15's named exact/lexical/graph fallback subpayload are byte-identical. - The enclosing response may add only a typed semantic/rerank outcome outside - that subpayload, its digest, and cursor identity; strict mode fails closed. - -## Plan 15 evaluation handoff and fresh-store reset - -Semantic retrieval ships a versioned sanitized corpus covering exact names, natural-language -intent, mixed queries, renamed symbols, same-name cross-scope cases, no-answer -queries, generated/vendor noise, large symbols, unsupported languages, and -incremental edits. It measures exact-hit retention, precision/recall/MRR/nDCG, -wrong-scope and no-answer error, worst strata, build/update time, p50/p95/p99, -CPU/RSS, model/vector/cache bytes, cancellation, and offline behavior. - -Plan 15 owns this corpus's partition/label policy, metrics, thresholds, and -activation recommendation. This plan owns reproducible Linux execution and truthful -`pass`, `fail`, or `pending` summaries. Activation requires a passing Plan 15 -evaluation showing no scope/privacy or protected -exact/no-answer/wrong-scope regression, demonstrated semantic gain, and -declared current/10x Linux resource observations. Sensitive or -ineligible bytes never enter documents, artifacts, metrics, explanations, or -model-assisted routes. - -Late interaction, quantization, and specialized ANN remain measured candidate -profiles, not semantic-delivery defaults. ANN is admitted only when it beats exact search's -reviewed resource budget while meeting exact-oracle average, tail, minimum, -and zero-recall-query gates under immutable-generation compatibility. No HNSW, -DiskANN, ScaNN, vector database, precision, or quantization choice is -mandatory. Public benchmark rank cannot select a production profile. - -Non-final vectors are never trusted, republished, or converted. They return -`ResetRequired`; an operator explicitly resets or recreates the generation from -current canonical documents without using the old vector as input. - -## Current delivery audit - -The audit below separates callable delivery from pending direct execution. A -present source file or type name is not completion by itself; the cited direct -regressions invoke the relevant boundary. Static fixture validation is useful, -but the current Linux evaluation remains pending. - -**Production lifecycle correction (updated 2026-07-26).** Commit `dd4adbe2a` -ships daemon-owned immutable `hf-hub` background acquisition. -`shared_lifecycle_owner()` opens a lifecycle-root-scoped -`HfHubModelMemberSourceV1`; startup queues acquisition when configured, and the -source resolves only the cataloged repository/revision before the lifecycle -checks every member's declared length and SHA-256 and atomically installs the -package. Status and Doctor expose downloading, verifying, installed, and -failed states. A clean temporary profile downloaded and digest-verified the -full default Jina model end to end, and packaged offline acceptance installs -the same verified model from the daemon-owned cache. The explicit local and -configured-HTTPS import APIs remain manual and production-unwired; they are not -being certified by this correction. Installed model bytes remain semantically -omitted until compatible vector indexing publishes readiness, preserving the -asynchronous non-blocking contract. - -**Why the old green gate was misleading.** Before `dd4adbe2a`, -`HfHubModelMemberSourceV1::fetch_member` unconditionally returned -`semantic model lifecycle operation rejected`, so no shipped binary could -acquire a model. The distribution gate's packaged-semantic half tested only -typed fallback/strict-unavailable behavior, the exact terminal absence produced -by that rejection, and therefore encoded the bug as its passing specification. -The corrected gate now runs packaged background acquisition and verifies the -installed Jina members; restoring the unconditional rejection makes that gate -fail, and restoring the source makes it pass. - -- **Library-first FastEmbed:** delivered. The root manifest keeps - `fastembed` optional with upstream defaults disabled, the - `semantic-fastembed` feature selects the native runtime, and - `FastEmbedEmbeddingRuntime::open_session` uses FastEmbed's user-defined local - byte constructor. Native model execution and resource evidence remain - pending. -- **Default production posture:** delivered in the root feature manifest. - FastEmbed/ORT is available on Linux and macOS. Windows production builds keep - the daemon, hosts, exact/lexical/graph retrieval, and the ORT-free Model2Vec - backend; selecting a FastEmbed model reports the typed unavailable runtime - state and never silently substitutes another embedding backend. -- **Daemon-owned immutable acquisition:** delivered. Startup queues the - selected catalog revision without blocking startup or query paths; the - lifecycle-root source cache is explicit, and verification plus atomic install - remain independent of `hf-hub`. -- **Local verified model bytes only:** delivered at the runtime boundary. - Model, tokenizer, and config bytes come from the installed manifest members; - runtime construction has no hub, ambient-cache, download, external-process, - or network-inference path. Background acquisition now creates the verified - installed manifest and members; explicit local/HTTPS imports remain - production-unwired. Direct tests validate constructor and integrity reads, - while cross-platform runtime coverage remains normal CI work. -- **Exact-flat semantic baseline:** delivered by - `SemanticCodeRetriever`/`SemanticVectorReadPort::scan_exact_flat`, with direct - deterministic ordering, provenance, generation, and coverage regression in - `exact_flat_scan_is_deterministic_and_emits_generic_semantic_evidence`. -- **Immutable generations and verified publication:** active integration work - uses the staged vector-generation store and the Plan 39 verified Grafeo - publication boundary. Staged work is not queryable, cancellation preserves - every prior verified generation, and failed publication cannot expose a - partial generation. Publication only adds immutable catalog state and may - advance the relational verified graph head; it never activates semantic - routing. Plan 20's linked configuration transition and activation receipt - are the sole active/rollback authority, while process-local runtime pointers - are observed readiness caches only. -- **Asynchronous, non-blocking indexing:** delivered at the projection, - generation-selection, runtime-routing, and bounded-session boundaries. - `only_a_current_receipt_routes_to_semantic_search` routes indexing, degraded, - unavailable, and rollback states to the frozen exact/lexical/graph fallback, while - `saturated_runtime_omits_semantics_without_entering_the_waiter_queue` proves - query work does not wait for semantic capacity. -- **Calibrated abstention:** delivered. Missing or shifted calibration invokes - no semantic authority and preserves fallback; distance and margin rejection - are versioned and generation-bound. Strict semantic returns typed unavailable. - Linux threshold evaluation remains pending. -- **Byte-stable exact/lexical/graph fallback:** delivered at the semantic service boundary. - Direct regressions retain the caller-owned validated fallback object through - augmentation and every tested abstention path. The versioned fallback bytes - and digest remain pending direct lexical verification. -- **Evaluation before activation:** still pending. Current runtime - routing requires an observed current activation receipt and rejects an - indexing receipt, but the Plan 15 Linux evaluation, current/10x measurements, - normal cross-platform CI, and rollback drill are still incomplete. - `result-pending.json` therefore keeps activation false and truthfully reports - `pending`. - (Update 2026-08-07: the named artifact is stale, no `result-pending.json` - exists in-tree (`git ls-files` returns zero hits). The behavior it stood for - is intact and is now asserted by the evaluation harness itself: - `tests/search_eval_cli_test.rs:53` - (`compare_reports_unmeasured_semantic_and_rerank_stages_as_pending`) requires - the `compare` CLI to exit non-zero and report unmeasured semantic/rerank - stages as `pending`. The gate is unchanged; only the file reference is - obsolete.) - -## Planned behavioral delivery and direct verification - -The native semantic delivery remains unfinished. The checkpoints below are required product behavior. -Paths, type spellings, fixture filenames, benchmark entrypoints, and test-suite -registration are non-normative historical suggestions. Completion follows the -callable semantic operation, direct regressions, the Plan 15 Linux evaluation, -and normal CI. - -1. **Contracts and capability admission:** add semantic-only domain values, - ports, manifest validators, ephemeral query-view rules, and split in-memory - adapters. Reuse federated retrieval's generic retrieval/fusion/hydration behavior. Direct - regressions cover contract validation, capability admission, and storage - independence. -2. **Artifact and runtime foundation:** implement one root-private artifact - verifier, manifest validator, FastEmbed adapter, and bounded session pool. - Direct regressions cover versioned manifest and SHA-256 member verification, - local and explicit HTTPS import, - traversal/expansion rejection, interrupted staging, atomic install, - quarantine/GC, cold and warm sessions, OOM, cancellation, - offline startup, no ambient cache, and Linux/Windows native-runtime - compatibility. -3. **Incremental vector projection:** implement the projector and vector-generation - store against Plan 04 projection/checkpoint semantics, Plan 02 - receipt/publication authority, and this plan's runtime ports. Daemon/service - orchestration owns asynchronous worktree-fair bounded scheduling; do not - assign it to Plan 04. Direct regressions cover changed-chunk batching, - deletion without inference, no-op zero inference, cross-worktree physical - reuse without identity reuse, superseded-batch cancellation, model-key - replay, search while indexing without waiting, omission of incomplete/ - incompatible generations, atomic publication, cancellation/failure - isolation, and offline lifecycle. -4. **Exact-flat semantic retrieval and shadow composition:** implement one - independent semantic outcome and compose it with frozen federated lane outputs - through the shared fusion/diversity/cursor behavior. Direct regressions - cover lane isolation, contribution provenance, protected exact results, - diversity/pagination, and byte-identical fallback. -5. **Late hydration, privacy, and rollback:** reuse federated hydration with semantic - profile admission, generation checks, authorization recheck, - domain-keyed caches, payload-safe receipts, active/rollback pointer CAS, and - cold offline rollback. Direct regressions cover hydration, authorization, - privacy-domain isolation, activation, and rollback. -6. **Developer evaluation:** use reproducible checked-in Linux workloads, - explicit reset/recreation checks, exact-flat oracle comparisons, focused - channel/fusion/calibration/rerank ablations, privacy/non-interference checks, - and current/10x resource observations. Optional ANN and reranker branches - begin only after the exact-flat shadow checkpoint and cannot enter the - critical path without a passing comparison. -7. **Activation and verification:** only a passing Plan 15 result may make - semantics eligible for the existing configuration activation. Run staged - shadow/cohort behavior, rollback, fresh-store reset, privacy, architecture, direct - tests, Linux evaluation, and normal all-feature cross-platform CI. Status - and Doctor behavior are exercised through the production lifecycle states; - activation still requires the remaining evaluation and indexing evidence. - This plan does not create a temporary public semantic endpoint or reserve later - surface contracts. - -A reproducible projection workload measures clean, warm one-symbol, deletion, -no-op, model-key replay, cancellation, and incompatible rebuild. A query -workload reports lexical, graph, semantic, fusion, rerank, and hydration time -separately at current and 10x corpus sizes, -including p50/p95/p99, CPU, peak RSS, model/vector/cache bytes, candidates per -channel, chunks embedded/reused/deleted, hydration fetch count, and fallback. -Channel ablations use equal candidate budgets; exact flat-vector search is the -semantic oracle. End-to-end performance work consumes these production -measurements, while Plan 15 owns quality/resource trade-off and activation -recommendations. - -The query workload also overlaps queries with a blocked projection worker and -with staged, partial, failed, cancelled, stale, and incompatible generations. -It records exact/lexical/graph completion independently, requires zero wait on -the semantic worker, compares the visible federated fallback bytes and rank before -and during indexing, and observes semantic candidates only after the complete -compatible generation is verified and Plan 20 has committed the matching -activation receipt. Those are ordered publication and configuration -transactions, not a graph-local pointer swap. - -Each workload manifest pins corpus/query digests, exact file/chunk/query counts, -language/source strata, seed, model/projection/fusion revisions, hardware and -runtime summary, cache state, and concurrency. The 10x workload contains -exactly ten times the eligible chunks of the current workload without copying -quality labels across partitions. Reports retain the raw Linux samples needed -for every statistic and label unsupported p99 or uncertainty claims `pending`. - -### Hard activation barrier - -No semantic comparison starts until callable exact/lexical/graph behavior -passes direct regressions and its versioned profile, exact-tier contract, -fallback-subpayload bytes, and quality fixtures are identified by real -content/revision digests. Historical artifact-name parity is not part of this -barrier. No activation occurs unless Plan 15 reports `pass`, -authorization/scope leakage is zero, protected exact results are -unchanged, the exact/lexical/graph fallback subpayload is byte-identical, -generation compatibility holds, search-during-indexing leaves fallback bytes and rank -unchanged, incomplete or stale generations contribute nothing, all -resource ceilings pass, and cold offline rollback succeeds. - -A `fail` or `pending` result leaves semantics disabled and no semantic profile -eligible. - -## Acceptance - -Semantic retrieval is complete when projection, atomic publication, lexical-fallback-preserving -search, bounded generic fusion/reranking/redundancy, artifact/offline lifecycle, -configuration, production status/Doctor behavior, corpus/resource/privacy -tests, fault recovery, rollback, explicit reset/recreation, the Linux developer -evaluation, and normal CI pass. Application, public transport, and dashboard -adapters remain required parts of the same product. No separate -semantic endpoint, vector database, browser inference runtime, or model-specific -transport is introduced. Queries never silently substitute a model/revision, -download at query time, cascade to an unmeasured representation, or treat -semantic similarity as identity, impact, lineage, or equivalence. - -- A no-op chunk manifest performs zero embedding calls. A one-symbol edit - embeds exactly its changed symbol chunks and affected file-level chunks and - tombstones explicit deletions. Changing only vector-affecting model profile - fields with unchanged canonical chunk inputs replays all eligible chunks with - zero parser/extractor calls; chunker/sanitizer/sensitivity changes follow - Plan 25's canonical rebuild path. -- Receipt fixtures reject missing, duplicate, extra, wrong-generation, wrong- - digest, and wrong-key entries; crash/cancellation leaves the previous active - retrieval configuration unchanged and no partial projection queryable. - Queries issued while - projection is blocked complete through exact/lexical/graph without waiting - and match the versioned federated fallback bytes and rank. Only a complete, - recovered-state-verified generation named by the current linked activation - receipt may add the semantic lane. -- The semantic adapter emits a separately inspectable generic - `RetrieverOutcome>`; the exact, - lexical, and graph outcomes are unchanged. Disabling or failing semantic - retrieval preserves Plan 15's named federated fallback subpayload bytes. Fused - explanations reproduce every declared generic profile - contribution and the complete comparator for every result. -- Exact-hit retention and first-relevant-protected-hit Recall@10 are 100% over - Plan 15's versioned protected-query set, with numerator, denominator, and - support reported separately for symbols, qualified names, compiler/runtime - errors, CLI flags, paths, quoted phrases, tool names, and configuration keys. - For multi-target queries, every relevant protected hit admitted by the - requested page size precedes every fuzzy, graph-only, or semantic-only hit. -- One hundred runs with shuffled candidate insertion and channel completion - order produce byte-identical channel lists, fused IDs/order/contributions, - diversity spillback, explanations, and cursors. -- Fixture profiles with `max_non_exact_per_source = 3` and - `max_non_exact_per_file = 2` enforce those caps before pagination, preserve - protected exact hits, and deterministically refill from overflow candidates. - A three-page fixture proves no duplicate, omission, or cap drift after cursor - resume. -- A page requesting five hydrated hits reads payloads and neighbors for at most - those five hits and the profile's declared neighbor budget. Hydration cannot - change rank; every neighbor matches the frozen generation and preserves graph - ordered path identity, coverage, and weakest edge authority. -- Denied authorization invokes zero retrieval ports and zero payload reads. - Revocation before hydration returns no payload. Privacy-domain/key-epoch - changes yield zero session/vector/result/hydration cache hits, and cross- - domain candidates, vectors, receipts, metrics, or explanations are zero. -- Split-adapter conformance produces identical results when lexical postings, - graph evidence, vectors, receipts, and hydration payloads use separate stores, - proving embeddings and no single physical table are authority. -- Artifact fixtures reject malformed versioned manifests, undeclared or - duplicate members, path traversal, links, size expansion, - digest/length mismatch, incompatible runtime/platform pins, interrupted - publication, and deletion of active/rollback artifacts. Query execution - performs zero network/import/cache-discovery operations. -- Distribution acceptance starts from an isolated profile, queues the - background worker, installs every digest-verified default Jina member, and - proves semantics remain omitted until indexing readiness. Its offline mode - consumes only the lifecycle-root cache and must fail if production - acquisition regresses to unconditional rejection. -- The checked-in benchmark corpus contains only sanitized fixtures and expected - opaque anchors; raw private queries/source are absent. Activation requires a - passing Plan 15 evaluation and cannot be inferred from public rank or - aggregate gain. diff --git a/docs/plans/tracedecay-v2/38-storage-retention-size-and-efficiency.md b/docs/plans/tracedecay-v2/38-storage-retention-size-and-efficiency.md index c7f9a8191f3..f53b66bc913 100644 --- a/docs/plans/tracedecay-v2/38-storage-retention-size-and-efficiency.md +++ b/docs/plans/tracedecay-v2/38-storage-retention-size-and-efficiency.md @@ -147,14 +147,9 @@ measurements, not inferred table sizes. ## Product contract -1. **Branch store lifecycle.** Deleting a branch (or a worktree whose branch - is gone) schedules its branch-DB removal through the daemon. A periodic - daemon sweep reconciles `branches/` against live git refs. `branch gc` - remains the manual verb; the automatic path is the default. - - **Dated amendment (2026-08-07, recorded decision, supersedes the - per-branch-DB model above).** Per-branch SQLite copies are retired as the - write-side mechanism. Non-default-branch writes now land in the single +1. **Branch store lifecycle.** Per-branch SQLite copies are retired as the + write-side mechanism (dated amendment, 2026-08-07). Non-default-branch + writes now land in the single project store's next branch-graph publication epoch, fenced by the sync lease and sealed through the branch-meta `graph_source` CAS (`feat(graph): fence branch graph publication and mutation epochs`, diff --git a/docs/plans/tracedecay-v2/39-embedded-grafeo-graph-database.md b/docs/plans/tracedecay-v2/39-embedded-grafeo-graph-database.md index c37a3d432bb..ffb9ec389a1 100644 --- a/docs/plans/tracedecay-v2/39-embedded-grafeo-graph-database.md +++ b/docs/plans/tracedecay-v2/39-embedded-grafeo-graph-database.md @@ -1,14 +1,5 @@ # Embedded Grafeo Graph Database Implementation Plan -> **SUPERSEDED BY LANDING (2026-08-13).** The checklist below predates the -> cutover and was never ticked; the work shipped regardless. Evidence: -> `crates/tracedecay-graph-db` is a workspace member pinned to Grafeo -> `=0.5.42` with `generation_runtime.rs` and the sealed-graph state digest; -> the daemon consumes it through -> `src/daemon/code_index_scheduler/graph_activation.rs`; and the SQLite graph -> authority was deleted in `79683eb527`. Treat the unchecked boxes as -> historical planning, not open work. - **Goal:** Replace custom adjacency structures and graph-shaped SQLite storage with one embedded Grafeo runtime boundary while retaining SQLite only for genuinely relational, transactional, and content-bearing records. **Architecture:** `tracedecay-graph-db` is the only workspace crate allowed @@ -20,9 +11,8 @@ content, and source manifests remain in their domain stores; Grafeo is the sole persisted and queried graph/vector projection over those sources. SQLite does not store shadow adjacency, graph indexes, or vector indexes. -**Tech Stack:** Rust 2024, the published Grafeo `=0.5.42` crates from -crates.io, TraceDecay domain/store ports, Tokio cancellation, Criterion, and -cargo-nextest. +**Tech Stack:** Rust 2024, the Grafeo crates, TraceDecay domain/store ports, +Tokio cancellation, Criterion, and cargo-nextest. ## Global Constraints @@ -35,9 +25,6 @@ cargo-nextest. `.grafeo.wal` sidecar while the file is open and checkpoints it into the database file on close. Do not inject an already-open file, replace Grafeo's storage engine, or add a private/secure filesystem adapter. -- Depend on the exact published `=0.5.42` release. Do not pin an unpublished Git - revision, maintain a TraceDecay fork, vendor Grafeo, or reimplement its - storage, WAL, checkpoint, or recovery machinery. - `tracedecay-graph-db` is the sole production direct `grafeo-*` dependency. - Do not introduce another persisted/query graph or vector database. A bounded in-memory algorithm is not a storage authority, but should use the standard @@ -79,8 +66,6 @@ Official implementation references: - [Grafeo repository and crate layout](https://github.com/GrafeoDB/grafeo) - [Embedded Rust API](https://grafeo.dev/user-guide/rust/) -- [Vector and hybrid search](https://grafeo.dev/user-guide/vector-search/) -- [Published Rust API for `0.5.42`](https://docs.rs/grafeo/0.5.42/grafeo/) - [Persistent storage and WAL](https://grafeo.dev/user-guide/persistence/persistent/) --- @@ -250,11 +235,10 @@ Expected: fail because the crate and interfaces do not exist. - [ ] **Step 2: Add the workspace crate and centralize Grafeo dependencies** -Use the exact published Grafeo `=0.5.42` crates in the new crate. Add -`tracedecay-graph-db` to `[workspace].members` and centralize those exact -versions in `[workspace.dependencies]`. Enable Grafeo's public `grafeo-file` -feature together with `wal`. No other workspace crate directly depends on -Grafeo, and no Grafeo dependency uses a Git source. +Use the workspace Grafeo crates in the new crate. Add `tracedecay-graph-db` +to `[workspace].members` and centralize those dependencies in +`[workspace.dependencies]`. Enable Grafeo's public `grafeo-file` feature +together with `wal`. No other workspace crate directly depends on Grafeo. - [ ] **Step 3: Implement typed open, snapshot, batch, traversal, and vector adapters** @@ -395,67 +379,6 @@ cargo clippy -p tracedecay-graph-db -p tracedecay-code-index -p tracedecay-query git commit -am "refactor(code-graph): route Grafeo through graph-db" ``` -## Task 4: Move semantic vectors and hybrid retrieval - -**Files:** -- Modify: `crates/tracedecay-semantic/src/projector.rs` -- Modify: `crates/tracedecay-semantic/src/runtime_query.rs` -- Modify: `crates/tracedecay-application/src/store/vector_generations.rs` -- Modify: `src/store/vector_generations.rs` -- Modify: `crates/tracedecay-query/src/retrieval/semantic/execution_authority.rs` -- Delete: SQLite code-semantic vector payload/state tables superseded by graph-db - -**Interfaces:** -- Consumes: `EmbeddingProjectionKey`, verified `EmbeddingVector`, source - generation, model/artifact digest, metric, dimensions, and normalization. -- Produces: recovered-state-verified vector publication and bounded Grafeo - vector similarity; TraceDecay composes hybrid retrieval above it. - -```rust -pub trait SemanticVectorStore { - fn publish_generation( - &self, - generation: &VectorGenerationId, - vectors: Vec, - cancellation: &CancellationToken, - ) -> Result; - - fn search( - &self, - request: &SemanticVectorSearchRequest, - ) -> Result; -} -``` - -- [ ] **Step 1: Add failing vector behavior tests** - -Cover cosine/dot/euclidean behavior as declared by the manifest, dimension mismatch, non-finite values, stale generation, cancelled query, filtered namespace, deterministic ties, reopen, and serving the previous generation during rebuild. - -- [ ] **Step 2: Move vector payloads and indexes to graph-db** - -Store the vector on the corresponding projected entity or a typed vector -entity linked to it. Keep model lifecycle/install metadata and publication -receipts relational. Publish vector and code graph generations in one graph-db -batch when they share a generation. - -- [ ] **Step 3: Implement hybrid retrieval above the storage primitive** - -Grafeo returns bounded vector candidates. `tracedecay-query` retains -BM25/lexical retrieval and performs authorization, fusion, normalization, -hydration, explanations, and stable cursors. - -- [ ] **Step 4: Delete SQLite vector payload duplication** - -Remove `semantic_vector_payload_v1`, `semantic_vector_state_slice_v1`, evaluation duplicates, and legacy vector blobs after every production reader moves. Retain only relational generation/model lifecycle records that are not duplicate vector authority. - -- [ ] **Step 5: Verify and commit** - -```bash -cargo nextest run -p tracedecay-semantic -p tracedecay-query -p tracedecay-application --no-fail-fast -cargo bench --bench code_search --no-run -git commit -am "refactor(semantic): move vector search to graph-db" -``` - ## Task 5: Move Git topology and evidence relations **Files:** @@ -958,30 +881,6 @@ recovered-digest proof in compact form with "relation scalar endpoints do not match native topology" while the same rows seal and serve in replay form. Toy-scale contracts pass, so the defect is scale- or ordering-dependent. -**Decision.** Live-container reclaim is `compact()` under the maintenance -lease, not a bespoke re-seal or a quarantine-and-rebuild: - -1. Reproduce the endpoint mismatch in the fork with a generation-scale - fixture (the sealed-store contract's `rich_manifest` grown to the failing - size), fix it there, and gate `COMPACT_ROUND_TRIPS_BYTES` on the recovered - digest proving equal in compact and replay form for that fixture. -2. Add a maintenance-cadence step beside `run_branch_compaction`: when a - project's live container exceeds twice the summed bytes of its verified - heads' sealed artifacts (both figures are already censused, see the - `RetentionBacklog` finding's sealed evidence), take the project store - maintenance lease, quiesce the graph owner, `compact()`, close, and reopen. - The reopen reconstructs the layered store from the `CompactStore` section; - later writes land in the overlay, so the store stays mutable. -3. Prove the swap with the existing recovered-digest verification before the - compacted container replaces the live file; a mismatch keeps the original - and reports a typed degradation. Vector generations, whose only durable - home is the live container, are covered by the same proof and never - re-embedded. - -**Non-goals.** No re-projection from canonical replay authorities as a -compaction path (that is corruption recovery and would re-embed vectors), and -no quarantine of a healthy container. - ### Correction (2026-09-12, later the same day, measured) The decision above rests on a wrong premise. Read against the pinned fork: @@ -1039,14 +938,10 @@ The decision above rests on a wrong premise. Read against the pinned fork: base rather than a replay into RAM, deletes land in the overlay's deletions section, and `recompact()` merges on the maintenance cadence. Blocked on validating TraceDecay's staging writes against a - `LayeredStore` (README gaps 3 and 4: `has_vector_index` planning and - the MVCC-free base). + `LayeredStore` (README gap 4, the MVCC-free base). b. Rebuild a fresh container by streaming the retained authorities, sealed heads and their dependency closure from their compact stores, - pending replays from `canonical_replay_source`, vector generations - from their sealed stores, then prove every generation's recovered - digest against the journal before swapping files under the - maintenance lease. Memory is bounded by the largest generation, not - the container. Blocked on vector generations: the sealed lane never - routes vector search to sealed stores, so a rebuilt live container - must re-stage the vector rows and rebuild the HNSW index on open. + and pending replays from `canonical_replay_source`, then prove every + generation's recovered digest against the journal before swapping + files under the maintenance lease. Memory is bounded by the largest + generation, not the container. diff --git a/docs/plans/tracedecay-v2/40-partitioned-generation-storage-and-zero-copy-replay.md b/docs/plans/tracedecay-v2/40-partitioned-generation-storage-and-zero-copy-replay.md index 2cdba660c4a..36e520ba0cf 100644 --- a/docs/plans/tracedecay-v2/40-partitioned-generation-storage-and-zero-copy-replay.md +++ b/docs/plans/tracedecay-v2/40-partitioned-generation-storage-and-zero-copy-replay.md @@ -1,16 +1,5 @@ # 40. Partitioned generation storage and zero-copy replay retirement -Status: ACTIVE (decided 2026-08-22). Supersedes the monolithic sealed-generation -envelope as the long-term storage design; stages 0–1 land first as compatible -fixes inside the current `format_revision`. - -Stage state (reconciled 2026-08-31 against the tree, not the commit log): -stages 0a, 0b, and 0c are landed; stages 1 and 2 are landed; stage 3 is -untouched. Partitioned writers emit `format_revision` 7 while readers retain -explicit revision-5/6 monolithic compatibility. The -§Problem measurements below describe the pre-stage-0 state and are retained -as the original evidence, not as a description of the current tree. - ## Problem (measured, 2026-08-21/22, this repository) - One sealed code generation for this repository is a single @@ -63,55 +52,6 @@ representation and the replay economics: 5. **No second whole-generation copy, ever.** Replay reads canonical artifacts; retention must never require a byte-duplicating insurance copy. -## Staged delivery (each stage ships alone, oldest-first compatible) - -- **Stage 0, zero-copy replay retirement** (this change set, three commits): - - **0a** `hydrate_sealed_code_generation` resolves the sealed payload from - the canonical `code-generations-v1/` root first and falls back to the - replay pool. Readers become location-agnostic; digest verification on read - makes both locations equally trustworthy. Retirement moves strictly - canonical→pool, so a canonical-then-pool probe cannot miss a live file. - - **0b** Retention retires a superseded generation by **atomic rename into - the replay pool** (no-clobber; content-identical `AlreadyExists` collapses - to unlink) instead of unlinking, whenever the graph projection has not yet - durably consumed it. Rename is metadata-only and crash-atomic: the file is - always in exactly one of the two roots. The existing release queue keeps - deleting pool entries once the graph append is durable. - - **0c** Delete the eager staging copy from the graph publish path - (`install_project_graph_replay_seal_at` / `stage_project_graph_replay_seal` - and the copy machinery in `seals.rs`). Steady state stages **zero bytes**; - the pool only ever holds generations that were retired while still needed. -- **Stage 1, single-parse activation**: graph publication consumes the - already-decoded active generation (the projection manifest already rides - along on first publication; extend the same guarantee to the recovery - branches via the Stage-0a fallback) so cold activation parses the sealed - payload at most once. -- **Stage 2, manifest + segment split** (`format_revision` bump): split the - sealed envelope per §Decision 1–2; integer-keyed, content-addressed - segments; the 2 GiB whole-envelope bound and whole-file rewrite per - generation disappear; unchanged-file segments are shared across - generations. -- **Stage 3. Grafeo startup authority**: manifest ↔ Grafeo head validation - replaces replay on clean startup per §Decision 3–4. - -## Invariant mapping (old mechanism → new mechanism) - -| Invariant | Today | After | -|---|---|---| -| Content addressing | digest-named monolithic JSON | digest-named manifest + digest-named segments | -| Atomic publish | temp+rename of envelope + pointer | segments/Grafeo commit first, then manifest temp+rename | -| Crash recovery | eager 1.39 GB pool copy + journal replay | rename-retired canonical files + scoped lazy replay | -| Retention | unlink source; release pool copy via queue | rename-to-pool when graph pending; queue unchanged | -| Verification | full-file digest on every decode | per-component digest on read (unchanged per file) | -| Branch identity | durable index entry evidence | unchanged (manifest carries it) | - -## Non-goals - -- No change to sealing semantics, projector revisions, or the relational - journal contract in Stage 0. -- Stage 0 does not change `format_revision`; on-disk envelopes stay readable - both directions across the stage-0 commits. - ## Dated amendment (2026-09-12, recorded decision): stage 2b, shared, compressed segments **Measured.** For one worktree of a 150 MB / 5,190-file repository the scoped From 576b02539f97e4841b7133adbce15fdb585da843 Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Thu, 5 Nov 2026 22:19:18 -0800 Subject: [PATCH 14/71] fix(evals): refuse recapturing a vanished first sample A removed first-response sample must not be silently recreated from a later case if it disappears while the second HTTP inference is in flight. Preserve the settled first row and allow recovery without a raw sample. Refs #4535 --- evals/intent_engine/run_http.py | 4 ++++ evals/intent_engine/test_run_http.py | 32 ++++++++++++++++++++++++++++ 2 files changed, 36 insertions(+) diff --git a/evals/intent_engine/run_http.py b/evals/intent_engine/run_http.py index b193784ca36..3946e616520 100644 --- a/evals/intent_engine/run_http.py +++ b/evals/intent_engine/run_http.py @@ -598,6 +598,10 @@ def main(argv=None): # that a later valid retry mistakes for a completed capture. serialized_row = serialize_result(row, questions, args.noul_temperature) if sample_path and not os.path.lexists(sample_path): + # The first raw response cannot be recaptured from a later + # case if its published sample disappeared during inference. + if index != 0: + raise ValueError("first-response sample disappeared; cannot capture a later case") serialized_sample = json.dumps({ "case_id": case["id"], "experiment_sha256": header["experiment_sha256"], diff --git a/evals/intent_engine/test_run_http.py b/evals/intent_engine/test_run_http.py index d86d230431a..61f5ee5afe9 100644 --- a/evals/intent_engine/test_run_http.py +++ b/evals/intent_engine/test_run_http.py @@ -1121,6 +1121,38 @@ def test_resumed_sample_cannot_impersonate_a_later_durable_case(self): self.assertFalse(self.partial.exists()) self.assertEqual(len(json.loads(self.output.read_text())["rows"]), 2) + def test_missing_first_sample_is_not_replaced_with_later_response(self): + sample = self.root / "first-response.json" + calls = 0 + + def lose_capture_during_second_request(url, body): + nonlocal calls + calls += 1 + if calls == 2: + self.assertTrue(sample.exists()) + sample.unlink() + return self.response(url, body) + + with patch.object(run_http, "post", side_effect=lose_capture_during_second): + with self.assertRaisesRegex(ValueError, "sample disappeared"): + self.run_main(["--sample-out", str(sample)]) + self.assertEqual(calls, 2) + self.assertFalse(sample.exists()) + self.assertFalse(self.output.exists()) + saved = [json.loads(line) for line in self.partial.read_text().splitlines()] + self.assertEqual([row["id"] for row in saved[1:]], ["first"]) + + # Recovery without a sample still reuses the first measured row + # instead of claiming the second response was the first capture. + with patch.object(run_http, "post", side_effect=self.response) as post: + self.run_main() + post.assert_called_once() + self.assertFalse(self.partial.exists()) + self.assertEqual( + [row["id"] for row in json.loads(self.output.read_text())["rows"]], + ["first", "second"], + ) + def test_multiple_configs_capture_only_first_configuration(self): sample = self.root / "first-response.json" first_config = "A_noul_prompt" From 81e9db457932e4da63b6aa1f727d56d87b13a5ec Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Thu, 5 Nov 2026 22:22:39 -0800 Subject: [PATCH 15/71] docs(storage): state that store budgets are not enforced yet Removing the Plan 38 reference left a target reading as a current fact. `Storage::maintain` does not exist in the code, and an over-budget store only produces a Doctor finding. Co-Authored-By: Claude Opus 5.5 --- docs/plans/storage/README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/plans/storage/README.md b/docs/plans/storage/README.md index e1ad1596656..323ccda22a8 100644 --- a/docs/plans/storage/README.md +++ b/docs/plans/storage/README.md @@ -622,7 +622,8 @@ changes a file that holds memory. - Plan 02's "No asynchronous database API" and its interactive repository transactions: every storage API is async and every transaction is one job. -- Store soft budgets have defaults and are enforced by `Storage::maintain`. +- Store soft budgets: enforcement through `Storage::maintain` is a target that + does not exist yet. Today an over-budget store only produces a Doctor finding. - Plan 39's move of graph-shaped data off SQLite: mutable graph data (memory and session relations) lives in SQLite; Grafeo remains an immutable sealed format. - Plan 40's monolithic per-worktree generation segments: segments are From 740b952ff5aa9c64e88b8d33a203259f39ba86c7 Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Thu, 5 Nov 2026 22:25:45 -0800 Subject: [PATCH 16/71] fix(automation): configure scheduler test backends explicitly Retain fail-closed Codex executable admission while making scheduler/manual and lock-order tests use an actual configured executable identity. Add a separate regression for the typed unconfigured-backend diagnostic. Resolve three automation_runner_test failures observed on master CI. --- .../tests/automation_runner_test/jobs.rs | 73 +++++++++++++++---- 1 file changed, 59 insertions(+), 14 deletions(-) diff --git a/crates/tracedecay/tests/automation_runner_test/jobs.rs b/crates/tracedecay/tests/automation_runner_test/jobs.rs index 740283d56e8..c373ae86047 100644 --- a/crates/tracedecay/tests/automation_runner_test/jobs.rs +++ b/crates/tracedecay/tests/automation_runner_test/jobs.rs @@ -86,6 +86,7 @@ fn enabled_job_config() -> AutomationSettingsV1 { struct ContentBackend { calls: AtomicUsize, content: &'static str, + executable: Option, } impl ContentBackend { @@ -93,9 +94,15 @@ impl ContentBackend { Self { calls: AtomicUsize::new(0), content, + executable: None, } } + fn with_executable(mut self, executable: &std::path::Path) -> Self { + self.executable = Some(executable.to_path_buf()); + self + } + fn calls(&self) -> usize { self.calls.load(Ordering::SeqCst) } @@ -126,7 +133,7 @@ impl AgentTaskBackend for ContentBackend { } fn executable(&self) -> Option<&std::path::Path> { - None + self.executable.as_deref() } } @@ -980,6 +987,28 @@ async fn user_job_backend_failure_records_failed_ledger_entry() { assert_eq!(records[0].status, AutomationRunStatus::Failed); } +#[tokio::test] +async fn scheduler_prefilter_reports_unconfigured_codex_before_schedule_admission() { + let temp = tempdir().unwrap(); + let dashboard_root = temp.path().join("dashboard"); + let mut job = sample_job("unconfigured-prefilter"); + job.schedule = Some("manual".to_owned()); + + let skip = evaluate_and_record_scheduler_skip( + &dashboard_root, + &enabled_job_config(), + None, + &job, + "unconfigured-prefilter-occurrence", + None, + ) + .await + .unwrap() + .expect("missing executable must produce a typed diagnostic"); + assert_eq!(skip.report["reason"], json!("backend_unconfigured")); + assert_eq!(skip.ledger_record.status, AutomationRunStatus::Skipped); +} + #[tokio::test] async fn scheduler_trigger_skips_repeat_and_respects_lock_discipline() { let temp = tempdir().unwrap(); @@ -988,6 +1017,7 @@ async fn scheduler_trigger_skips_repeat_and_respects_lock_discipline() { let mut job = sample_job("sched-job"); job.schedule = Some("manual".to_string()); let config = enabled_job_config(); + let executable = tempfile::NamedTempFile::new().unwrap(); // Manual schedule: every scheduler tick of the same occurrence reaches // the prefilter, which records the skip once. @@ -996,7 +1026,7 @@ async fn scheduler_trigger_skips_repeat_and_respects_lock_discipline() { let run = evaluate_and_record_scheduler_skip( &dashboard_root, &config, - None, + Some(executable.path()), &job, "sched-occurrence", None, @@ -1019,7 +1049,7 @@ async fn scheduler_trigger_skips_repeat_and_respects_lock_discipline() { let repeated = evaluate_and_record_scheduler_skip( &dashboard_root, &config, - None, + Some(executable.path()), &job, "sched-occurrence", None, @@ -1205,16 +1235,23 @@ async fn scheduler_prefilter_live_lock_wins_over_not_due_summary() { append_run_record(&dashboard_root, &recent).await.unwrap(); let config = enabled_job_config(); + let executable = tempfile::NamedTempFile::new().unwrap(); let occurrence = "locked-prefilter-occurrence"; - let prefilter = - evaluate_and_record_scheduler_skip(&dashboard_root, &config, None, &job, occurrence, None) - .await - .unwrap() - .expect("the live job lock must produce a diagnostic"); + let prefilter = evaluate_and_record_scheduler_skip( + &dashboard_root, + &config, + Some(executable.path()), + &job, + occurrence, + None, + ) + .await + .unwrap() + .expect("the live job lock must produce a diagnostic"); assert_eq!(prefilter.report["reason"], json!("scheduler_lock_active")); assert_ne!(prefilter.run_id, occurrence); - let backend = ContentBackend::new("unused"); + let backend = ContentBackend::new("unused").with_executable(executable.path()); let admitted_job = tracedecay_automation_runtime::automation::jobs::admit_job(&job).unwrap(); let retained = run_user_job_with_backend_for_retained_settlement( &dashboard_root, @@ -1261,13 +1298,21 @@ async fn retained_scheduler_runner_reacquires_after_due_prefilter() { let mut job = sample_job("prefilter-gap"); job.schedule = Some("every 1h".to_owned()); let config = enabled_job_config(); + let executable = tempfile::NamedTempFile::new().unwrap(); let occurrence = "prefilter-gap-occurrence"; assert!( - evaluate_and_record_scheduler_skip(&dashboard_root, &config, None, &job, occurrence, None) - .await - .unwrap() - .is_none(), + evaluate_and_record_scheduler_skip( + &dashboard_root, + &config, + Some(executable.path()), + &job, + occurrence, + None, + ) + .await + .unwrap() + .is_none(), "a new interval job is advisory-due before effect admission" ); let _raced_in = @@ -1276,7 +1321,7 @@ async fn retained_scheduler_runner_reacquires_after_due_prefilter() { .unwrap() .expect("the advisory prefilter must release its reservation lock"); - let backend = ContentBackend::new("unused"); + let backend = ContentBackend::new("unused").with_executable(executable.path()); let admitted_job = tracedecay_automation_runtime::automation::jobs::admit_job(&job).unwrap(); let retained = run_user_job_with_backend_for_retained_settlement( &dashboard_root, From 9cf67ec9b64bef5d570300fc8da89d3774bada3b Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Thu, 5 Nov 2026 23:19:16 -0800 Subject: [PATCH 17/71] fix(evals): require HTTP response fingerprints for durable evidence Reject resumed HTTP rows and scored experiment files that omit or malform their recorded raw response digest. An incomplete evidence row must not be silently treated as a verified model observation. Cover rejection before reinference, scored-summary preservation and exact recovery once the durable row is restored. Refs #4535 --- evals/intent_engine/run_http.py | 4 ++++ evals/intent_engine/score.py | 3 +++ evals/intent_engine/test_run_http.py | 27 +++++++++++++++++++++++ evals/intent_engine/test_score.py | 32 +++++++++++++++++++++++++++- 4 files changed, 65 insertions(+), 1 deletion(-) diff --git a/evals/intent_engine/run_http.py b/evals/intent_engine/run_http.py index 3946e616520..88a9aa40d08 100644 --- a/evals/intent_engine/run_http.py +++ b/evals/intent_engine/run_http.py @@ -292,6 +292,10 @@ def noul_probabilities(response, questions): def serialize_result(row, questions, temperature): """Admit fresh and resumed rows before any durable output is changed.""" probabilities(row.get("probs"), questions) + # Every durable HTTP row must retain its raw response fingerprint. + response_sha = row.get("response_sha256") + if not isinstance(response_sha, str) or not re.fullmatch(r"[0-9a-f]{64}", response_sha): + raise ValueError("result row is missing a valid response digest") for field in ("ms", "server_ms", "input_tokens", "state_tokens", "dropped"): if field not in row: raise ValueError(f"result row is missing {field} for {row.get('id')}") diff --git a/evals/intent_engine/score.py b/evals/intent_engine/score.py index 0217210e86a..7fd1f2b52ef 100644 --- a/evals/intent_engine/score.py +++ b/evals/intent_engine/score.py @@ -100,6 +100,9 @@ def validate_http_provenance(result, rows): request_sha = row.get("request_sha256") if not isinstance(request_sha, str) or not re.fullmatch(r"[0-9a-f]{64}", request_sha): raise ValueError(f"HTTP case {row['id']} has an invalid request fingerprint") + response_sha = row.get("response_sha256") + if not isinstance(response_sha, str) or not re.fullmatch(r"[0-9a-f]{64}", response_sha): + raise ValueError(f"HTTP case {row['id']} is missing a valid response digest") requests.append([row["id"], request_sha]) if experiment.get("requests_sha256") != evidence_digest(requests): raise ValueError("HTTP request fingerprints disagree with experiment identity") diff --git a/evals/intent_engine/test_run_http.py b/evals/intent_engine/test_run_http.py index ab0b1b963e5..be102d9364a 100644 --- a/evals/intent_engine/test_run_http.py +++ b/evals/intent_engine/test_run_http.py @@ -1041,6 +1041,33 @@ def test_existing_sample_from_other_run_refuses_without_inference(self): self.assertFalse(self.partial.exists()) self.assertFalse(self.output.exists()) + def test_missing_response_digest_refuses_resume_before_inference(self): + self.interrupt() + original = self.partial.read_bytes() + lines = original.splitlines() + self.assertEqual(len(lines), 2) + for invalid in ("missing", "malformed"): + with self.subTest(invalid=invalid): + saved = json.loads(lines[1]) + if invalid == "missing": + del saved["response_sha256"] + else: + saved["response_sha256"] = "not-a-sha256" + altered = lines[0] + b"\n" + json.dumps(saved).encode("utf-8") + b"\n" + self.partial.write_bytes(altered) + with patch.object(run_http, "post") as post: + with self.assertRaisesRegex(ValueError, "response digest"): + self.run_main() + post.assert_not_called() + self.assertEqual(self.partial.read_bytes(), altered) + self.assertFalse(self.output.exists()) + self.partial.write_bytes(original) + with patch.object(run_http, "post", side_effect=self.response) as post: + self.run_main() + post.assert_called_once() + self.assertFalse(self.partial.exists()) + self.assertEqual(len(json.loads(self.output.read_text())["rows"]), 2) + def test_sample_identity_and_raw_response_must_match_durable_resume(self): sample = self.root / "first-response.json" extra = ["--sample-out", str(sample)] diff --git a/evals/intent_engine/test_score.py b/evals/intent_engine/test_score.py index 52e734a20a4..a00d7e0310d 100644 --- a/evals/intent_engine/test_score.py +++ b/evals/intent_engine/test_score.py @@ -30,7 +30,8 @@ def row(case_id, labels=(), context=False, choice=None): def signed_http_result(config, rows, temperature=None): # Fixture independently reproduces the fingerprint scheme in run_http.py. rows = [ - {**item, "request_sha256": hashlib.sha256(item["id"].encode()).hexdigest()} + {**item, "request_sha256": hashlib.sha256(item["id"].encode()).hexdigest(), + "response_sha256": hashlib.sha256((item["id"] + ":response").encode()).hexdigest()} for item in rows ] @@ -460,6 +461,35 @@ def test_http_provenance_cannot_be_downgraded_to_local_scoring(self): self.assertIn("missing its experiment identity", attempt.stderr) self.assertEqual(summary.read_bytes(), b"last verified summary\n") + def test_http_response_digest_required_before_scoring_or_publishing(self): + original = signed_http_result("B_noul_text", [row("first"), row("second")]) + self.assertEqual(score.score_config(original)["latency"]["total_cases"], 2) + for bad_digest in (None, "", "not-a-sha256", "G" * 64, 1): + changed = json.loads(json.dumps(original)) + if bad_digest is None: + del changed["rows"][0]["response_sha256"] + else: + changed["rows"][0]["response_sha256"] = bad_digest + with self.subTest(bad_digest=bad_digest): + with self.assertRaisesRegex(ValueError, "response digest"): + score.score_config(changed) + + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + self.stage_scripts(root) + (root / "results").mkdir() + damaged = json.loads(json.dumps(original)) + del damaged["rows"][0]["response_sha256"] + (root / "results" / "damaged.json").write_text( + json.dumps(damaged), encoding="utf-8" + ) + summary = root / "summary.json" + summary.write_bytes(b"last verified summary\n") + attempt = self.run_script(root, "score.py") + self.assertNotEqual(attempt.returncode, 0) + self.assertIn("response digest", attempt.stderr) + self.assertEqual(summary.read_bytes(), b"last verified summary\n") + def test_http_result_fingerprints_bind_case_order_and_experiment(self): original = signed_http_result("B_noul_text", [row("first"), row("second")]) self.assertEqual(score.score_config(original)["latency"]["total_cases"], 2) From 25c870139e7c6f975cabf304cc9fbed800f5c349 Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Fri, 6 Nov 2026 00:20:26 -0800 Subject: [PATCH 18/71] fix(storage): deduplicate SQLite TEMP spill descriptors Count each unlinked SQLite TEMP physical file once when multiple Unix file descriptors refer to the same inode. Preserve unreadable-entry coverage and other platforms' behavior. Regress duplicate aliases that otherwise produce a false over-budget doctor finding. Refs #4152 --- .../src/retention/footprint.rs | 52 ++++++++++++++++++- 1 file changed, 51 insertions(+), 1 deletion(-) diff --git a/crates/tracedecay-maintenance/src/retention/footprint.rs b/crates/tracedecay-maintenance/src/retention/footprint.rs index 7c6d4ead0c0..e14b1e9eee0 100644 --- a/crates/tracedecay-maintenance/src/retention/footprint.rs +++ b/crates/tracedecay-maintenance/src/retention/footprint.rs @@ -7,6 +7,10 @@ //! cheap is smaller than one real graph container. use std::collections::BTreeMap; +#[cfg(unix)] +use std::collections::HashSet; +#[cfg(unix)] +use std::os::unix::fs::MetadataExt; use std::path::{Path, PathBuf}; use serde::{Deserialize, Serialize}; @@ -408,6 +412,8 @@ pub fn census_sqlite_temp_spill( ) -> Option { let mut bytes = 0u64; let mut unavailable_entry_count = 0usize; + #[cfg(unix)] + let mut seen_inodes = HashSet::new(); for entry in std::fs::read_dir(fd_directory).ok()? { let fd = match entry { Ok(fd) => fd, @@ -431,7 +437,15 @@ pub fn census_sqlite_temp_spill( .is_some_and(|name| name.starts_with("etilqs_") && name.ends_with(" (deleted)")); if unlinked_temp { match std::fs::metadata(fd.path()) { - Ok(metadata) => bytes = bytes.saturating_add(metadata.len()), + Ok(metadata) => { + #[cfg(unix)] + let unique = seen_inodes.insert((metadata.dev(), metadata.ino())); + #[cfg(not(unix))] + let unique = true; + if unique { + bytes = bytes.saturating_add(metadata.len()); + } + } // A closed descriptor no longer consumes space. Err(error) if error.kind() == std::io::ErrorKind::NotFound => {} Err(_) => unavailable_entry_count = unavailable_entry_count.saturating_add(1), @@ -741,6 +755,42 @@ mod tests { ); } + #[cfg(unix)] + #[test] + fn temp_spill_census_counts_physical_files_not_fd_aliases() { + let temporary = tempfile::tempdir().expect("temporary directory"); + let fd_directory = temporary.path().join("fd"); + std::fs::create_dir(&fd_directory).expect("fd directory"); + let first = temporary.path().join("etilqs_primary (deleted)"); + let alias = temporary.path().join("etilqs_alias (deleted)"); + let second = temporary.path().join("etilqs_distinct (deleted)"); + sized(&first, 650 * MIB); + std::fs::hard_link(&first, &alias).expect("second name for same inode"); + sized(&second, 200 * MIB); + for (fd, target) in [("3", &first), ("4", &alias), ("5", &second)] { + std::os::unix::fs::symlink(target, fd_directory.join(fd)).expect("fd link"); + } + + let entry = census_sqlite_temp_spill(&fd_directory, &BTreeMap::new()) + .expect("readable fd directory"); + assert_eq!(entry.bytes, 850 * MIB); + assert_eq!(entry.unavailable_entry_count, 0); + let footprint = ProfileFootprintV1 { + entries: vec![entry], + total_bytes: 0, + unavailable_entry_count: 0, + truncated: false, + total_bound_bytes: PROFILE_TOTAL_BOUND_BYTES, + }; + assert_eq!( + findings(&footprint)[SQLITE_TEMP_SPILL_KEY] + .finding() + .state(), + DoctorEvidenceStateV1::HealthyCompleteCoverage, + "duplicated descriptors must not create a false over-budget finding", + ); + } + #[cfg(unix)] #[test] fn temp_spill_census_reports_unreadable_fd_entries_without_fabricating_health() { From 3edf4b8d7904f5d7af70229e33309818b4c9c330 Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Fri, 6 Nov 2026 01:14:27 -0800 Subject: [PATCH 19/71] fix(storage): exclude non-file SQLite TEMP fd targets Ignore nonregular entries if a descriptor is reused between resolving its (deleted) pathname and sampling metadata. Keep existing inode deduplication and unreadable-entry reporting. Regression covers a directory with a matching TEMP basename without changing the valid spill total. Refs #4152 --- .../tracedecay-maintenance/src/retention/footprint.rs | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/crates/tracedecay-maintenance/src/retention/footprint.rs b/crates/tracedecay-maintenance/src/retention/footprint.rs index e14b1e9eee0..3fc71f40334 100644 --- a/crates/tracedecay-maintenance/src/retention/footprint.rs +++ b/crates/tracedecay-maintenance/src/retention/footprint.rs @@ -437,7 +437,7 @@ pub fn census_sqlite_temp_spill( .is_some_and(|name| name.starts_with("etilqs_") && name.ends_with(" (deleted)")); if unlinked_temp { match std::fs::metadata(fd.path()) { - Ok(metadata) => { + Ok(metadata) if metadata.is_file() => { #[cfg(unix)] let unique = seen_inodes.insert((metadata.dev(), metadata.ino())); #[cfg(not(unix))] @@ -446,6 +446,9 @@ pub fn census_sqlite_temp_spill( bytes = bytes.saturating_add(metadata.len()); } } + // The fd can be reused between read_link and metadata; a + // directory or other non-file target is not an SQLite spill. + Ok(_) => {} // A closed descriptor no longer consumes space. Err(error) if error.kind() == std::io::ErrorKind::NotFound => {} Err(_) => unavailable_entry_count = unavailable_entry_count.saturating_add(1), @@ -730,6 +733,11 @@ mod tests { sized(&target, bytes); std::os::unix::fs::symlink(&target, fd_directory.join(fd)).expect("fd link"); } + // A reused descriptor with a matching name but a non-file target + // must not inflate the spill total or fabricate an over-budget state. + let non_file = temporary.path().join("etilqs_directory (deleted)"); + std::fs::create_dir(&non_file).expect("non-file target"); + std::os::unix::fs::symlink(&non_file, fd_directory.join("7")).expect("non-file fd link"); let entry = census_sqlite_temp_spill(&fd_directory, &BTreeMap::new()) .expect("a readable fd directory"); From 6459ca22496d4b1c467524edb1a94a749f889ac0 Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Fri, 6 Nov 2026 03:22:16 -0800 Subject: [PATCH 20/71] fix(sqlite): preserve live writer locks across verified readers Avoid opening and closing independent Unix file descriptors for verified read snapshots: their close releases unrelated POSIX SQLite record locks. Verify the canonical inode on both sides of SQLite HAS_MOVED and reject unsupported VFS proof without a raw pin. Preserve Windows pinned handles. Exercise rollback and WAL writer locks against an external process while live and immutable readers open and close, then verify committed data. Refs #4513 --- .../src/connection/mod.rs | 59 +++++++++++++++---- .../src/connection/path_identity_tests.rs | 53 +++++++++++++++++ 2 files changed, 100 insertions(+), 12 deletions(-) diff --git a/crates/tracedecay-rusqlite-runtime/src/connection/mod.rs b/crates/tracedecay-rusqlite-runtime/src/connection/mod.rs index fd4dae1b6ff..a22bd4308d2 100644 --- a/crates/tracedecay-rusqlite-runtime/src/connection/mod.rs +++ b/crates/tracedecay-rusqlite-runtime/src/connection/mod.rs @@ -262,7 +262,7 @@ impl OpenedDatabaseFile { // HAS_MOVED below still observes the SQLite handle's different inode. self.verify_current_path(canonical_path)?; #[cfg(unix)] - if sqlite_connection_has_moved(_connection)? { + if sqlite_connection_has_moved(_connection)?.unwrap_or(false) { return Err(OpenedDatabaseFileError::Replaced); } #[cfg(unix)] @@ -416,7 +416,9 @@ fn metadata_identity(metadata: &std::fs::Metadata) -> u64 { } #[cfg(unix)] -fn sqlite_connection_has_moved(connection: &Connection) -> Result { +fn sqlite_connection_has_moved( + connection: &Connection, +) -> Result, OpenedDatabaseFileError> { let database_name = CString::new("main").expect("static SQLite database name"); let mut moved = 0_i32; // SAFETY: `connection` owns a live SQLite handle, `database_name` is a @@ -431,10 +433,10 @@ fn sqlite_connection_has_moved(connection: &Connection) -> Result Ok(moved != 0), - // VFS implementations predating HAS_MOVED report NOTFOUND. The - // pinned dev/inode check remains authoritative in that case. - rusqlite::ffi::SQLITE_NOTFOUND => Ok(false), + rusqlite::ffi::SQLITE_OK => Ok(Some(moved != 0)), + // Workers with a retained raw pin may use the pinned identity when + // HAS_MOVED is unsupported. Unpinned verified readers must refuse it. + rusqlite::ffi::SQLITE_NOTFOUND => Ok(None), _ => Err(OpenedDatabaseFileError::Inspect), } } @@ -606,12 +608,13 @@ impl std::error::Error for VerifiedReaderError { /// Foreign-database reader bound to the physical file actually /// opened by SQLite rather than a later pathname observation. pub struct VerifiedReader { - // Keep the connection before the raw pin in declaration order: closing - // any raw descriptor on Unix can release another SQLite connection's - // process-wide record locks. SQLite must close its handle first. connection: Connection, canonical_path: PathBuf, file_identity: u64, + // On Unix a raw pin's eventual close would release POSIX record locks + // belonging to unrelated SQLite connections on the same inode. The + // SQLite handle itself retains the file; Windows keeps its pin. + #[cfg(not(unix))] _opened_file: OpenedDatabaseFile, } @@ -632,8 +635,8 @@ impl VerifiedReader { self.file_identity } - /// The verified canonical pathname. It remains bound to this reader's - /// retained physical file until the SQLite connection is closed. + /// The verified canonical pathname, bound to the SQLite connection's + /// physical file until the connection is closed. pub fn canonical_path(&self) -> &Path { &self.canonical_path } @@ -862,20 +865,52 @@ fn open_verified_reader( after_open: impl FnOnce(), ) -> Result { let canonical_path = path.canonicalize().map_err(VerifiedReaderError::Resolve)?; + // Opening and later closing even a read-only raw descriptor on Unix + // releases POSIX record locks of other live SQLite connections to this + // inode. Observe the pathname without opening it and let SQLite own its + // only descriptor. Check identity on both sides of HAS_MOVED to reject + // a concurrent replacement, including a swap restored after open. + #[cfg(unix)] + let file_identity = + live_path_identity(&canonical_path).map_err(VerifiedReaderError::Identity)?; + #[cfg(unix)] + let open_path = canonical_path.clone(); + + #[cfg(not(unix))] let pinned = OpenedDatabaseFile::pin(&canonical_path).map_err(VerifiedReaderError::Identity)?; + #[cfg(not(unix))] let open_path = pinned .reader_open_path(&canonical_path) .map_err(VerifiedReaderError::Identity)?; + #[cfg(not(unix))] + let file_identity = pinned.identity(); + after_pin(); let connection = open_reader(&open_path).map_err(VerifiedReaderError::Policy)?; after_open(); + + #[cfg(unix)] + { + let before = live_path_identity(&canonical_path).map_err(VerifiedReaderError::Identity)?; + let moved = sqlite_connection_has_moved(&connection) + .map_err(VerifiedReaderError::Identity)? + .ok_or(VerifiedReaderError::Identity(OpenedDatabaseFileError::Inspect))?; + let after = live_path_identity(&canonical_path).map_err(VerifiedReaderError::Identity)?; + if before != file_identity || moved || after != file_identity { + return Err(VerifiedReaderError::Identity( + OpenedDatabaseFileError::Replaced, + )); + } + } + #[cfg(not(unix))] pinned .verify_connection(&connection, &canonical_path) .map_err(VerifiedReaderError::Identity)?; Ok(VerifiedReader { connection, canonical_path, - file_identity: pinned.identity(), + file_identity, + #[cfg(not(unix))] _opened_file: pinned, }) } diff --git a/crates/tracedecay-rusqlite-runtime/src/connection/path_identity_tests.rs b/crates/tracedecay-rusqlite-runtime/src/connection/path_identity_tests.rs index a8c0ce5fcf0..c4ea540d907 100644 --- a/crates/tracedecay-rusqlite-runtime/src/connection/path_identity_tests.rs +++ b/crates/tracedecay-rusqlite-runtime/src/connection/path_identity_tests.rs @@ -377,3 +377,56 @@ fn path_checks_keep_pinned_identity_and_reject_missing_or_replaced_files() { Err(OpenedDatabaseFileError::Replaced), ); } + +/// Closing a verified read snapshot must not release another SQLite +/// connection's process-wide POSIX locks. A second connection in this process +/// cannot prove this: run the lock/journal contender in a separate process. +#[test] +fn verified_reader_release_preserves_independent_writer_exclusion() { + for journal in ["DELETE", "WAL"] { + for immutable in [false, true] { + let temp = TempDir::new().expect("tempdir"); + let path = temp.path().join("sessions.db"); + let writer = Connection::open(&path).expect("writer"); + writer + .execute_batch(&format!( + "PRAGMA journal_mode = {journal}; + CREATE TABLE sample(value INTEGER); + INSERT INTO sample VALUES (1); + BEGIN IMMEDIATE; + UPDATE sample SET value = 2;" + )) + .expect("hold SQLite writer lock"); + let operation = if journal == "WAL" { "journal" } else { "write" }; + probe_external_connection(&path, operation, true); + + let reader = if immutable { + super::open_verified_immutable_reader(&path) + } else { + super::open_verified_read_snapshot(&path) + } + .expect("verified read-only connection"); + assert_eq!(reader.file_identity(), live_path_identity(&path).unwrap()); + assert_eq!( + reader + .connection() + .query_row("SELECT 1", [], |row| row.get::<_, i64>(0)), + Ok(1) + ); + drop(reader); + probe_external_connection(&path, operation, true); + + writer.execute_batch("COMMIT").expect("commit writer"); + if journal == "WAL" { + probe_external_connection(&path, "journal", true); + } else { + probe_external_connection(&path, "write", false); + } + drop(writer); + if journal == "WAL" { + probe_external_connection(&path, "journal", false); + } + assert_committed_data(&path); + } + } +} From abdbe4ef36b99891681003755ba75bd454d22162 Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Fri, 6 Nov 2026 04:13:34 -0800 Subject: [PATCH 21/71] fix(storage): align verified-reader lifetime test with Unix locking Require exactly one SQLite database descriptor for a verified Unix reader; keep clone and final-drop assertions while rejecting the obsolete raw-pin contract. Format the fail-closed HAS_MOVED refusal as required by rustfmt. Refs #4513 --- .../src/connection/mod.rs | 4 +++- .../src/runtime/host_sqlite.rs | 15 ++++++++------- 2 files changed, 11 insertions(+), 8 deletions(-) diff --git a/crates/tracedecay-rusqlite-runtime/src/connection/mod.rs b/crates/tracedecay-rusqlite-runtime/src/connection/mod.rs index a22bd4308d2..74df7011a33 100644 --- a/crates/tracedecay-rusqlite-runtime/src/connection/mod.rs +++ b/crates/tracedecay-rusqlite-runtime/src/connection/mod.rs @@ -894,7 +894,9 @@ fn open_verified_reader( let before = live_path_identity(&canonical_path).map_err(VerifiedReaderError::Identity)?; let moved = sqlite_connection_has_moved(&connection) .map_err(VerifiedReaderError::Identity)? - .ok_or(VerifiedReaderError::Identity(OpenedDatabaseFileError::Inspect))?; + .ok_or(VerifiedReaderError::Identity( + OpenedDatabaseFileError::Inspect, + ))?; let after = live_path_identity(&canonical_path).map_err(VerifiedReaderError::Identity)?; if before != file_identity || moved || after != file_identity { return Err(VerifiedReaderError::Identity( diff --git a/crates/tracedecay-sessions/src/runtime/host_sqlite.rs b/crates/tracedecay-sessions/src/runtime/host_sqlite.rs index b48ccd3cf04..9a358d06b64 100644 --- a/crates/tracedecay-sessions/src/runtime/host_sqlite.rs +++ b/crates/tracedecay-sessions/src/runtime/host_sqlite.rs @@ -1,8 +1,8 @@ use std::sync::{Arc, Mutex, PoisonError}; -/// A plain SQLite read or a verified foreign read. The verified case retains -/// its inode pin for the whole connection lifetime instead of unpacking and -/// dropping it immediately after admission. +/// A plain SQLite read or a verified foreign read. The verified case keeps +/// the SQLite connection alive through the last cloned handle. Unix must not +/// retain an additional raw inode pin: closing it releases unrelated locks. enum SqliteReadHandle { Plain(rusqlite::Connection), Verified(tracedecay_rusqlite_runtime::VerifiedReader), @@ -72,7 +72,7 @@ mod tests { } #[test] - fn verified_foreign_reader_retains_its_pin_until_the_last_handle_closes() { + fn verified_foreign_reader_keeps_its_sqlite_handle_until_last_clone_closes() { let dir = tempfile::tempdir().expect("test directory"); let path = dir.path().join("foreign.sqlite3"); { @@ -87,9 +87,10 @@ mod tests { let verified = tracedecay_rusqlite_runtime::open_verified_immutable_reader(&path) .expect("open verified foreign database"); let pinned_descriptors = descriptors_for(&path); - assert!( - pinned_descriptors >= 2, - "SQLite and its verified identity pin each retain an inode handle" + assert_eq!( + pinned_descriptors, + 1, + "verified Unix reads must not open a second raw descriptor to the SQLite inode" ); let reader = SqliteReadConn::new_verified(verified); From dd162ef600a7c65dd1a3b9dbd58f6451e4ab4285 Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Fri, 6 Nov 2026 05:09:18 -0800 Subject: [PATCH 22/71] fix(storage): bound sealed census to its own directory Do not traverse nested symbolic links when reporting sealed generation bytes. Otherwise a link to a foreign directory charges unrelated files, and a cyclic link can make the storage census never complete. Add a Unix regression with a real sealed head and a foreign directory symlink; retained/superseded byte accounting stays isolated. Refs #4616 --- .../tracedecay-graph-db/src/sealed_store.rs | 3 +- .../sealed_store.rs | 32 +++++++++++++++++++ 2 files changed, 34 insertions(+), 1 deletion(-) diff --git a/crates/tracedecay-graph-db/src/sealed_store.rs b/crates/tracedecay-graph-db/src/sealed_store.rs index e6441489824..afd60d38cc3 100644 --- a/crates/tracedecay-graph-db/src/sealed_store.rs +++ b/crates/tracedecay-graph-db/src/sealed_store.rs @@ -596,7 +596,8 @@ fn directory_bytes( while let Some(directory) = pending.pop() { for entry in std::fs::read_dir(&directory)? { let entry = entry?; - let metadata = entry.metadata()?; + // Sealed census must never follow a nested link outside its root. + let metadata = std::fs::symlink_metadata(entry.path())?; if metadata.is_dir() { pending.push(entry.path()); continue; diff --git a/crates/tracedecay-graph-db/tests/graph_db_suite/verified_generation_contract/sealed_store.rs b/crates/tracedecay-graph-db/tests/graph_db_suite/verified_generation_contract/sealed_store.rs index 3bb165ee0f6..db1f4f09ca2 100644 --- a/crates/tracedecay-graph-db/tests/graph_db_suite/verified_generation_contract/sealed_store.rs +++ b/crates/tracedecay-graph-db/tests/graph_db_suite/verified_generation_contract/sealed_store.rs @@ -1368,6 +1368,38 @@ fn sealed_census_separates_heads_from_superseded_and_abandoned_bytes() { assert_eq!(census.unrecognized_count, 1); assert!(census.head_bytes > 0 && census.superseded_bytes > 0); + #[cfg(unix)] + { + use std::os::unix::fs::symlink; + + // A foreign directory link inside a seal must not charge its target + // to the live generation (or recursively escape the census root). + let head = std::fs::read_dir(&root) + .unwrap() + .map(Result::unwrap) + .find(|entry| { + std::fs::read_to_string(entry.path().join("sealed.json")) + .is_ok_and(|receipt| receipt.contains("\"generation\": \"census-g2\"")) + }) + .unwrap() + .path(); + let outside = temp.path().join("unrelated"); + std::fs::create_dir_all(&outside).unwrap(); + std::fs::write(outside.join("foreign.data"), vec![0_u8; 1024 * 1024]).unwrap(); + let alias = head.join("foreign-directory-link"); + symlink(&outside, &alias).unwrap(); + + let after = + tracedecay_graph_db::census_sealed_store(&support::graph_path(temp.path()), &heads) + .unwrap(); + assert_eq!( + after.head_bytes, + census.head_bytes + std::fs::symlink_metadata(&alias).unwrap().len(), + "sealed census must charge only the link, not external target bytes" + ); + assert_eq!(after.superseded_bytes, census.superseded_bytes); + } + // With no heads journaled every sealed artifact reads as superseded, and // a store with no sealed root at all is an empty census, not an error. let none = tracedecay_graph_db::census_sealed_store( From 91e146cd35c0e40a04d405d6faa842298d83b77e Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Fri, 6 Nov 2026 05:25:36 -0800 Subject: [PATCH 23/71] style(sessions): format Unix SQLite reader fixture Apply the rustfmt 2024 layout required by repository gates for the verified single-descriptor regression. Preserve its behavioral checks. --- crates/tracedecay-sessions/src/runtime/host_sqlite.rs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/crates/tracedecay-sessions/src/runtime/host_sqlite.rs b/crates/tracedecay-sessions/src/runtime/host_sqlite.rs index 9a358d06b64..ad1405e428d 100644 --- a/crates/tracedecay-sessions/src/runtime/host_sqlite.rs +++ b/crates/tracedecay-sessions/src/runtime/host_sqlite.rs @@ -88,8 +88,7 @@ mod tests { .expect("open verified foreign database"); let pinned_descriptors = descriptors_for(&path); assert_eq!( - pinned_descriptors, - 1, + pinned_descriptors, 1, "verified Unix reads must not open a second raw descriptor to the SQLite inode" ); From a121f85409617ad559ca85385427c1738916d35b Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Fri, 6 Nov 2026 06:24:16 -0800 Subject: [PATCH 24/71] fix(graph): attribute shared sealed bytes to live heads Classify sealed directories before charging hard-linked base inodes. This keeps read_dir order and unknown crash debris from charging an active head's bytes to retired or abandoned generations. A Unix regression forces unfavorable traversal order and verifies ownership. Refs #4616 --- .../tracedecay-graph-db/src/sealed_store.rs | 121 +++++++++++++++++- 1 file changed, 114 insertions(+), 7 deletions(-) diff --git a/crates/tracedecay-graph-db/src/sealed_store.rs b/crates/tracedecay-graph-db/src/sealed_store.rs index afd60d38cc3..43992608165 100644 --- a/crates/tracedecay-graph-db/src/sealed_store.rs +++ b/crates/tracedecay-graph-db/src/sealed_store.rs @@ -547,8 +547,17 @@ pub fn census_sealed_store( } Err(error) => return Err(error), }; + census_sealed_store_entries(entries, head_generations) +} + +fn census_sealed_store_entries( + entries: impl IntoIterator>, + head_generations: &std::collections::BTreeSet, +) -> std::io::Result { let mut census = SealedStoreCensusV1::default(); - let mut counted = std::collections::HashSet::new(); + let mut heads = Vec::new(); + let mut superseded = Vec::new(); + let mut abandoned = Vec::new(); for entry in entries { let entry = entry?; if !entry.file_type()?.is_dir() { @@ -556,14 +565,13 @@ pub fn census_sealed_store( continue; } let path = entry.path(); - let bytes = directory_bytes(&path, &mut counted)?; - let name = entry.file_name(); - if name + if entry + .file_name() .to_str() .is_some_and(|name| name.starts_with(".staging-")) { census.abandoned_staging_count += 1; - census.abandoned_staging_bytes += bytes; + abandoned.push(path); continue; } let receipt = std::fs::read(path.join(SEALED_STORE_RECEIPT_FILE)) @@ -575,12 +583,32 @@ pub fn census_sealed_store( }; if head_generations.contains(&receipt.generation) { census.head_count += 1; - census.head_bytes += bytes; + heads.push(path); } else { census.superseded_count += 1; - census.superseded_bytes += bytes; + superseded.push(path); } } + + // A layered generation can hard-link its base across directories. Charge + // that inode to its live head before any superseded or abandoned alias, + // regardless of read_dir order. Unknown receipts have no charging authority. + let mut counted = std::collections::HashSet::new(); + for path in heads { + census.head_bytes = census + .head_bytes + .saturating_add(directory_bytes(&path, &mut counted)?); + } + for path in superseded { + census.superseded_bytes = census + .superseded_bytes + .saturating_add(directory_bytes(&path, &mut counted)?); + } + for path in abandoned { + census.abandoned_staging_bytes = census + .abandoned_staging_bytes + .saturating_add(directory_bytes(&path, &mut counted)?); + } Ok(census) } @@ -617,6 +645,85 @@ fn directory_bytes( Ok(total) } +#[cfg(all(test, unix))] +mod sealed_census_tests { + use std::collections::BTreeSet; + use std::fs; + + use super::{SEALED_STORE_RECEIPT_FILE, census_sealed_store_entries}; + + #[test] + fn shared_sealed_bytes_belong_to_live_head_regardless_of_directory_order() { + let temp = tempfile::tempdir().unwrap(); + let root = temp.path(); + for name in ["unrecognized", ".staging-pending", "old", "head"] { + fs::create_dir(root.join(name)).unwrap(); + } + let receipt = |generation: &str| { + serde_json::to_vec(&serde_json::json!({ + "version": 1, + "form": "compact", + "namespace": "code", + "projection": "index", + "generation": generation, + "physical_namespace": "generation:unused", + "recovered_digest": "unused", + "entities": 0, + "relations": 0, + "graph_format": 1, + "row_sum": null, + "base": null, + "source_generation": null, + "watermark": null + })) + .unwrap() + }; + let head_receipt = receipt("live"); + let old_receipt = receipt("retired"); + fs::write(root.join("head").join(SEALED_STORE_RECEIPT_FILE), &head_receipt).unwrap(); + fs::write(root.join("old").join(SEALED_STORE_RECEIPT_FILE), &old_receipt).unwrap(); + let shared = root.join("head/shared"); + fs::write(&shared, vec![0_u8; 8192]).unwrap(); + for directory in ["old", ".staging-pending", "unrecognized"] { + fs::hard_link(&shared, root.join(directory).join("shared")).unwrap(); + } + + // Put unknown, staging and retired paths first, reproducing the order + // that previously consumed the live inode before its head was visited. + let ordered = || { + let mut entries = fs::read_dir(root) + .unwrap() + .collect::>>() + .unwrap(); + entries.sort_by_key(|entry| match entry.file_name().to_str().unwrap() { + "unrecognized" => 0, + ".staging-pending" => 1, + "old" => 2, + "head" => 3, + _ => unreachable!(), + }); + entries.into_iter().map(Ok).collect::>() + }; + let census = census_sealed_store_entries(ordered(), &BTreeSet::from(["live".to_owned()])) + .unwrap(); + assert_eq!((census.head_count, census.superseded_count), (1, 1)); + assert_eq!( + (census.abandoned_staging_count, census.unrecognized_count), + (1, 1) + ); + assert_eq!(census.head_bytes, 8192 + head_receipt.len() as u64); + assert_eq!(census.superseded_bytes, old_receipt.len() as u64); + assert_eq!(census.abandoned_staging_bytes, 0); + + let no_head = census_sealed_store_entries(ordered(), &BTreeSet::new()).unwrap(); + assert_eq!(no_head.head_bytes, 0); + assert_eq!( + no_head.superseded_bytes, + 8192 + head_receipt.len() as u64 + old_receipt.len() as u64 + ); + } +} + /// Removes every sealed generation beside `database_path` whose receipt does /// not name the current graph format: after a superseded-format rebuild those /// containers can never serve, while a generation this build already sealed From df7ea08029dfd5e1b76139341c24a178498a20db Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Fri, 6 Nov 2026 07:21:13 -0800 Subject: [PATCH 25/71] fix(storage): preserve newer and unknown sealed artifacts Retire only well-identified older-format sealed generations. A downgrade must never erase a future-format generation, and a malformed receipt or foreign entry is not proof of owned obsolete data. Exercise the old-format retirement and preservation cases against real filesystem entries. Refs #4616 --- .../tracedecay-graph-db/src/sealed_store.rs | 130 +++++++++++++++--- 1 file changed, 108 insertions(+), 22 deletions(-) diff --git a/crates/tracedecay-graph-db/src/sealed_store.rs b/crates/tracedecay-graph-db/src/sealed_store.rs index 43992608165..bf4c9dafdf0 100644 --- a/crates/tracedecay-graph-db/src/sealed_store.rs +++ b/crates/tracedecay-graph-db/src/sealed_store.rs @@ -724,10 +724,10 @@ mod sealed_census_tests { } } -/// Removes every sealed generation beside `database_path` whose receipt does -/// not name the current graph format: after a superseded-format rebuild those -/// containers can never serve, while a generation this build already sealed -/// keeps serving. In-flight `.staging-*` seals are left to their builder. +/// Removes sealed generations with a proven older graph format. A receipt +/// written by a newer build, an unrecognized artifact, or a non-generation +/// entry is not safe to delete after a downgrade. In-flight `.staging-*` +/// directories remain owned by their builder. pub(crate) fn discard_superseded_sealed_generations( database_path: &Path, ) -> Result<(), GraphDbError> { @@ -740,34 +740,120 @@ pub(crate) fn discard_superseded_sealed_generations( for entry in entries { let entry = entry.map_err(|error| sealed_store_io_failure("sealed root read failed", error))?; + if !entry + .file_type() + .map_err(|error| sealed_store_io_failure("sealed entry type failed", error))? + .is_dir() + { + continue; + } + let name = entry.file_name(); + let Some(name) = name.to_str() else { + continue; + }; + if name.len() != 64 || !name.bytes().all(|byte| byte.is_ascii_hexdigit()) { + continue; + } let path = entry.path(); - if entry - .file_name() - .to_str() - .is_some_and(|name| name.starts_with(".staging-")) + let Some(receipt) = load_sealed_store_receipt(&path).ok().flatten() else { + continue; + }; + if receipt.version != SEALED_STORE_RECEIPT_VERSION + || receipt.physical_namespace != format!("generation:{name}") + || receipt.graph_format >= GraphFormatVersion::current().get() { continue; } - let current = load_sealed_store_receipt(&path) - .ok() - .flatten() - .is_some_and(|receipt| receipt.graph_format == GraphFormatVersion::current().get()); - if !current { - match std::fs::remove_dir_all(&path) { - Ok(()) => {} - Err(error) if error.kind() == std::io::ErrorKind::NotFound => {} - Err(error) => { - return Err(sealed_store_io_failure( - "superseded sealed generation removal failed", - error, - )); - } + match std::fs::remove_dir_all(&path) { + Ok(()) => {} + Err(error) if error.kind() == std::io::ErrorKind::NotFound => {} + Err(error) => { + return Err(sealed_store_io_failure( + "superseded sealed generation removal failed", + error, + )); } } } Ok(()) } +#[cfg(test)] +mod superseded_sealed_cleanup_tests { + use super::*; + + #[test] + fn only_proven_older_generation_artifacts_can_be_removed() { + let temp = tempfile::tempdir().unwrap(); + let database = temp.path().join("graph.grafeo"); + let root = sealed_store_root(&database); + std::fs::create_dir(&root).unwrap(); + let current = GraphFormatVersion::current().get(); + assert!(current > 0); + let obsolete = "a".repeat(64); + let future = "b".repeat(64); + let wrong_identity = "c".repeat(64); + let unreadable = "d".repeat(64); + let ordinary_file = "e".repeat(64); + let current_name = "f".repeat(64); + let unrelated = "operator-data"; + + let receipt = |name: &str, format: u32, physical_namespace: String| { + let directory = root.join(name); + std::fs::create_dir(&directory).unwrap(); + std::fs::write( + directory.join(SEALED_STORE_RECEIPT_FILE), + serde_json::to_vec(&serde_json::json!({ + "version": SEALED_STORE_RECEIPT_VERSION, + "form": "compact", + "namespace": "code", + "projection": "index", + "generation": "g", + "physical_namespace": physical_namespace, + "recovered_digest": "digest", + "entities": 0, + "relations": 0, + "graph_format": format, + "row_sum": null, + "base": null, + "source_generation": null, + "watermark": null + })) + .unwrap(), + ) + .unwrap(); + }; + receipt(&obsolete, current - 1, format!("generation:{obsolete}")); + receipt(&future, current + 1, format!("generation:{future}")); + receipt(&wrong_identity, current - 1, "generation:foreign".to_owned()); + receipt(¤t_name, current, format!("generation:{current_name}")); + receipt(unrelated, current - 1, format!("generation:{unrelated}")); + std::fs::create_dir(root.join(&unreadable)).unwrap(); + std::fs::write( + root.join(&unreadable).join(SEALED_STORE_RECEIPT_FILE), + b"not json", + ) + .unwrap(); + std::fs::write(root.join(&ordinary_file), b"operator file").unwrap(); + std::fs::create_dir(root.join(".staging-active")).unwrap(); + + discard_superseded_sealed_generations(&database).unwrap(); + + assert!(!root.join(&obsolete).exists(), "old verified format is retired"); + for name in [ + future.as_str(), + wrong_identity.as_str(), + unreadable.as_str(), + ordinary_file.as_str(), + current_name.as_str(), + unrelated, + ".staging-active", + ] { + assert!(root.join(name).exists(), "must preserve {name}"); + } + } +} + /// Removes every `.staging-*` directory a seal left behind under the store's /// sealed root. /// From bab192af8628a7a4c6ac6f6f5d4332378717c4d6 Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Fri, 6 Nov 2026 07:26:06 -0800 Subject: [PATCH 26/71] style(graph): format shared-inode census regression Match the repository's rustfmt 2024 output for the shared-inode census test. Preserve the test's unfavorable traversal order, byte-accounting assertions, and sealed-store cleanup behavior. --- crates/tracedecay-graph-db/src/sealed_store.rs | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/crates/tracedecay-graph-db/src/sealed_store.rs b/crates/tracedecay-graph-db/src/sealed_store.rs index bf4c9dafdf0..f8e894a7cb8 100644 --- a/crates/tracedecay-graph-db/src/sealed_store.rs +++ b/crates/tracedecay-graph-db/src/sealed_store.rs @@ -680,8 +680,16 @@ mod sealed_census_tests { }; let head_receipt = receipt("live"); let old_receipt = receipt("retired"); - fs::write(root.join("head").join(SEALED_STORE_RECEIPT_FILE), &head_receipt).unwrap(); - fs::write(root.join("old").join(SEALED_STORE_RECEIPT_FILE), &old_receipt).unwrap(); + fs::write( + root.join("head").join(SEALED_STORE_RECEIPT_FILE), + &head_receipt, + ) + .unwrap(); + fs::write( + root.join("old").join(SEALED_STORE_RECEIPT_FILE), + &old_receipt, + ) + .unwrap(); let shared = root.join("head/shared"); fs::write(&shared, vec![0_u8; 8192]).unwrap(); for directory in ["old", ".staging-pending", "unrecognized"] { @@ -704,8 +712,8 @@ mod sealed_census_tests { }); entries.into_iter().map(Ok).collect::>() }; - let census = census_sealed_store_entries(ordered(), &BTreeSet::from(["live".to_owned()])) - .unwrap(); + let census = + census_sealed_store_entries(ordered(), &BTreeSet::from(["live".to_owned()])).unwrap(); assert_eq!((census.head_count, census.superseded_count), (1, 1)); assert_eq!( (census.abandoned_staging_count, census.unrecognized_count), From 4a26c92c50edadc293445496ea12371ca7636d04 Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Fri, 6 Nov 2026 08:19:10 -0800 Subject: [PATCH 27/71] fix(graph): refuse symlinked sealed roots during cleanup Prevent sealed artifact retirement, incompatible-format cleanup and abandoned staging/spill sweeps from traversing a foreign directory through a symlinked sealed root. Require a real root for census too. Add a Unix regression proving outside artifacts remain untouched. Refs #4616 --- .../tracedecay-graph-db/src/sealed_store.rs | 115 +++++++++++++++--- 1 file changed, 101 insertions(+), 14 deletions(-) diff --git a/crates/tracedecay-graph-db/src/sealed_store.rs b/crates/tracedecay-graph-db/src/sealed_store.rs index f8e894a7cb8..28b4de733d2 100644 --- a/crates/tracedecay-graph-db/src/sealed_store.rs +++ b/crates/tracedecay-graph-db/src/sealed_store.rs @@ -535,17 +535,35 @@ pub struct SealedStoreCensusV1 { /// Metadata only: every entry is one receipt read plus a directory size walk, /// never a container open. An unreadable receipt counts as unrecognized so the /// census stays truthful when a receipt is mid-write or corrupt. +// A symlinked sealed root must not redirect cleanup or accounting into +// unrelated data. A missing root is normal before the first seal. +fn sealed_root_is_real_directory(root: &Path) -> std::io::Result { + match std::fs::symlink_metadata(root) { + Ok(metadata) if metadata.is_dir() => Ok(true), + Ok(_) => Err(std::io::Error::new( + std::io::ErrorKind::InvalidData, + "sealed store root is not a real directory", + )), + Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(false), + Err(error) => Err(error), + } +} + +fn sealed_root_entries(root: &Path) -> std::io::Result> { + if sealed_root_is_real_directory(root)? { + std::fs::read_dir(root).map(Some) + } else { + Ok(None) + } +} + pub fn census_sealed_store( database_path: &Path, head_generations: &std::collections::BTreeSet, ) -> std::io::Result { let root = sealed_store_root(database_path); - let entries = match std::fs::read_dir(&root) { - Ok(entries) => entries, - Err(error) if error.kind() == std::io::ErrorKind::NotFound => { - return Ok(SealedStoreCensusV1::default()); - } - Err(error) => return Err(error), + let Some(entries) = sealed_root_entries(&root)? else { + return Ok(SealedStoreCensusV1::default()); }; census_sealed_store_entries(entries, head_generations) } @@ -740,9 +758,9 @@ pub(crate) fn discard_superseded_sealed_generations( database_path: &Path, ) -> Result<(), GraphDbError> { let root = sealed_store_root(database_path); - let entries = match std::fs::read_dir(&root) { - Ok(entries) => entries, - Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(()), + let entries = match sealed_root_entries(&root) { + Ok(Some(entries)) => entries, + Ok(None) => return Ok(()), Err(error) => return Err(sealed_store_io_failure("sealed root read failed", error)), }; for entry in entries { @@ -790,6 +808,57 @@ pub(crate) fn discard_superseded_sealed_generations( mod superseded_sealed_cleanup_tests { use super::*; + #[cfg(unix)] + #[test] + fn foreign_directory_behind_a_symlinked_sealed_root_is_never_cleaned() { + let temp = tempfile::tempdir().unwrap(); + let database = temp.path().join("graph.grafeo"); + let root = sealed_store_root(&database); + let foreign = temp.path().join("foreign"); + std::fs::create_dir(&foreign).unwrap(); + + let current = GraphFormatVersion::current().get(); + assert!(current > 0); + let obsolete = "a".repeat(64); + let obsolete_path = foreign.join(&obsolete); + std::fs::create_dir(&obsolete_path).unwrap(); + std::fs::write( + obsolete_path.join(SEALED_STORE_RECEIPT_FILE), + serde_json::to_vec(&serde_json::json!({ + "version": SEALED_STORE_RECEIPT_VERSION, + "form": "compact", + "namespace": "code", + "projection": "index", + "generation": "foreign", + "physical_namespace": format!("generation:{obsolete}"), + "recovered_digest": "digest", + "entities": 0, + "relations": 0, + "graph_format": current - 1, + "row_sum": null, + "base": null, + "source_generation": null, + "watermark": null + })) + .unwrap(), + ) + .unwrap(); + for name in [".staging-abandoned", ".rows-0-foreign"] { + std::fs::create_dir(foreign.join(name)).unwrap(); + } + std::os::unix::fs::symlink(&foreign, &root).unwrap(); + + assert!(census_sealed_store(&database, &Default::default()).is_err()); + assert!(discard_superseded_sealed_generations(&database).is_err()); + sweep_abandoned_sealed_staging(&database); + sweep_abandoned_row_spills(&database); + assert!(remove_sealed_directory_checked(&root.join(&obsolete)).is_err()); + + for name in [obsolete.as_str(), ".staging-abandoned", ".rows-0-foreign"] { + assert!(foreign.join(name).is_dir(), "foreign artifact {name} was removed"); + } + } + #[test] fn only_proven_older_generation_artifacts_can_be_removed() { let temp = tempfile::tempdir().unwrap(); @@ -874,9 +943,9 @@ mod superseded_sealed_cleanup_tests { /// this store is in flight. pub(crate) fn sweep_abandoned_sealed_staging(database_path: &Path) { let root = sealed_store_root(database_path); - let entries = match std::fs::read_dir(&root) { - Ok(entries) => entries, - Err(error) if error.kind() == std::io::ErrorKind::NotFound => return, + let entries = match sealed_root_entries(&root) { + Ok(Some(entries)) => entries, + Ok(None) => return, Err(error) => { tracing::warn!( event = "sealed_staging_sweep_unreadable", @@ -928,8 +997,18 @@ const ROW_SPILL_PREFIX: &str = ".rows-"; /// whose spill removes itself when it drops. pub(crate) fn sweep_abandoned_row_spills(database_path: &Path) { let root = sealed_store_root(database_path); - let Ok(entries) = std::fs::read_dir(&root) else { - return; + let entries = match sealed_root_entries(&root) { + Ok(Some(entries)) => entries, + Ok(None) => return, + Err(error) => { + tracing::warn!( + event = "graph_row_spill_sweep_unreadable", + root = %root.display(), + %error, + "sealed root cannot be safely enumerated" + ); + return; + } }; let own = format!("{ROW_SPILL_PREFIX}{}-", std::process::id()); for entry in entries.flatten() { @@ -954,6 +1033,14 @@ pub(crate) fn sweep_abandoned_row_spills(database_path: &Path) { } fn remove_sealed_directory_checked(directory: &Path) -> Result<(), GraphDbError> { + let root = directory + .parent() + .ok_or_else(|| GraphDbError::unavailable("sealed artifact has no parent directory"))?; + match sealed_root_is_real_directory(root) { + Ok(true) => {} + Ok(false) => return Ok(()), + Err(error) => return Err(sealed_store_io_failure("sealed root validation failed", error)), + } match std::fs::remove_dir_all(directory) { Ok(()) => Ok(()), Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(()), From ab6875502074b72f00e653770887c686df29ed5c Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Fri, 6 Nov 2026 08:20:25 -0800 Subject: [PATCH 28/71] fix(graph): preserve unverified seals during format reset Align the format-reset regression with receipt-authenticated cleanup: retire a proven prior-format generation, retain an unverified artifact, and preserve the current sealed generation. Apply exact rustfmt 2024 formatting to the sealed-cleanup regression. --- .../tracedecay-graph-db/src/sealed_store.rs | 11 +++++- .../tests/graph_db_suite/runtime_contract.rs | 38 ++++++++++++++++++- 2 files changed, 45 insertions(+), 4 deletions(-) diff --git a/crates/tracedecay-graph-db/src/sealed_store.rs b/crates/tracedecay-graph-db/src/sealed_store.rs index 28b4de733d2..172aa27b940 100644 --- a/crates/tracedecay-graph-db/src/sealed_store.rs +++ b/crates/tracedecay-graph-db/src/sealed_store.rs @@ -902,7 +902,11 @@ mod superseded_sealed_cleanup_tests { }; receipt(&obsolete, current - 1, format!("generation:{obsolete}")); receipt(&future, current + 1, format!("generation:{future}")); - receipt(&wrong_identity, current - 1, "generation:foreign".to_owned()); + receipt( + &wrong_identity, + current - 1, + "generation:foreign".to_owned(), + ); receipt(¤t_name, current, format!("generation:{current_name}")); receipt(unrelated, current - 1, format!("generation:{unrelated}")); std::fs::create_dir(root.join(&unreadable)).unwrap(); @@ -916,7 +920,10 @@ mod superseded_sealed_cleanup_tests { discard_superseded_sealed_generations(&database).unwrap(); - assert!(!root.join(&obsolete).exists(), "old verified format is retired"); + assert!( + !root.join(&obsolete).exists(), + "old verified format is retired" + ); for name in [ future.as_str(), wrong_identity.as_str(), diff --git a/crates/tracedecay-graph-db/tests/graph_db_suite/runtime_contract.rs b/crates/tracedecay-graph-db/tests/graph_db_suite/runtime_contract.rs index 123b4e9f3f7..e57bebb16e7 100644 --- a/crates/tracedecay-graph-db/tests/graph_db_suite/runtime_contract.rs +++ b/crates/tracedecay-graph-db/tests/graph_db_suite/runtime_contract.rs @@ -1026,7 +1026,7 @@ fn wrong_tracedecay_format_requires_reset() { } #[test] -fn superseded_format_store_is_rebuilt_fresh_keeping_only_current_sealed_generations() { +fn superseded_format_store_rebuild_removes_proven_old_seals_but_keeps_unproven() { let temp = TempDir::new().unwrap(); let path = graph_path(temp.path()); // Format 4 kept a relation's payload properties on its locator as well as @@ -1063,13 +1063,43 @@ fn superseded_format_store_is_rebuilt_fresh_keeping_only_current_sealed_generati ) .unwrap(); raw.close().unwrap(); - let sealed_generation = temp.path().join("graph.sealed").join("0".repeat(64)); + let sealed_generation_name = "0".repeat(64); + let sealed_generation = temp.path().join("graph.sealed").join(&sealed_generation_name); std::fs::create_dir_all(&sealed_generation).unwrap(); std::fs::write( sealed_generation.join("generation.grafeo"), b"format 4 bytes", ) .unwrap(); + // A prior-format sealed directory is deletable only when its receipt + // proves both the old format and the matching physical namespace. + std::fs::write( + sealed_generation.join("sealed.json"), + serde_json::json!({ + "version": 1, + "form": "compact", + "namespace": "project", + "projection": "code", + "generation": "g0", + "physical_namespace": format!("generation:{sealed_generation_name}"), + "recovered_digest": "sha256:old", + "entities": 1, + "relations": 0, + "graph_format": previous_format as u32, + }) + .to_string(), + ) + .unwrap(); + + // An unverified artifact may belong to a future build. A format reset + // cannot infer its age or ownership from a directory name alone. + let unproven_generation = temp.path().join("graph.sealed").join("2".repeat(64)); + std::fs::create_dir_all(&unproven_generation).unwrap(); + std::fs::write( + unproven_generation.join("generation.grafeo"), + b"unknown sealed bytes", + ) + .unwrap(); // A generation this build sealed before the staging container was // replaced, as a publication racing the rebuild leaves it. @@ -1096,6 +1126,10 @@ fn superseded_format_store_is_rebuilt_fresh_keeping_only_current_sealed_generati let (_registered, db) = RegisteredGraph::open_lease(temp.path()).unwrap(); assert!(!sealed_generation.exists()); + assert!( + unproven_generation.join("generation.grafeo").exists(), + "an unverified generation must survive a format reset" + ); assert!(current_generation.join("sealed.json").exists()); assert_eq!( db.entity(&namespace(), &entity_id("stale"), live()) From d0a09f6eeb76cd3579ed918aeb7fbc00268b3286 Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Fri, 6 Nov 2026 08:21:34 -0800 Subject: [PATCH 29/71] docs(graph): keep census contract on its entry point Keep the public sealed census documentation attached to its function, not the adjacent filesystem root validation helper. Refs #4616 --- crates/tracedecay-graph-db/src/sealed_store.rs | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/crates/tracedecay-graph-db/src/sealed_store.rs b/crates/tracedecay-graph-db/src/sealed_store.rs index 172aa27b940..517efe4af82 100644 --- a/crates/tracedecay-graph-db/src/sealed_store.rs +++ b/crates/tracedecay-graph-db/src/sealed_store.rs @@ -529,12 +529,6 @@ pub struct SealedStoreCensusV1 { pub unrecognized_count: u64, } -/// Measure the sealed root beside `database_path` against the generation ids -/// of the store's current verified heads (as journaled: `:`). -/// -/// Metadata only: every entry is one receipt read plus a directory size walk, -/// never a container open. An unreadable receipt counts as unrecognized so the -/// census stays truthful when a receipt is mid-write or corrupt. // A symlinked sealed root must not redirect cleanup or accounting into // unrelated data. A missing root is normal before the first seal. fn sealed_root_is_real_directory(root: &Path) -> std::io::Result { @@ -557,6 +551,12 @@ fn sealed_root_entries(root: &Path) -> std::io::Result> } } +/// Measure the sealed root beside `database_path` against the generation ids +/// of the store's current verified heads (as journaled: `:`). +/// +/// Metadata only: every entry is one receipt read plus a directory size walk, +/// never a container open. An unreadable receipt counts as unrecognized so the +/// census stays truthful when a receipt is mid-write or corrupt. pub fn census_sealed_store( database_path: &Path, head_generations: &std::collections::BTreeSet, From fb80599a3347ba226035f8c987c3dc57c4a1be9d Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Fri, 6 Nov 2026 09:16:58 -0800 Subject: [PATCH 30/71] fix(graph-db): require bound sealed census receipts Refs #4616 --- .../tracedecay-graph-db/src/sealed_store.rs | 112 +++++++++++++++--- 1 file changed, 93 insertions(+), 19 deletions(-) diff --git a/crates/tracedecay-graph-db/src/sealed_store.rs b/crates/tracedecay-graph-db/src/sealed_store.rs index 517efe4af82..2a722cc3bf3 100644 --- a/crates/tracedecay-graph-db/src/sealed_store.rs +++ b/crates/tracedecay-graph-db/src/sealed_store.rs @@ -599,6 +599,19 @@ fn census_sealed_store_entries( census.unrecognized_count += 1; continue; }; + let name = entry.file_name(); + let Some(name) = name.to_str() else { + census.unrecognized_count += 1; + continue; + }; + if name.len() != 64 + || !name.bytes().all(|byte| byte.is_ascii_hexdigit()) + || receipt.version != SEALED_STORE_RECEIPT_VERSION + || receipt.physical_namespace != format!("generation:{name}") + { + census.unrecognized_count += 1; + continue; + } if head_generations.contains(&receipt.generation) { census.head_count += 1; heads.push(path); @@ -668,23 +681,36 @@ mod sealed_census_tests { use std::collections::BTreeSet; use std::fs; - use super::{SEALED_STORE_RECEIPT_FILE, census_sealed_store_entries}; + use super::{ + SEALED_STORE_RECEIPT_FILE, SEALED_STORE_RECEIPT_VERSION, census_sealed_store_entries, + }; #[test] fn shared_sealed_bytes_belong_to_live_head_regardless_of_directory_order() { let temp = tempfile::tempdir().unwrap(); let root = temp.path(); - for name in ["unrecognized", ".staging-pending", "old", "head"] { + let old = "a".repeat(64); + let head = "b".repeat(64); + let wrong_namespace = "c".repeat(64); + let future_version = "d".repeat(64); + for name in [ + "unrecognized", + ".staging-pending", + old.as_str(), + head.as_str(), + wrong_namespace.as_str(), + future_version.as_str(), + ] { fs::create_dir(root.join(name)).unwrap(); } - let receipt = |generation: &str| { + let receipt = |generation: &str, namespace: &str, version: u32| { serde_json::to_vec(&serde_json::json!({ - "version": 1, + "version": version, "form": "compact", "namespace": "code", "projection": "index", "generation": generation, - "physical_namespace": "generation:unused", + "physical_namespace": namespace, "recovered_digest": "unused", "entities": 0, "relations": 0, @@ -696,21 +722,54 @@ mod sealed_census_tests { })) .unwrap() }; - let head_receipt = receipt("live"); - let old_receipt = receipt("retired"); + let head_receipt = receipt( + "live", + &format!("generation:{head}"), + SEALED_STORE_RECEIPT_VERSION, + ); + let old_receipt = receipt( + "retired", + &format!("generation:{old}"), + SEALED_STORE_RECEIPT_VERSION, + ); fs::write( - root.join("head").join(SEALED_STORE_RECEIPT_FILE), + root.join(&head).join(SEALED_STORE_RECEIPT_FILE), &head_receipt, ) .unwrap(); fs::write( - root.join("old").join(SEALED_STORE_RECEIPT_FILE), + root.join(&old).join(SEALED_STORE_RECEIPT_FILE), &old_receipt, ) .unwrap(); - let shared = root.join("head/shared"); + // A parseable receipt cannot claim live bytes without matching the + // generation directory, physical namespace and receipt version. + for (directory, namespace, version) in [ + ( + "unrecognized", + "generation:unrecognized".to_owned(), + SEALED_STORE_RECEIPT_VERSION, + ), + ( + wrong_namespace.as_str(), + "generation:wrong".to_owned(), + SEALED_STORE_RECEIPT_VERSION, + ), + ( + future_version.as_str(), + format!("generation:{future_version}"), + SEALED_STORE_RECEIPT_VERSION + 1, + ), + ] { + fs::write( + root.join(directory).join(SEALED_STORE_RECEIPT_FILE), + receipt("live", &namespace, version), + ) + .unwrap(); + } + let shared = root.join(&head).join("shared"); fs::write(&shared, vec![0_u8; 8192]).unwrap(); - for directory in ["old", ".staging-pending", "unrecognized"] { + for directory in [old.as_str(), ".staging-pending", "unrecognized"] { fs::hard_link(&shared, root.join(directory).join("shared")).unwrap(); } @@ -721,12 +780,22 @@ mod sealed_census_tests { .unwrap() .collect::>>() .unwrap(); - entries.sort_by_key(|entry| match entry.file_name().to_str().unwrap() { - "unrecognized" => 0, - ".staging-pending" => 1, - "old" => 2, - "head" => 3, - _ => unreachable!(), + entries.sort_by_key(|entry| { + let name = entry.file_name(); + let name = name.to_str().unwrap(); + if name == "unrecognized" { + 0 + } else if name == ".staging-pending" { + 1 + } else if name == old { + 2 + } else if name == wrong_namespace || name == future_version { + 3 + } else if name == head { + 4 + } else { + unreachable!() + } }); entries.into_iter().map(Ok).collect::>() }; @@ -735,7 +804,7 @@ mod sealed_census_tests { assert_eq!((census.head_count, census.superseded_count), (1, 1)); assert_eq!( (census.abandoned_staging_count, census.unrecognized_count), - (1, 1) + (1, 3) ); assert_eq!(census.head_bytes, 8192 + head_receipt.len() as u64); assert_eq!(census.superseded_bytes, old_receipt.len() as u64); @@ -1046,7 +1115,12 @@ fn remove_sealed_directory_checked(directory: &Path) -> Result<(), GraphDbError> match sealed_root_is_real_directory(root) { Ok(true) => {} Ok(false) => return Ok(()), - Err(error) => return Err(sealed_store_io_failure("sealed root validation failed", error)), + Err(error) => { + return Err(sealed_store_io_failure( + "sealed root validation failed", + error, + )); + } } match std::fs::remove_dir_all(directory) { Ok(()) => Ok(()), From 046bc8550e3ca8b82445e70af163358a3a79d596 Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Fri, 6 Nov 2026 09:22:38 -0800 Subject: [PATCH 31/71] fix(graph): bind sealed census heads to projection identity A generation ID can recur in different graph projections. Keep the journal's namespace and projection when deciding which receipt is live, so a foreign projection cannot claim a head's charged sealed bytes. Cover the collision while retaining shared-inode accounting. --- .../src/doctor_kernel.rs | 13 +++-- .../tracedecay-graph-db/src/sealed_store.rs | 52 +++++++++++++++---- .../sealed_store.rs | 6 ++- 3 files changed, 56 insertions(+), 15 deletions(-) diff --git a/crates/tracedecay-daemon-service/src/doctor_kernel.rs b/crates/tracedecay-daemon-service/src/doctor_kernel.rs index 167442e0a0f..4624fcc55d7 100644 --- a/crates/tracedecay-daemon-service/src/doctor_kernel.rs +++ b/crates/tracedecay-daemon-service/src/doctor_kernel.rs @@ -825,11 +825,11 @@ pub async fn collect_code_generation_retention_findings( /// absent table on a store that never published, a lock, a corrupt row). async fn live_sealed_generations( graph: &tracedecay_runtime_core::db::Database, -) -> Option> { +) -> Option> { let mut rows = graph .read_connection() .query( - "SELECT replay.generation + "SELECT replay.namespace, replay.projection, replay.generation FROM graph_publication_replay_v1 AS replay LEFT JOIN graph_verified_heads_v1 AS head ON head.shard_id = replay.shard_id @@ -838,14 +838,19 @@ async fn live_sealed_generations( WHERE head.replay_sequence IS NULL OR replay.sequence >= head.replay_sequence UNION - SELECT generation FROM graph_publication_replay_dependencies_v1", + SELECT namespace, projection, generation + FROM graph_publication_replay_dependencies_v1", (), ) .await .ok()?; let mut heads = BTreeSet::new(); while let Some(row) = rows.next().await.ok()? { - heads.insert(row.get::(0).ok()?); + heads.insert(( + row.get::(0).ok()?, + row.get::(1).ok()?, + row.get::(2).ok()?, + )); } Some(heads) } diff --git a/crates/tracedecay-graph-db/src/sealed_store.rs b/crates/tracedecay-graph-db/src/sealed_store.rs index 2a722cc3bf3..3292111c13f 100644 --- a/crates/tracedecay-graph-db/src/sealed_store.rs +++ b/crates/tracedecay-graph-db/src/sealed_store.rs @@ -551,15 +551,15 @@ fn sealed_root_entries(root: &Path) -> std::io::Result> } } -/// Measure the sealed root beside `database_path` against the generation ids -/// of the store's current verified heads (as journaled: `:`). +/// Measure the sealed root beside `database_path` against the full +/// namespace, projection, and generation identities of verified heads. /// /// Metadata only: every entry is one receipt read plus a directory size walk, /// never a container open. An unreadable receipt counts as unrecognized so the /// census stays truthful when a receipt is mid-write or corrupt. pub fn census_sealed_store( database_path: &Path, - head_generations: &std::collections::BTreeSet, + head_generations: &std::collections::BTreeSet<(String, String, String)>, ) -> std::io::Result { let root = sealed_store_root(database_path); let Some(entries) = sealed_root_entries(&root)? else { @@ -570,7 +570,7 @@ pub fn census_sealed_store( fn census_sealed_store_entries( entries: impl IntoIterator>, - head_generations: &std::collections::BTreeSet, + head_generations: &std::collections::BTreeSet<(String, String, String)>, ) -> std::io::Result { let mut census = SealedStoreCensusV1::default(); let mut heads = Vec::new(); @@ -612,7 +612,11 @@ fn census_sealed_store_entries( census.unrecognized_count += 1; continue; } - if head_generations.contains(&receipt.generation) { + if head_generations.contains(&( + receipt.namespace.clone(), + receipt.projection.clone(), + receipt.generation.clone(), + )) { census.head_count += 1; heads.push(path); } else { @@ -693,6 +697,7 @@ mod sealed_census_tests { let head = "b".repeat(64); let wrong_namespace = "c".repeat(64); let future_version = "d".repeat(64); + let foreign_projection = "e".repeat(64); for name in [ "unrecognized", ".staging-pending", @@ -700,6 +705,7 @@ mod sealed_census_tests { head.as_str(), wrong_namespace.as_str(), future_version.as_str(), + foreign_projection.as_str(), ] { fs::create_dir(root.join(name)).unwrap(); } @@ -767,6 +773,21 @@ mod sealed_census_tests { ) .unwrap(); } + // A different projection may legitimately reuse the same generation + // ID; its sealed artifact must not borrow the verified head's status. + let mut foreign_receipt: serde_json::Value = serde_json::from_slice(&receipt( + "live", + &format!("generation:{foreign_projection}"), + SEALED_STORE_RECEIPT_VERSION, + )) + .unwrap(); + foreign_receipt["projection"] = serde_json::json!("other-projection"); + let foreign_receipt = serde_json::to_vec(&foreign_receipt).unwrap(); + fs::write( + root.join(&foreign_projection).join(SEALED_STORE_RECEIPT_FILE), + &foreign_receipt, + ) + .unwrap(); let shared = root.join(&head).join("shared"); fs::write(&shared, vec![0_u8; 8192]).unwrap(); for directory in [old.as_str(), ".staging-pending", "unrecognized"] { @@ -789,7 +810,7 @@ mod sealed_census_tests { 1 } else if name == old { 2 - } else if name == wrong_namespace || name == future_version { + } else if name == wrong_namespace || name == future_version || name == foreign_projection { 3 } else if name == head { 4 @@ -800,21 +821,32 @@ mod sealed_census_tests { entries.into_iter().map(Ok).collect::>() }; let census = - census_sealed_store_entries(ordered(), &BTreeSet::from(["live".to_owned()])).unwrap(); - assert_eq!((census.head_count, census.superseded_count), (1, 1)); + census_sealed_store_entries( + ordered(), + &BTreeSet::from([( + "code".to_owned(), + "index".to_owned(), + "live".to_owned(), + )]), + ) + .unwrap(); + assert_eq!((census.head_count, census.superseded_count), (1, 2)); assert_eq!( (census.abandoned_staging_count, census.unrecognized_count), (1, 3) ); assert_eq!(census.head_bytes, 8192 + head_receipt.len() as u64); - assert_eq!(census.superseded_bytes, old_receipt.len() as u64); + assert_eq!( + census.superseded_bytes, + (old_receipt.len() + foreign_receipt.len()) as u64 + ); assert_eq!(census.abandoned_staging_bytes, 0); let no_head = census_sealed_store_entries(ordered(), &BTreeSet::new()).unwrap(); assert_eq!(no_head.head_bytes, 0); assert_eq!( no_head.superseded_bytes, - 8192 + head_receipt.len() as u64 + old_receipt.len() as u64 + 8192 + head_receipt.len() as u64 + old_receipt.len() as u64 + foreign_receipt.len() as u64 ); } } diff --git a/crates/tracedecay-graph-db/tests/graph_db_suite/verified_generation_contract/sealed_store.rs b/crates/tracedecay-graph-db/tests/graph_db_suite/verified_generation_contract/sealed_store.rs index db1f4f09ca2..2d91adfcc9e 100644 --- a/crates/tracedecay-graph-db/tests/graph_db_suite/verified_generation_contract/sealed_store.rs +++ b/crates/tracedecay-graph-db/tests/graph_db_suite/verified_generation_contract/sealed_store.rs @@ -1354,7 +1354,11 @@ fn sealed_census_separates_heads_from_superseded_and_abandoned_bytes() { std::fs::create_dir_all(&unreadable).unwrap(); std::fs::write(unreadable.join("generation.grafeo"), b"no receipt").unwrap(); - let heads = std::collections::BTreeSet::from(["census-g2".to_owned()]); + let heads = std::collections::BTreeSet::from([( + identity.namespace.as_str().to_owned(), + identity.projection.as_str().to_owned(), + "census-g2".to_owned(), + )]); let census = tracedecay_graph_db::census_sealed_store(&support::graph_path(temp.path()), &heads) .unwrap(); From c28d2c5e1149ed1183b0fe3bf7dfe8ac2492fc1f Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Fri, 6 Nov 2026 09:23:48 -0800 Subject: [PATCH 32/71] style(graph): format projection-bound census regression --- .../tracedecay-graph-db/src/sealed_store.rs | 28 +++++++++++-------- 1 file changed, 16 insertions(+), 12 deletions(-) diff --git a/crates/tracedecay-graph-db/src/sealed_store.rs b/crates/tracedecay-graph-db/src/sealed_store.rs index 3292111c13f..37818ef6fdb 100644 --- a/crates/tracedecay-graph-db/src/sealed_store.rs +++ b/crates/tracedecay-graph-db/src/sealed_store.rs @@ -810,7 +810,10 @@ mod sealed_census_tests { 1 } else if name == old { 2 - } else if name == wrong_namespace || name == future_version || name == foreign_projection { + } else if name == wrong_namespace + || name == future_version + || name == foreign_projection + { 3 } else if name == head { 4 @@ -820,16 +823,15 @@ mod sealed_census_tests { }); entries.into_iter().map(Ok).collect::>() }; - let census = - census_sealed_store_entries( - ordered(), - &BTreeSet::from([( - "code".to_owned(), - "index".to_owned(), - "live".to_owned(), - )]), - ) - .unwrap(); + let census = census_sealed_store_entries( + ordered(), + &BTreeSet::from([( + "code".to_owned(), + "index".to_owned(), + "live".to_owned(), + )]), + ) + .unwrap(); assert_eq!((census.head_count, census.superseded_count), (1, 2)); assert_eq!( (census.abandoned_staging_count, census.unrecognized_count), @@ -846,7 +848,9 @@ mod sealed_census_tests { assert_eq!(no_head.head_bytes, 0); assert_eq!( no_head.superseded_bytes, - 8192 + head_receipt.len() as u64 + old_receipt.len() as u64 + foreign_receipt.len() as u64 + 8192 + head_receipt.len() as u64 + + old_receipt.len() as u64 + + foreign_receipt.len() as u64 ); } } From 4c5406c53ae48950de4ec27dbb985924f8d3be95 Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Fri, 6 Nov 2026 09:25:34 -0800 Subject: [PATCH 33/71] style(graph): align sealed cleanup fixtures with rustfmt 2024 Apply the exact Bazel repository-gates diffs without changing cleanup semantics or assertions. --- crates/tracedecay-graph-db/src/sealed_store.rs | 5 ++++- .../tests/graph_db_suite/runtime_contract.rs | 5 ++++- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/crates/tracedecay-graph-db/src/sealed_store.rs b/crates/tracedecay-graph-db/src/sealed_store.rs index 37818ef6fdb..179c94f0adf 100644 --- a/crates/tracedecay-graph-db/src/sealed_store.rs +++ b/crates/tracedecay-graph-db/src/sealed_store.rs @@ -960,7 +960,10 @@ mod superseded_sealed_cleanup_tests { assert!(remove_sealed_directory_checked(&root.join(&obsolete)).is_err()); for name in [obsolete.as_str(), ".staging-abandoned", ".rows-0-foreign"] { - assert!(foreign.join(name).is_dir(), "foreign artifact {name} was removed"); + assert!( + foreign.join(name).is_dir(), + "foreign artifact {name} was removed" + ); } } diff --git a/crates/tracedecay-graph-db/tests/graph_db_suite/runtime_contract.rs b/crates/tracedecay-graph-db/tests/graph_db_suite/runtime_contract.rs index e57bebb16e7..c69dbf3222a 100644 --- a/crates/tracedecay-graph-db/tests/graph_db_suite/runtime_contract.rs +++ b/crates/tracedecay-graph-db/tests/graph_db_suite/runtime_contract.rs @@ -1064,7 +1064,10 @@ fn superseded_format_store_rebuild_removes_proven_old_seals_but_keeps_unproven() .unwrap(); raw.close().unwrap(); let sealed_generation_name = "0".repeat(64); - let sealed_generation = temp.path().join("graph.sealed").join(&sealed_generation_name); + let sealed_generation = temp + .path() + .join("graph.sealed") + .join(&sealed_generation_name); std::fs::create_dir_all(&sealed_generation).unwrap(); std::fs::write( sealed_generation.join("generation.grafeo"), From 46b72850010d14bec9381358b2d0f849830ac522 Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Fri, 6 Nov 2026 10:12:04 -0800 Subject: [PATCH 34/71] fix(storage): preserve unknown sealed forms on format cleanup A parseable older-format receipt is not sufficient deletion authority when its sealed representation is unknown or inconsistent with its base. Retain those artifacts while still removing proven obsolete compact generations. Refs #4616 --- .../tracedecay-graph-db/src/sealed_store.rs | 25 +++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/crates/tracedecay-graph-db/src/sealed_store.rs b/crates/tracedecay-graph-db/src/sealed_store.rs index 179c94f0adf..306b8d60425 100644 --- a/crates/tracedecay-graph-db/src/sealed_store.rs +++ b/crates/tracedecay-graph-db/src/sealed_store.rs @@ -889,9 +889,16 @@ pub(crate) fn discard_superseded_sealed_generations( let Some(receipt) = load_sealed_store_receipt(&path).ok().flatten() else { continue; }; + // The old graph format alone is not proof that this build owns the + // artifact: a newer or malformed sealed form must survive a downgrade. + let known_form = matches!( + (&receipt.base, receipt.form.as_str()), + (None, SEALED_STORE_FORM_COMPACT) | (Some(_), SEALED_STORE_FORM_LAYERED) + ); if receipt.version != SEALED_STORE_RECEIPT_VERSION || receipt.physical_namespace != format!("generation:{name}") || receipt.graph_format >= GraphFormatVersion::current().get() + || !known_form { continue; } @@ -981,6 +988,8 @@ mod superseded_sealed_cleanup_tests { let unreadable = "d".repeat(64); let ordinary_file = "e".repeat(64); let current_name = "f".repeat(64); + let unknown_form = "0".repeat(64); + let inconsistent_form = "1".repeat(64); let unrelated = "operator-data"; let receipt = |name: &str, format: u32, physical_namespace: String| { @@ -1016,6 +1025,20 @@ mod superseded_sealed_cleanup_tests { "generation:foreign".to_owned(), ); receipt(¤t_name, current, format!("generation:{current_name}")); + // A parseable receipt with an older graph-format number can belong to + // a newer sealed representation, or be inconsistent with its base. + // Neither is a proven artifact that this build is allowed to remove. + for (name, form) in [ + (&unknown_form, "future-container"), + (&inconsistent_form, "layered"), + ] { + receipt(name, current - 1, format!("generation:{name}")); + let receipt_path = root.join(name).join(SEALED_STORE_RECEIPT_FILE); + let mut value: serde_json::Value = + serde_json::from_slice(&std::fs::read(&receipt_path).unwrap()).unwrap(); + value["form"] = form.into(); + std::fs::write(&receipt_path, serde_json::to_vec(&value).unwrap()).unwrap(); + } receipt(unrelated, current - 1, format!("generation:{unrelated}")); std::fs::create_dir(root.join(&unreadable)).unwrap(); std::fs::write( @@ -1038,6 +1061,8 @@ mod superseded_sealed_cleanup_tests { unreadable.as_str(), ordinary_file.as_str(), current_name.as_str(), + unknown_form.as_str(), + inconsistent_form.as_str(), unrelated, ".staging-active", ] { From 5f60b304775ae53ba6f3149fad49b127ee710c32 Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Fri, 6 Nov 2026 10:20:44 -0800 Subject: [PATCH 35/71] style(graph): format sealed census regression Apply exactly the two rustfmt 2024 transformations reported by the current integration Repository gates run. Preserve test semantics. --- crates/tracedecay-graph-db/src/sealed_store.rs | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/crates/tracedecay-graph-db/src/sealed_store.rs b/crates/tracedecay-graph-db/src/sealed_store.rs index 306b8d60425..77066e0a617 100644 --- a/crates/tracedecay-graph-db/src/sealed_store.rs +++ b/crates/tracedecay-graph-db/src/sealed_store.rs @@ -784,7 +784,8 @@ mod sealed_census_tests { foreign_receipt["projection"] = serde_json::json!("other-projection"); let foreign_receipt = serde_json::to_vec(&foreign_receipt).unwrap(); fs::write( - root.join(&foreign_projection).join(SEALED_STORE_RECEIPT_FILE), + root.join(&foreign_projection) + .join(SEALED_STORE_RECEIPT_FILE), &foreign_receipt, ) .unwrap(); @@ -825,11 +826,7 @@ mod sealed_census_tests { }; let census = census_sealed_store_entries( ordered(), - &BTreeSet::from([( - "code".to_owned(), - "index".to_owned(), - "live".to_owned(), - )]), + &BTreeSet::from([("code".to_owned(), "index".to_owned(), "live".to_owned())]), ) .unwrap(); assert_eq!((census.head_count, census.superseded_count), (1, 2)); From 968c505c19be09e9a414f2715a278fea5362b5f9 Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Fri, 6 Nov 2026 11:18:30 -0800 Subject: [PATCH 36/71] fix(graph): refuse unknown sealed forms in live census Only recognized compact and layered receipt/base pairings may claim live-generation counts and shared physical bytes. Reuse the cleanup policy and exercise future and inconsistent receipts with hard links. Refs #4616 --- .../tracedecay-graph-db/src/sealed_store.rs | 52 ++++++++++++++++--- 1 file changed, 45 insertions(+), 7 deletions(-) diff --git a/crates/tracedecay-graph-db/src/sealed_store.rs b/crates/tracedecay-graph-db/src/sealed_store.rs index 77066e0a617..dde57188220 100644 --- a/crates/tracedecay-graph-db/src/sealed_store.rs +++ b/crates/tracedecay-graph-db/src/sealed_store.rs @@ -271,6 +271,15 @@ impl SealedStoreReceiptV1 { } } +// A future sealed representation must not acquire live-generation ownership +// merely by putting a parseable receipt at a familiar generation path. +fn recognized_sealed_form(receipt: &SealedStoreReceiptV1) -> bool { + matches!( + (&receipt.base, receipt.form.as_str()), + (None, SEALED_STORE_FORM_COMPACT) | (Some(_), SEALED_STORE_FORM_LAYERED) + ) +} + /// Digest and row counts that authorize a staging-row release without opening /// the sealed engine. pub(crate) struct SealedReleaseEvidence { @@ -608,6 +617,7 @@ fn census_sealed_store_entries( || !name.bytes().all(|byte| byte.is_ascii_hexdigit()) || receipt.version != SEALED_STORE_RECEIPT_VERSION || receipt.physical_namespace != format!("generation:{name}") + || !recognized_sealed_form(&receipt) { census.unrecognized_count += 1; continue; @@ -698,6 +708,8 @@ mod sealed_census_tests { let wrong_namespace = "c".repeat(64); let future_version = "d".repeat(64); let foreign_projection = "e".repeat(64); + let unknown_form = "0".repeat(64); + let missing_layer_base = "1".repeat(64); for name in [ "unrecognized", ".staging-pending", @@ -706,6 +718,8 @@ mod sealed_census_tests { wrong_namespace.as_str(), future_version.as_str(), foreign_projection.as_str(), + unknown_form.as_str(), + missing_layer_base.as_str(), ] { fs::create_dir(root.join(name)).unwrap(); } @@ -789,9 +803,35 @@ mod sealed_census_tests { &foreign_receipt, ) .unwrap(); + // Parseable, identity-bound receipts with unknown or inconsistent forms + // must not borrow a verified head's count or claim its physical bytes. + for (directory, form) in [ + (unknown_form.as_str(), "future-container"), + (missing_layer_base.as_str(), "layered"), + ] { + let mut value: serde_json::Value = serde_json::from_slice(&receipt( + "live", + &format!("generation:{directory}"), + SEALED_STORE_RECEIPT_VERSION, + )) + .unwrap(); + value["form"] = serde_json::json!(form); + fs::write( + root.join(directory).join(SEALED_STORE_RECEIPT_FILE), + serde_json::to_vec(&value).unwrap(), + ) + .unwrap(); + } + fs::write(root.join(&unknown_form).join("foreign"), vec![0_u8; 4096]).unwrap(); let shared = root.join(&head).join("shared"); fs::write(&shared, vec![0_u8; 8192]).unwrap(); - for directory in [old.as_str(), ".staging-pending", "unrecognized"] { + for directory in [ + old.as_str(), + ".staging-pending", + "unrecognized", + unknown_form.as_str(), + missing_layer_base.as_str(), + ] { fs::hard_link(&shared, root.join(directory).join("shared")).unwrap(); } @@ -814,6 +854,8 @@ mod sealed_census_tests { } else if name == wrong_namespace || name == future_version || name == foreign_projection + || name == unknown_form + || name == missing_layer_base { 3 } else if name == head { @@ -832,7 +874,7 @@ mod sealed_census_tests { assert_eq!((census.head_count, census.superseded_count), (1, 2)); assert_eq!( (census.abandoned_staging_count, census.unrecognized_count), - (1, 3) + (1, 5) ); assert_eq!(census.head_bytes, 8192 + head_receipt.len() as u64); assert_eq!( @@ -888,14 +930,10 @@ pub(crate) fn discard_superseded_sealed_generations( }; // The old graph format alone is not proof that this build owns the // artifact: a newer or malformed sealed form must survive a downgrade. - let known_form = matches!( - (&receipt.base, receipt.form.as_str()), - (None, SEALED_STORE_FORM_COMPACT) | (Some(_), SEALED_STORE_FORM_LAYERED) - ); if receipt.version != SEALED_STORE_RECEIPT_VERSION || receipt.physical_namespace != format!("generation:{name}") || receipt.graph_format >= GraphFormatVersion::current().get() - || !known_form + || !recognized_sealed_form(&receipt) { continue; } From 65e993ce6814291adb4da983061cf981b6061d5d Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Fri, 6 Nov 2026 11:28:23 -0800 Subject: [PATCH 37/71] fix(graph): exclude unsupported graph formats from sealed census A parseable receipt with graph format zero or newer than this build may not claim a live head's shared bytes or authorize format cleanup. Keep known historical graph formats reportable and add both refusal cases to the ordered hard-link census regression. Refs #4616 --- .../tracedecay-graph-db/src/sealed_store.rs | 43 ++++++++++++++++--- 1 file changed, 38 insertions(+), 5 deletions(-) diff --git a/crates/tracedecay-graph-db/src/sealed_store.rs b/crates/tracedecay-graph-db/src/sealed_store.rs index dde57188220..3443054d7fc 100644 --- a/crates/tracedecay-graph-db/src/sealed_store.rs +++ b/crates/tracedecay-graph-db/src/sealed_store.rs @@ -274,10 +274,13 @@ impl SealedStoreReceiptV1 { // A future sealed representation must not acquire live-generation ownership // merely by putting a parseable receipt at a familiar generation path. fn recognized_sealed_form(receipt: &SealedStoreReceiptV1) -> bool { - matches!( - (&receipt.base, receipt.form.as_str()), - (None, SEALED_STORE_FORM_COMPACT) | (Some(_), SEALED_STORE_FORM_LAYERED) - ) + // Unknown graph formats cannot authorize ownership or deletion, even if + // the receipt otherwise resembles a current compact or layered store. + (1..=GraphFormatVersion::current().get()).contains(&receipt.graph_format) + && matches!( + (&receipt.base, receipt.form.as_str()), + (None, SEALED_STORE_FORM_COMPACT) | (Some(_), SEALED_STORE_FORM_LAYERED) + ) } /// Digest and row counts that authorize a staging-row release without opening @@ -710,6 +713,8 @@ mod sealed_census_tests { let foreign_projection = "e".repeat(64); let unknown_form = "0".repeat(64); let missing_layer_base = "1".repeat(64); + let future_graph_format = "2".repeat(64); + let zero_graph_format = "3".repeat(64); for name in [ "unrecognized", ".staging-pending", @@ -720,6 +725,8 @@ mod sealed_census_tests { foreign_projection.as_str(), unknown_form.as_str(), missing_layer_base.as_str(), + future_graph_format.as_str(), + zero_graph_format.as_str(), ] { fs::create_dir(root.join(name)).unwrap(); } @@ -822,6 +829,28 @@ mod sealed_census_tests { ) .unwrap(); } + // A current receipt version with an unsupported graph format is not + // proof of a readable store, even when its generation identity matches. + for (directory, graph_format) in [ + ( + future_graph_format.as_str(), + super::GraphFormatVersion::current().get() + 1, + ), + (zero_graph_format.as_str(), 0), + ] { + let mut value: serde_json::Value = serde_json::from_slice(&receipt( + "live", + &format!("generation:{directory}"), + SEALED_STORE_RECEIPT_VERSION, + )) + .unwrap(); + value["graph_format"] = serde_json::json!(graph_format); + fs::write( + root.join(directory).join(SEALED_STORE_RECEIPT_FILE), + serde_json::to_vec(&value).unwrap(), + ) + .unwrap(); + } fs::write(root.join(&unknown_form).join("foreign"), vec![0_u8; 4096]).unwrap(); let shared = root.join(&head).join("shared"); fs::write(&shared, vec![0_u8; 8192]).unwrap(); @@ -831,6 +860,8 @@ mod sealed_census_tests { "unrecognized", unknown_form.as_str(), missing_layer_base.as_str(), + future_graph_format.as_str(), + zero_graph_format.as_str(), ] { fs::hard_link(&shared, root.join(directory).join("shared")).unwrap(); } @@ -856,6 +887,8 @@ mod sealed_census_tests { || name == foreign_projection || name == unknown_form || name == missing_layer_base + || name == future_graph_format + || name == zero_graph_format { 3 } else if name == head { @@ -874,7 +907,7 @@ mod sealed_census_tests { assert_eq!((census.head_count, census.superseded_count), (1, 2)); assert_eq!( (census.abandoned_staging_count, census.unrecognized_count), - (1, 5) + (1, 7) ); assert_eq!(census.head_bytes, 8192 + head_receipt.len() as u64); assert_eq!( From 6cd9a91ab4e9b6247740a634a777ef7d206becf1 Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Fri, 6 Nov 2026 12:16:16 -0800 Subject: [PATCH 38/71] fix(retention): isolate unsafe unreferenced artifact entries Keep the durable referenced descriptor and active staging checks fail-closed. Skip and report unreferenced symlinks and directories instead of aborting other verified generation and text-artifact reclamation. Exercise real retirement of four generations and an orphan alongside foreign paths; retain an explicit referenced-symlink refusal regression. Refs #4616 --- .../src/code_index_generations/tests.rs | 99 +++++++++++++++++-- .../code_index_generations/text_artifacts.rs | 14 ++- 2 files changed, 100 insertions(+), 13 deletions(-) diff --git a/crates/tracedecay-code-index-retention/src/code_index_generations/tests.rs b/crates/tracedecay-code-index-retention/src/code_index_generations/tests.rs index 7fc9f340678..9b235bd2af7 100644 --- a/crates/tracedecay-code-index-retention/src/code_index_generations/tests.rs +++ b/crates/tracedecay-code-index-retention/src/code_index_generations/tests.rs @@ -1227,22 +1227,103 @@ fn text_artifact_retention_refuses_tamper_and_publish_cas_movement() { #[cfg(unix)] #[test] -fn text_artifact_retention_refuses_symlinked_inventory_entries() { - let (store, _generations) = fixture_store(1); +fn unreferenced_nonregular_text_artifacts_do_not_block_verified_retirement() { + let (store, generations) = fixture_store(5); + let active = generations.last().expect("active generation"); + let referenced = attach_fixture_text_artifact(&store, active, b"durably referenced"); + let referenced_path = + code_text_artifact_path(store.path(), &referenced).expect("referenced artifact path"); + let valid = text_artifact_for_bytes(&active.id, b"verified orphan beside unsafe entries"); + let valid_path = write_text_artifact(&store, &valid, b"verified orphan beside unsafe entries"); + let artifacts_root = code_text_artifacts_root(store.path()); - std::fs::create_dir_all(&artifacts_root).expect("create artifact root"); - let outside = tempfile::NamedTempFile::new().expect("outside artifact"); - let symlink = artifacts_root.join(format!("text-artifact-{}.bin", "d".repeat(64))); - std::os::unix::fs::symlink(outside.path(), &symlink).expect("create symlink"); + let outside = tempfile::NamedTempFile::new().expect("foreign artifact"); + std::fs::write(outside.path(), b"unrelated operator data").expect("write foreign data"); + let link_name = format!("text-artifact-{}.bin", "d".repeat(64)); + let link = artifacts_root.join(&link_name); + std::os::unix::fs::symlink(outside.path(), &link).expect("create foreign link"); + let directory_name = format!("text-artifact-{}.bin", "e".repeat(64)); + let directory = artifacts_root.join(&directory_name); + std::fs::create_dir(&directory).expect("create foreign directory"); + std::fs::write(directory.join("keep"), b"unrelated directory data").unwrap(); + + let plan = prepare_next_code_generation_retention_cancellable( + store.path(), + &BTreeSet::new(), + &|| false, + None, + ) + .expect("unsafe unreferenced entries cannot veto valid retirement"); + assert_eq!(plan.collectable_generations.len(), 4); + assert!( + plan.collectable_text_artifacts + .iter() + .any(|entry| entry.artifact_file == valid.artifact_file) + ); + for name in [&link_name, &directory_name] { + assert!( + plan.collectable_text_artifacts + .iter() + .all(|entry| &entry.artifact_file != name), + "an untrusted entry must never become a deletion candidate" + ); + } + + let report = execute_code_generation_retention( + store.path(), + plan, + CodeGenerationRetentionModeV1::Apply, + UtcMicros(52), + None, + ) + .expect("collect independent verified generations and orphan"); + assert_eq!(report.deleted_generations.len(), 4); + assert!(!valid_path.exists(), "verified orphan was reclaimed"); + assert!(referenced_path.is_file(), "durable referenced target remains"); + assert!( + std::fs::symlink_metadata(&link) + .unwrap() + .file_type() + .is_symlink(), + "untrusted symlink remains untouched" + ); + assert_eq!( + std::fs::read(outside.path()).unwrap(), + b"unrelated operator data" + ); + assert_eq!( + std::fs::read(directory.join("keep")).unwrap(), + b"unrelated directory data" + ); +} + +#[cfg(unix)] +#[test] +fn referenced_text_artifact_symlink_still_refuses_retention() { + let (store, generations) = fixture_store(1); + let active = generations.last().expect("active generation"); + let referenced = attach_fixture_text_artifact(&store, active, b"durably referenced"); + let target = code_text_artifact_path(store.path(), &referenced).unwrap(); + let outside = tempfile::NamedTempFile::new().expect("foreign artifact"); + std::fs::write(outside.path(), b"unrelated operator data").unwrap(); + std::fs::remove_file(&target).unwrap(); + std::os::unix::fs::symlink(outside.path(), &target).unwrap(); + let error = plan_code_generation_retention(store.path(), &BTreeSet::new()) - .expect_err("symlinked artifact entry must fail closed"); + .expect_err("a referenced descriptor cannot be replaced by a symlink"); assert!(matches!( error, CodeGenerationRetentionErrorV1::UnsafeState(_) )); assert!( - symlink.exists(), - "refusal must preserve foreign link evidence" + std::fs::symlink_metadata(&target) + .unwrap() + .file_type() + .is_symlink() + ); + assert_eq!( + std::fs::read(outside.path()).unwrap(), + b"unrelated operator data" ); } diff --git a/crates/tracedecay-code-index-retention/src/code_index_generations/text_artifacts.rs b/crates/tracedecay-code-index-retention/src/code_index_generations/text_artifacts.rs index 6c4e749d67e..bf322f61afa 100644 --- a/crates/tracedecay-code-index-retention/src/code_index_generations/text_artifacts.rs +++ b/crates/tracedecay-code-index-retention/src/code_index_generations/text_artifacts.rs @@ -451,10 +451,16 @@ pub(super) fn plan_collectable_text_artifacts_budgeted_page_cancellable( Err(error) => return Err(storage(error)), }; if !metadata.file_type().is_file() { - return Err(CodeGenerationRetentionErrorV1::UnsafeState(format!( - "code text artifact inventory path '{}' is not a regular file", - path.display() - ))); + // Referenced descriptors were verified above and remain fail-closed. + // Unreferenced symlinks/directories have no deletion authority, + // but must not veto a separate verified artifact or generation. + tracing::warn!( + event = "code_text_artifact_retention_candidate_preserved", + artifact = %path.display(), + reason = "non_regular_inventory_entry", + "unreferenced non-regular artifact preserved during retention" + ); + continue; } let staging_source = staging_text_artifact_source_digest(&file_name) .or_else(|| staging_sidecar_text_artifact_source_digest(&file_name)); From 9e3068363916d309f91417d512b21757f68fa418 Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Fri, 6 Nov 2026 16:14:30 -0800 Subject: [PATCH 39/71] style(retention): format unsafe inventory regression --- .../src/code_index_generations/tests.rs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/crates/tracedecay-code-index-retention/src/code_index_generations/tests.rs b/crates/tracedecay-code-index-retention/src/code_index_generations/tests.rs index 9b235bd2af7..2c06f5572ee 100644 --- a/crates/tracedecay-code-index-retention/src/code_index_generations/tests.rs +++ b/crates/tracedecay-code-index-retention/src/code_index_generations/tests.rs @@ -1279,7 +1279,10 @@ fn unreferenced_nonregular_text_artifacts_do_not_block_verified_retirement() { .expect("collect independent verified generations and orphan"); assert_eq!(report.deleted_generations.len(), 4); assert!(!valid_path.exists(), "verified orphan was reclaimed"); - assert!(referenced_path.is_file(), "durable referenced target remains"); + assert!( + referenced_path.is_file(), + "durable referenced target remains" + ); assert!( std::fs::symlink_metadata(&link) .unwrap() From 81548e7bf13e7dbcb29a66eea28cd7846920790f Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Fri, 6 Nov 2026 16:14:31 -0800 Subject: [PATCH 40/71] fix(graph): bound sealed receipt reads before cleanup Refs #4616 --- .../tracedecay-graph-db/src/sealed_store.rs | 75 ++++++++++++++++--- 1 file changed, 66 insertions(+), 9 deletions(-) diff --git a/crates/tracedecay-graph-db/src/sealed_store.rs b/crates/tracedecay-graph-db/src/sealed_store.rs index 3443054d7fc..7f02cb2de67 100644 --- a/crates/tracedecay-graph-db/src/sealed_store.rs +++ b/crates/tracedecay-graph-db/src/sealed_store.rs @@ -44,6 +44,7 @@ //! digest before the store serves a read. use std::collections::{BTreeMap, HashMap}; +use std::io::Read; use std::panic::{AssertUnwindSafe, catch_unwind}; use std::path::{Path, PathBuf}; use std::sync::Arc; @@ -112,10 +113,10 @@ pub(crate) fn open_direct_sealed_generation( sealed_generation_directory(&sealed_store_root(database_path), &physical_namespace); let receipt_path = directory.join(SEALED_STORE_RECEIPT_FILE); let sealed_path = directory.join(SEALED_STORE_DATABASE_FILE); - let receipt_bytes = match std::fs::read(&receipt_path) { - Ok(bytes) => bytes, - Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(None), - Err(error) => return Err(sealed_store_io_failure("receipt read failed", error)), + let Some(receipt_bytes) = read_sealed_receipt_bytes(&receipt_path) + .map_err(|error| sealed_store_io_failure("receipt read failed", error))? + else { + return Ok(None); }; let receipt: SealedStoreReceiptV1 = serde_json::from_slice(&receipt_bytes) .map_err(|error| GraphDbError::unavailable(format!("sealed receipt decode: {error}")))?; @@ -207,6 +208,8 @@ pub(crate) fn open_direct_sealed_generation( const SEALED_COPY_GUARD_CHUNK_ROWS: usize = 4096; const SEALED_STORE_RECEIPT_VERSION: u32 = 1; +/// Sealed receipts carry only bounded generation identity and row metadata. +const MAX_SEALED_STORE_RECEIPT_BYTES: u64 = 64 * 1024; pub(crate) const SEALED_STORE_DATABASE_FILE: &str = "generation.grafeo"; const SEALED_STORE_RECEIPT_FILE: &str = "sealed.json"; /// Digest recorded only after a successful post-reopen proof. A `sealed.json` @@ -604,8 +607,9 @@ fn census_sealed_store_entries( abandoned.push(path); continue; } - let receipt = std::fs::read(path.join(SEALED_STORE_RECEIPT_FILE)) + let receipt = read_sealed_receipt_bytes(&path.join(SEALED_STORE_RECEIPT_FILE)) .ok() + .flatten() .and_then(|bytes| serde_json::from_slice::(&bytes).ok()); let Some(receipt) = receipt else { census.unrecognized_count += 1; @@ -1107,6 +1111,17 @@ mod superseded_sealed_cleanup_tests { value["form"] = form.into(); std::fs::write(&receipt_path, serde_json::to_vec(&value).unwrap()).unwrap(); } + // Whitespace-padded JSON is still a parseable old-format receipt. + // It must not authorize deletion or a full allocation in the census. + let oversized = "4".repeat(64); + receipt(&oversized, current - 1, format!("generation:{oversized}")); + let oversized_directory = root.join(&oversized); + let oversized_receipt = oversized_directory.join(SEALED_STORE_RECEIPT_FILE); + let mut padded = std::fs::read(&oversized_receipt).unwrap(); + padded.resize(MAX_SEALED_STORE_RECEIPT_BYTES as usize + 1, b' '); + assert!(serde_json::from_slice::(&padded).is_ok()); + std::fs::write(&oversized_receipt, &padded).unwrap(); + assert!(load_sealed_store_receipt(&oversized_directory).is_err()); receipt(unrelated, current - 1, format!("generation:{unrelated}")); std::fs::create_dir(root.join(&unreadable)).unwrap(); std::fs::write( @@ -1131,6 +1146,7 @@ mod superseded_sealed_cleanup_tests { current_name.as_str(), unknown_form.as_str(), inconsistent_form.as_str(), + oversized.as_str(), unrelated, ".staging-active", ] { @@ -2924,14 +2940,55 @@ fn open_sealed_store( open_sealed_store_checked(directory, identity, expected, &|| Ok(()), None) } +/// Read only a regular sealed receipt, never an arbitrary-length on-disk +/// payload. Check both the path and opened descriptor, since identity can +/// change between metadata sampling and opening the file. The capped read +/// protects against growth after both checks. +fn read_sealed_receipt_bytes(path: &Path) -> std::io::Result>> { + let metadata = match std::fs::symlink_metadata(path) { + Ok(metadata) => metadata, + Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(None), + Err(error) => return Err(error), + }; + if !metadata.file_type().is_file() { + return Err(std::io::Error::new( + std::io::ErrorKind::InvalidData, + "sealed receipt must be a regular file", + )); + } + if metadata.len() > MAX_SEALED_STORE_RECEIPT_BYTES { + return Err(std::io::Error::new( + std::io::ErrorKind::InvalidData, + "sealed receipt exceeds its byte limit", + )); + } + let file = std::fs::File::open(path)?; + if !file.metadata()?.is_file() { + return Err(std::io::Error::new( + std::io::ErrorKind::InvalidData, + "sealed receipt descriptor is not a regular file", + )); + } + let mut bytes = Vec::new(); + file.take(MAX_SEALED_STORE_RECEIPT_BYTES + 1) + .read_to_end(&mut bytes)?; + if bytes.len() as u64 > MAX_SEALED_STORE_RECEIPT_BYTES { + return Err(std::io::Error::new( + std::io::ErrorKind::InvalidData, + "sealed receipt grew past its byte limit", + )); + } + Ok(Some(bytes)) +} + fn load_sealed_store_receipt( directory: &Path, ) -> Result, GraphDbError> { let receipt_path = directory.join(SEALED_STORE_RECEIPT_FILE); - let receipt_bytes = match std::fs::read(&receipt_path) { - Ok(bytes) => bytes, - Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(None), - Err(error) => return Err(sealed_store_io_failure("receipt read failed", error)), + let Some(receipt_bytes) = read_sealed_receipt_bytes(&receipt_path) + .map_err(|error| sealed_store_io_failure("receipt read failed", error))? + else { + return Ok(None); }; serde_json::from_slice(&receipt_bytes) .map(Some) From dd3f9c411a00a2e054b37b99662195069fe838d3 Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Fri, 6 Nov 2026 16:14:33 -0800 Subject: [PATCH 41/71] fix(graph): bound and atomically replace sealed proof markers Refs #4616. Refuse symlinked and oversized verification markers, read only digest-sized bytes, and publish a new proof marker through an atomic same-directory rename rather than following a stale link outside the store. Add real filesystem regressions for foreign target preservation, bounded marker matching and fail-closed linked generation directories. --- .../tracedecay-graph-db/src/sealed_store.rs | 117 ++++++++++++++++-- 1 file changed, 110 insertions(+), 7 deletions(-) diff --git a/crates/tracedecay-graph-db/src/sealed_store.rs b/crates/tracedecay-graph-db/src/sealed_store.rs index 7f02cb2de67..32f9738f312 100644 --- a/crates/tracedecay-graph-db/src/sealed_store.rs +++ b/crates/tracedecay-graph-db/src/sealed_store.rs @@ -44,7 +44,7 @@ //! digest before the store serves a read. use std::collections::{BTreeMap, HashMap}; -use std::io::Read; +use std::io::{Read, Write}; use std::panic::{AssertUnwindSafe, catch_unwind}; use std::path::{Path, PathBuf}; use std::sync::Arc; @@ -2995,24 +2995,127 @@ fn load_sealed_store_receipt( .map_err(|error| GraphDbError::unavailable(format!("sealed receipt decode: {error}"))) } +/// A proof marker is only a digest, not a trusted size-unbounded payload. +/// Never follow a symlink as marker authority; cap the read even when the +/// file changes between the path and opened-descriptor checks. fn sealed_store_check_matches( directory: &Path, expected_digest: &str, ) -> Result { let path = directory.join(SEALED_STORE_CHECKED_FILE); - match std::fs::read_to_string(&path) { - Ok(digest) => Ok(digest == expected_digest), - Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(false), - Err(error) => Err(sealed_store_io_failure("sealed check read failed", error)), + let metadata = match std::fs::symlink_metadata(&path) { + Ok(metadata) => metadata, + Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(false), + Err(error) => return Err(sealed_store_io_failure("sealed check inspect failed", error)), + }; + let expected_len = expected_digest.len() as u64; + if expected_len > MAX_SEALED_STORE_RECEIPT_BYTES + || !metadata.file_type().is_file() + || metadata.len() != expected_len + { + return Ok(false); + } + let file = match std::fs::File::open(&path) { + Ok(file) => file, + Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(false), + Err(error) => return Err(sealed_store_io_failure("sealed check open failed", error)), + }; + let opened = file + .metadata() + .map_err(|error| sealed_store_io_failure("sealed check inspect failed", error))?; + if !opened.is_file() || opened.len() != expected_len { + return Ok(false); } + let mut bytes = Vec::new(); + file.take(expected_len + 1) + .read_to_end(&mut bytes) + .map_err(|error| sealed_store_io_failure("sealed check read failed", error))?; + Ok(bytes == expected_digest.as_bytes()) } fn persist_sealed_store_check(directory: &Path, expected_digest: &str) -> Result<(), GraphDbError> { if sealed_store_check_matches(directory, expected_digest)? { return Ok(()); } - std::fs::write(directory.join(SEALED_STORE_CHECKED_FILE), expected_digest) - .map_err(|error| sealed_store_io_failure("sealed check persist failed", error)) + // The marker is a re-verification optimization, never authority to write + // through a caller-controlled symlink to data outside the sealed store. + let root = directory + .parent() + .ok_or_else(|| GraphDbError::unavailable("sealed marker has no parent root"))?; + let root_is_real = sealed_root_is_real_directory(root) + .map_err(|error| sealed_store_io_failure("sealed marker root inspect failed", error))?; + let directory_is_real = std::fs::symlink_metadata(directory) + .map(|metadata| metadata.file_type().is_dir()) + .map_err(|error| sealed_store_io_failure("sealed marker directory inspect failed", error))?; + if !root_is_real || !directory_is_real { + return Err(GraphDbError::unavailable( + "sealed marker directory is not a real directory", + )); + } + let mut staged = tempfile::NamedTempFile::new_in(directory) + .map_err(|error| sealed_store_io_failure("sealed check stage failed", error))?; + staged + .write_all(expected_digest.as_bytes()) + .map_err(|error| sealed_store_io_failure("sealed check write failed", error))?; + staged + .as_file() + .sync_all() + .map_err(|error| sealed_store_io_failure("sealed check sync failed", error))?; + staged + .persist(directory.join(SEALED_STORE_CHECKED_FILE)) + .map_err(|error| sealed_store_io_failure("sealed check persist failed", error.error))?; + Ok(()) +} + +#[cfg(test)] +mod sealed_marker_tests { + use super::{SEALED_STORE_CHECKED_FILE, persist_sealed_store_check, sealed_store_check_matches}; + + #[test] + fn proof_marker_requires_exact_bounded_digest_and_replaces_stale_bytes() { + let root = tempfile::tempdir().unwrap(); + let directory = root.path().join("generation"); + std::fs::create_dir(&directory).unwrap(); + let marker = directory.join(SEALED_STORE_CHECKED_FILE); + let digest = "a".repeat(64); + + assert!(!sealed_store_check_matches(&directory, &digest).unwrap()); + std::fs::write(&marker, vec![b'a'; 1024 * 1024]).unwrap(); + assert!(!sealed_store_check_matches(&directory, &digest).unwrap()); + persist_sealed_store_check(&directory, &digest).unwrap(); + assert!(sealed_store_check_matches(&directory, &digest).unwrap()); + assert_eq!(std::fs::read(&marker).unwrap(), digest.as_bytes()); + assert_eq!(std::fs::read_dir(&directory).unwrap().count(), 1); + + std::fs::write(&marker, b"stale").unwrap(); + assert!(!sealed_store_check_matches(&directory, &digest).unwrap()); + persist_sealed_store_check(&directory, &digest).unwrap(); + assert!(sealed_store_check_matches(&directory, &digest).unwrap()); + } + + #[cfg(unix)] + #[test] + fn marker_reproof_does_not_clobber_a_foreign_symlink_target() { + let root = tempfile::tempdir().unwrap(); + let directory = root.path().join("generation"); + std::fs::create_dir(&directory).unwrap(); + let foreign = root.path().join("foreign-durable-data"); + std::fs::write(&foreign, b"must survive intact").unwrap(); + let marker = directory.join(SEALED_STORE_CHECKED_FILE); + std::os::unix::fs::symlink(&foreign, &marker).unwrap(); + let digest = "b".repeat(64); + + assert!(!sealed_store_check_matches(&directory, &digest).unwrap()); + persist_sealed_store_check(&directory, &digest).unwrap(); + assert_eq!(std::fs::read(&foreign).unwrap(), b"must survive intact"); + assert!(std::fs::symlink_metadata(&marker).unwrap().is_file()); + assert!(sealed_store_check_matches(&directory, &digest).unwrap()); + + let linked_directory = root.path().join("linked-generation"); + std::os::unix::fs::symlink(&directory, &linked_directory).unwrap(); + assert!(persist_sealed_store_check(&linked_directory, &"c".repeat(64)).is_err()); + assert_eq!(std::fs::read(&foreign).unwrap(), b"must survive intact"); + } } fn open_sealed_store_checked( From 115b493957b1d71892071e789bbcfe7dc5c10e9b Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Fri, 6 Nov 2026 16:14:34 -0800 Subject: [PATCH 42/71] fix(graph): reuse no-follow bounded reads for sealed proof files --- .../tracedecay-graph-db/src/sealed_store.rs | 94 +++++-------------- 1 file changed, 23 insertions(+), 71 deletions(-) diff --git a/crates/tracedecay-graph-db/src/sealed_store.rs b/crates/tracedecay-graph-db/src/sealed_store.rs index 32f9738f312..51c634ca62a 100644 --- a/crates/tracedecay-graph-db/src/sealed_store.rs +++ b/crates/tracedecay-graph-db/src/sealed_store.rs @@ -44,7 +44,7 @@ //! digest before the store serves a read. use std::collections::{BTreeMap, HashMap}; -use std::io::{Read, Write}; +use std::io::Write; use std::panic::{AssertUnwindSafe, catch_unwind}; use std::path::{Path, PathBuf}; use std::sync::Arc; @@ -2940,47 +2940,12 @@ fn open_sealed_store( open_sealed_store_checked(directory, identity, expected, &|| Ok(()), None) } -/// Read only a regular sealed receipt, never an arbitrary-length on-disk -/// payload. Check both the path and opened descriptor, since identity can -/// change between metadata sampling and opening the file. The capped read -/// protects against growth after both checks. +/// Use the shared no-follow, handle-validated bounded file reader. A pathname +/// can be swapped after a metadata check, so opening a symlink must fail at +/// the actual open operation, not only at an earlier path inspection. fn read_sealed_receipt_bytes(path: &Path) -> std::io::Result>> { - let metadata = match std::fs::symlink_metadata(path) { - Ok(metadata) => metadata, - Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(None), - Err(error) => return Err(error), - }; - if !metadata.file_type().is_file() { - return Err(std::io::Error::new( - std::io::ErrorKind::InvalidData, - "sealed receipt must be a regular file", - )); - } - if metadata.len() > MAX_SEALED_STORE_RECEIPT_BYTES { - return Err(std::io::Error::new( - std::io::ErrorKind::InvalidData, - "sealed receipt exceeds its byte limit", - )); - } - let file = std::fs::File::open(path)?; - if !file.metadata()?.is_file() { - return Err(std::io::Error::new( - std::io::ErrorKind::InvalidData, - "sealed receipt descriptor is not a regular file", - )); - } - let mut bytes = Vec::new(); - file.take(MAX_SEALED_STORE_RECEIPT_BYTES + 1) - .read_to_end(&mut bytes)?; - if bytes.len() as u64 > MAX_SEALED_STORE_RECEIPT_BYTES { - return Err(std::io::Error::new( - std::io::ErrorKind::InvalidData, - "sealed receipt grew past its byte limit", - )); - } - Ok(Some(bytes)) + tracedecay_private_fs::framed_log::read_bounded(path, MAX_SEALED_STORE_RECEIPT_BYTES as usize) } - fn load_sealed_store_receipt( directory: &Path, ) -> Result, GraphDbError> { @@ -2995,44 +2960,31 @@ fn load_sealed_store_receipt( .map_err(|error| GraphDbError::unavailable(format!("sealed receipt decode: {error}"))) } -/// A proof marker is only a digest, not a trusted size-unbounded payload. -/// Never follow a symlink as marker authority; cap the read even when the -/// file changes between the path and opened-descriptor checks. +/// A proof marker is only a digest, not a trusted file. Reuse the same +/// no-follow reader as receipts; a symlink swapped in during open must never +/// supply proof bytes, and oversized or nonregular markers do not match. fn sealed_store_check_matches( directory: &Path, expected_digest: &str, ) -> Result { let path = directory.join(SEALED_STORE_CHECKED_FILE); - let metadata = match std::fs::symlink_metadata(&path) { - Ok(metadata) => metadata, - Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(false), - Err(error) => return Err(sealed_store_io_failure("sealed check inspect failed", error)), - }; - let expected_len = expected_digest.len() as u64; - if expected_len > MAX_SEALED_STORE_RECEIPT_BYTES - || !metadata.file_type().is_file() - || metadata.len() != expected_len - { - return Ok(false); - } - let file = match std::fs::File::open(&path) { - Ok(file) => file, - Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(false), - Err(error) => return Err(sealed_store_io_failure("sealed check open failed", error)), - }; - let opened = file - .metadata() - .map_err(|error| sealed_store_io_failure("sealed check inspect failed", error))?; - if !opened.is_file() || opened.len() != expected_len { - return Ok(false); + let limit = expected_digest + .len() + .min(MAX_SEALED_STORE_RECEIPT_BYTES as usize); + match tracedecay_private_fs::framed_log::read_bounded(&path, limit) { + Ok(Some(bytes)) => Ok(bytes == expected_digest.as_bytes()), + Ok(None) => Ok(false), + Err(error) + if matches!( + error.kind(), + std::io::ErrorKind::InvalidInput | std::io::ErrorKind::InvalidData + ) => + { + Ok(false) + } + Err(error) => Err(sealed_store_io_failure("sealed check read failed", error)), } - let mut bytes = Vec::new(); - file.take(expected_len + 1) - .read_to_end(&mut bytes) - .map_err(|error| sealed_store_io_failure("sealed check read failed", error))?; - Ok(bytes == expected_digest.as_bytes()) } - fn persist_sealed_store_check(directory: &Path, expected_digest: &str) -> Result<(), GraphDbError> { if sealed_store_check_matches(directory, expected_digest)? { return Ok(()); From 18e1c7f52d459aae7e287387fc29d7e6261c8290 Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Fri, 6 Nov 2026 16:14:35 -0800 Subject: [PATCH 43/71] style(graph): normalize sealed-reader helper formatting --- crates/tracedecay-graph-db/src/sealed_store.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/crates/tracedecay-graph-db/src/sealed_store.rs b/crates/tracedecay-graph-db/src/sealed_store.rs index 51c634ca62a..c5dac4d9ecc 100644 --- a/crates/tracedecay-graph-db/src/sealed_store.rs +++ b/crates/tracedecay-graph-db/src/sealed_store.rs @@ -2946,6 +2946,7 @@ fn open_sealed_store( fn read_sealed_receipt_bytes(path: &Path) -> std::io::Result>> { tracedecay_private_fs::framed_log::read_bounded(path, MAX_SEALED_STORE_RECEIPT_BYTES as usize) } + fn load_sealed_store_receipt( directory: &Path, ) -> Result, GraphDbError> { @@ -2968,9 +2969,7 @@ fn sealed_store_check_matches( expected_digest: &str, ) -> Result { let path = directory.join(SEALED_STORE_CHECKED_FILE); - let limit = expected_digest - .len() - .min(MAX_SEALED_STORE_RECEIPT_BYTES as usize); + let limit = expected_digest.len().min(MAX_SEALED_STORE_RECEIPT_BYTES as usize); match tracedecay_private_fs::framed_log::read_bounded(&path, limit) { Ok(Some(bytes)) => Ok(bytes == expected_digest.as_bytes()), Ok(None) => Ok(false), @@ -2985,6 +2984,7 @@ fn sealed_store_check_matches( Err(error) => Err(sealed_store_io_failure("sealed check read failed", error)), } } + fn persist_sealed_store_check(directory: &Path, expected_digest: &str) -> Result<(), GraphDbError> { if sealed_store_check_matches(directory, expected_digest)? { return Ok(()); From 1b6af8fe2876665a699af87a1451f0b0dade3f27 Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Fri, 6 Nov 2026 16:14:36 -0800 Subject: [PATCH 44/71] fix(retention): safely publish bounded cold-store cursors Refs #4787. Guards the maintenance cursor only; the canonical-body and replay-metadata CPU-loop cause remains unproven. --- .../src/retention/cold_store.rs | 134 ++++++++++++++++-- 1 file changed, 126 insertions(+), 8 deletions(-) diff --git a/crates/tracedecay-maintenance/src/retention/cold_store.rs b/crates/tracedecay-maintenance/src/retention/cold_store.rs index 4ccaa70331d..778b4522295 100644 --- a/crates/tracedecay-maintenance/src/retention/cold_store.rs +++ b/crates/tracedecay-maintenance/src/retention/cold_store.rs @@ -4,6 +4,7 @@ //! (`store_maintenance::run_registered_code_generation_retention`). use std::collections::BTreeSet; +use std::io::Read; use std::path::{Path, PathBuf}; use std::time::Instant; @@ -17,6 +18,7 @@ use crate::clock::now_secs_i64; const COLD_STORE_PAGE_LIMIT: usize = 8; const CHECKPOINT_DIRECTORY: &str = "maintenance"; const CHECKPOINT_FILE: &str = "retention-cold-store-cursor-v1.json"; +const MAX_CHECKPOINT_BYTES: u64 = 64 * 1024; #[derive(Debug, Clone, PartialEq, Eq, Deserialize, Serialize)] struct ColdStoreCursorV1 { @@ -247,8 +249,51 @@ fn checkpoint_path(profile_root: &Path) -> PathBuf { #[tracing::instrument(name = "maintenance.cold_store.load_cursor", level = "trace", skip_all)] fn load_cursor(path: &Path) -> Option { - let bytes = std::fs::read(path).ok()?; - serde_json::from_slice(&bytes).ok() + let metadata = match std::fs::symlink_metadata(path) { + Ok(metadata) => metadata, + Err(error) if error.kind() == std::io::ErrorKind::NotFound => return None, + Err(error) => { + tracing::warn!( + event = "retention_cold_store_cursor_invalid", + %error, + "cannot inspect the saved maintenance cursor; restarting its bounded census" + ); + return None; + } + }; + if !metadata.file_type().is_file() || metadata.len() > MAX_CHECKPOINT_BYTES { + tracing::warn!( + event = "retention_cold_store_cursor_invalid", + reason = "nonregular_or_oversized", + "saved maintenance cursor is unsafe; restarting its bounded census" + ); + return None; + } + // Check the byte limit again on the open handle: the path may change after + // metadata inspection, and an invalid checkpoint must never load unbounded data. + let file = std::fs::File::open(path).ok()?; + let mut limited = file.take(MAX_CHECKPOINT_BYTES + 1); + let mut bytes = Vec::new(); + limited.read_to_end(&mut bytes).ok()?; + if bytes.len() as u64 > MAX_CHECKPOINT_BYTES { + tracing::warn!( + event = "retention_cold_store_cursor_invalid", + reason = "oversized_after_open", + "saved maintenance cursor grew during its read; restarting its bounded census" + ); + return None; + } + match serde_json::from_slice(&bytes) { + Ok(cursor) => Some(cursor), + Err(error) => { + tracing::warn!( + event = "retention_cold_store_cursor_invalid", + %error, + "saved maintenance cursor is invalid; restarting its bounded census" + ); + None + } + } } #[tracing::instrument( @@ -261,12 +306,16 @@ fn persist_cursor(path: &Path, cursor: &ColdStoreCursorV1) -> std::io::Result<() .parent() .ok_or_else(|| std::io::Error::other("maintenance cursor has no parent"))?; std::fs::create_dir_all(parent)?; - let temporary = path.with_extension("json.tmp"); + // The old fixed .json.tmp name could be replaced with a symlink and + // File::create would truncate its foreign target. Random exclusive staging + // keeps the write private; the final rename replaces, never follows, a + // substituted destination symlink. + let mut temporary = tempfile::NamedTempFile::new_in(parent)?; let bytes = serde_json::to_vec(cursor).map_err(std::io::Error::other)?; - let mut file = std::fs::File::create(&temporary)?; - std::io::Write::write_all(&mut file, &bytes)?; - file.sync_all()?; - std::fs::rename(temporary, path) + std::io::Write::write_all(temporary.as_file_mut(), &bytes)?; + temporary.as_file_mut().sync_all()?; + temporary.persist(path).map_err(|error| error.error)?; + Ok(()) } fn retention_window_secs(days: u64) -> i64 { @@ -306,10 +355,79 @@ mod tests { persist_cursor(&path, &expected).expect("persist cursor"); - assert_eq!(load_cursor(&path), Some(expected)); + assert_eq!(load_cursor(&path), Some(expected.clone())); + let updated = ColdStoreCursorV1 { + after_project_id: Some("project-c".to_owned()), + after_unregistered_directory: None, + }; + persist_cursor(&path, &updated).expect("replace existing checkpoint"); + assert_eq!(load_cursor(&path), Some(updated)); assert!(!path.with_extension("json.tmp").exists()); } + #[cfg(unix)] + #[test] + fn checkpoint_publication_does_not_follow_foreign_file_symlinks() { + use std::os::unix::fs::symlink; + + let root = tempfile::tempdir().expect("checkpoint root"); + let path = checkpoint_path(root.path()); + std::fs::create_dir_all(path.parent().unwrap()).unwrap(); + let foreign = root.path().join("foreign-data"); + std::fs::write(&foreign, b"unrelated durable bytes").unwrap(); + // Both a leftover fixed temporary name and the destination may be + // replaced by a link. Neither may grant write authority to its target. + let legacy_temporary = path.with_extension("json.tmp"); + symlink(&foreign, &legacy_temporary).unwrap(); + symlink(&foreign, &path).unwrap(); + assert!( + load_cursor(&path).is_none(), + "foreign checkpoint links cannot be adopted as cursors" + ); + + let expected = ColdStoreCursorV1 { + after_project_id: Some("project-c".to_owned()), + after_unregistered_directory: None, + }; + persist_cursor(&path, &expected).expect("replace checkpoint safely"); + assert_eq!(std::fs::read(&foreign).unwrap(), b"unrelated durable bytes"); + assert!( + std::fs::symlink_metadata(&legacy_temporary) + .unwrap() + .file_type() + .is_symlink(), + "foreign legacy temporary entries must not be overwritten" + ); + assert!( + std::fs::symlink_metadata(&path) + .unwrap() + .file_type() + .is_file(), + "a checkpoint symlink is replaced by an ordinary file" + ); + assert_eq!(load_cursor(&path), Some(expected)); + } + + #[test] + fn malformed_and_oversized_checkpoints_do_not_authorize_unbounded_reads() { + let root = tempfile::tempdir().expect("checkpoint root"); + let path = checkpoint_path(root.path()); + std::fs::create_dir_all(path.parent().unwrap()).unwrap(); + + std::fs::write(&path, vec![b' '; (MAX_CHECKPOINT_BYTES + 1) as usize]) + .expect("oversized checkpoint"); + assert_eq!(load_cursor(&path), None); + std::fs::write(&path, b"{invalid").expect("malformed checkpoint"); + assert_eq!(load_cursor(&path), None); + + let valid = ColdStoreCursorV1 { + after_project_id: None, + after_unregistered_directory: None, + }; + persist_cursor(&path, &valid).expect("restore valid checkpoint"); + assert_eq!(load_cursor(&path), Some(valid)); + } + #[test] fn state_distinguishes_missing_unreadable_and_cancelled() { assert_eq!( From 834a4a65719ab4da03809e997d54447ba5c1fd5a Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Fri, 6 Nov 2026 16:14:37 -0800 Subject: [PATCH 45/71] fix(retention): expose bounded page progress Refs #4787 --- .../src/observation/canonical_body_gc.rs | 28 ++++++++--- .../observation/canonical_body_gc/tests.rs | 49 +++++++++++++++++++ .../src/retention/replay_metadata.rs | 11 +++++ .../src/retention/replay_metadata/tests.rs | 34 +++++++++++++ crates/tracedecay/src/daemon/maintenance.rs | 18 +++++++ 5 files changed, 133 insertions(+), 7 deletions(-) diff --git a/crates/tracedecay-global-db/src/observation/canonical_body_gc.rs b/crates/tracedecay-global-db/src/observation/canonical_body_gc.rs index 6c5af50fea3..7b5c0b056f4 100644 --- a/crates/tracedecay-global-db/src/observation/canonical_body_gc.rs +++ b/crates/tracedecay-global-db/src/observation/canonical_body_gc.rs @@ -25,6 +25,8 @@ pub const MAX_CANONICAL_BODY_GC_ROWS: usize = 48; pub struct CanonicalBodyGcReport { pub roots_scanned: u64, pub bodies_scanned: u64, + /// Candidates discharged in the committed page, even if their body stays live. + pub candidates_discharged: u64, pub bodies_deleted: u64, /// Stored BLOB bytes deleted, including the effect of their existing /// compression. This is not database-file or filesystem space reclaimed. @@ -323,13 +325,25 @@ async fn collect_candidates( drop(deleted); // Referenced candidates are discharged too. The final owner removal // records them again; idle ticks never revisit every retained body. - conn.execute( - "DELETE FROM canonical_body_gc_candidates - WHERE content_hash IN (SELECT value FROM json_each(?1))", - params![hashes], - ) - .await - .map_err(db_error)?; + // Deleting an unreferenced body cascades to its candidate row. The + // remaining explicit removals discharge referenced bodies that stay live. + let explicitly_discharged = conn + .execute( + "DELETE FROM canonical_body_gc_candidates + WHERE content_hash IN (SELECT value FROM json_each(?1))", + params![hashes], + ) + .await + .map_err(db_error)?; + let discharged = explicitly_discharged + .checked_add(report.bodies_deleted) + .ok_or_else(|| integrity_error("canonical body candidate discharge overflow"))?; + if discharged != report.bodies_scanned { + return Err(integrity_error( + "canonical body candidate page was not fully discharged", + )); + } + report.candidates_discharged = discharged; Ok(()) } diff --git a/crates/tracedecay-global-db/src/observation/canonical_body_gc/tests.rs b/crates/tracedecay-global-db/src/observation/canonical_body_gc/tests.rs index 9da4f654323..9b2accfd09e 100644 --- a/crates/tracedecay-global-db/src/observation/canonical_body_gc/tests.rs +++ b/crates/tracedecay-global-db/src/observation/canonical_body_gc/tests.rs @@ -506,3 +506,52 @@ async fn nonpositive_legacy_owner_ids_cannot_be_skipped_by_bootstrap() { ); } } + +#[asupersync::test] +async fn zero_delete_pages_discharge_distinct_candidates_and_quiesce() { + let store = Store::open().await; + for ordinal in 0..52 { + let body = + StoredCanonicalBody::pack(format!("still referenced {ordinal}").as_bytes()).unwrap(); + insert_body(store.conn(), &body).await; + store + .conn() + .execute( + "INSERT INTO lcm_raw_messages(storage_kind, content, content_hash) + VALUES ('inline', NULL, ?1)", + params![body.content_hash.as_str()], + ) + .await + .unwrap(); + } + store.install().await; + + let mut discharged_pages = Vec::new(); + for _ in 0..16 { + let report = store.tick(48).await; + assert_eq!(report.bodies_deleted, 0, "referenced bodies stay live"); + if report.candidates_discharged > 0 { + assert_eq!(report.candidates_discharged, report.bodies_scanned); + discharged_pages.push(report.candidates_discharged); + } + if !report.has_more { + break; + } + } + assert_eq!(discharged_pages, vec![48, 4]); + assert!(!has_candidates(store.conn()).await.unwrap()); + assert_eq!( + store.tick(48).await, + CanonicalBodyGcReport { + bootstrap_complete: true, + ..CanonicalBodyGcReport::default() + } + ); + let mut rows = store + .conn() + .query("SELECT COUNT(*) FROM session_canonical_bodies", ()) + .await + .unwrap(); + let count: i64 = rows.next().await.unwrap().unwrap().get(0).unwrap(); + assert_eq!(count, 52); +} diff --git a/crates/tracedecay-lcm/src/retention/replay_metadata.rs b/crates/tracedecay-lcm/src/retention/replay_metadata.rs index b7be38726d9..8c6998f4965 100644 --- a/crates/tracedecay-lcm/src/retention/replay_metadata.rs +++ b/crates/tracedecay-lcm/src/retention/replay_metadata.rs @@ -25,6 +25,9 @@ const MAX_BATCH_BYTES: u64 = 4 * 1024 * 1024; #[derive(Clone, Debug, Default, PartialEq, Eq, Serialize, Deserialize)] pub struct ReplayMetadataCompactionReport { pub rows_scanned: u64, + /// Transactional store-id frontier consumed by this page, absent on idle reads. + pub frontier_before: Option, + pub frontier_after: Option, pub rows_compacted: u64, /// Eligible rows retained because their representation, sanitizer result, /// or byte bound could not prove this narrow lossless transformation. @@ -150,6 +153,7 @@ async fn step( max_rows: usize, ) -> Result { let mut after = frontier(conn).await?; + let before = after; let candidates = candidates(conn, after, max_rows).await?; let mut report = ReplayMetadataCompactionReport::default(); let mut remaining = MAX_BATCH_BYTES; @@ -199,6 +203,13 @@ async fn step( if guard_suspended { summary_convergence::restore_raw_update_guard(conn).await?; } + if report.rows_scanned > 0 { + if after <= before { + return Err(db_error("replay metadata page did not advance its frontier")); + } + report.frontier_before = Some(before); + report.frontier_after = Some(after); + } schema::set_gc_meta(conn, FRONTIER_KEY, &after.to_string()).await?; report.has_more = has_work(conn, after).await?; Ok(report) diff --git a/crates/tracedecay-lcm/src/retention/replay_metadata/tests.rs b/crates/tracedecay-lcm/src/retention/replay_metadata/tests.rs index 1bc50de1ce3..38a405796cb 100644 --- a/crates/tracedecay-lcm/src/retention/replay_metadata/tests.rs +++ b/crates/tracedecay-lcm/src/retention/replay_metadata/tests.rs @@ -369,3 +369,37 @@ async fn row_and_byte_budgets_advance_a_durable_frontier_and_finished_pages_are_ let idle = store.tick(48).await; assert_eq!(idle, ReplayMetadataCompactionReport::default()); } + +#[asupersync::test] +async fn zero_compaction_pages_advance_distinct_durable_frontiers() { + let store = Store::open().await; + for ordinal in 1..=52 { + store + .insert( + ordinal, + "canonical source", + Value::String("different replay".to_owned()), + false, + ) + .await; + } + let first = store.tick(48).await; + assert_eq!(first.rows_scanned, 48); + assert_eq!(first.rows_compacted, 0); + assert_eq!(first.frontier_before, Some(0)); + assert_eq!(first.frontier_after, Some(48)); + assert!(first.has_more); + + let second = store.tick(48).await; + assert_eq!(second.rows_scanned, 4); + assert_eq!(second.rows_compacted, 0); + assert_eq!(second.frontier_before, Some(48)); + assert_eq!(second.frontier_after, Some(52)); + assert!(!second.has_more); + assert_eq!(frontier(&store.conn).await.unwrap(), 52); + + assert_eq!( + store.tick(48).await, + ReplayMetadataCompactionReport::default() + ); +} diff --git a/crates/tracedecay/src/daemon/maintenance.rs b/crates/tracedecay/src/daemon/maintenance.rs index ea2070195f9..9e293532be4 100644 --- a/crates/tracedecay/src/daemon/maintenance.rs +++ b/crates/tracedecay/src/daemon/maintenance.rs @@ -386,6 +386,10 @@ fn record_canonical_body_gc_outcome( &[ ("roots_scanned", report.roots_scanned.to_string()), ("bodies_scanned", report.bodies_scanned.to_string()), + ( + "candidates_discharged", + report.candidates_discharged.to_string(), + ), ("bodies_deleted", report.bodies_deleted.to_string()), ( "stored_bytes_deleted", @@ -428,6 +432,20 @@ fn record_replay_metadata_outcome( "retention_replay_metadata", &[ ("rows_scanned", report.rows_scanned.to_string()), + ( + "frontier_before", + report.frontier_before.map_or_else( + || "not_scanned".to_owned(), + |value| value.to_string(), + ), + ), + ( + "frontier_after", + report.frontier_after.map_or_else( + || "not_scanned".to_owned(), + |value| value.to_string(), + ), + ), ("rows_compacted", report.rows_compacted.to_string()), ("rows_skipped", report.rows_skipped.to_string()), ( From fd964edfcf2161cb41fa854f6e0d706afd869fff Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Fri, 6 Nov 2026 16:21:32 -0800 Subject: [PATCH 46/71] style: align bounded retention readers with rustfmt 2024 --- crates/tracedecay-graph-db/src/sealed_store.rs | 12 +++++++++--- .../tracedecay-lcm/src/retention/replay_metadata.rs | 4 +++- 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/crates/tracedecay-graph-db/src/sealed_store.rs b/crates/tracedecay-graph-db/src/sealed_store.rs index c5dac4d9ecc..e41aeda64ed 100644 --- a/crates/tracedecay-graph-db/src/sealed_store.rs +++ b/crates/tracedecay-graph-db/src/sealed_store.rs @@ -2969,7 +2969,9 @@ fn sealed_store_check_matches( expected_digest: &str, ) -> Result { let path = directory.join(SEALED_STORE_CHECKED_FILE); - let limit = expected_digest.len().min(MAX_SEALED_STORE_RECEIPT_BYTES as usize); + let limit = expected_digest + .len() + .min(MAX_SEALED_STORE_RECEIPT_BYTES as usize); match tracedecay_private_fs::framed_log::read_bounded(&path, limit) { Ok(Some(bytes)) => Ok(bytes == expected_digest.as_bytes()), Ok(None) => Ok(false), @@ -2998,7 +3000,9 @@ fn persist_sealed_store_check(directory: &Path, expected_digest: &str) -> Result .map_err(|error| sealed_store_io_failure("sealed marker root inspect failed", error))?; let directory_is_real = std::fs::symlink_metadata(directory) .map(|metadata| metadata.file_type().is_dir()) - .map_err(|error| sealed_store_io_failure("sealed marker directory inspect failed", error))?; + .map_err(|error| { + sealed_store_io_failure("sealed marker directory inspect failed", error) + })?; if !root_is_real || !directory_is_real { return Err(GraphDbError::unavailable( "sealed marker directory is not a real directory", @@ -3021,7 +3025,9 @@ fn persist_sealed_store_check(directory: &Path, expected_digest: &str) -> Result #[cfg(test)] mod sealed_marker_tests { - use super::{SEALED_STORE_CHECKED_FILE, persist_sealed_store_check, sealed_store_check_matches}; + use super::{ + SEALED_STORE_CHECKED_FILE, persist_sealed_store_check, sealed_store_check_matches, + }; #[test] fn proof_marker_requires_exact_bounded_digest_and_replaces_stale_bytes() { diff --git a/crates/tracedecay-lcm/src/retention/replay_metadata.rs b/crates/tracedecay-lcm/src/retention/replay_metadata.rs index 8c6998f4965..72910e4798a 100644 --- a/crates/tracedecay-lcm/src/retention/replay_metadata.rs +++ b/crates/tracedecay-lcm/src/retention/replay_metadata.rs @@ -205,7 +205,9 @@ async fn step( } if report.rows_scanned > 0 { if after <= before { - return Err(db_error("replay metadata page did not advance its frontier")); + return Err(db_error( + "replay metadata page did not advance its frontier", + )); } report.frontier_before = Some(before); report.frontier_after = Some(after); From 9d5693c628a4b3bc81d55c67c57f034e381fc22d Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Fri, 6 Nov 2026 17:10:24 -0800 Subject: [PATCH 47/71] fix(storage): refuse symlinked footprint walk roots Refs #4099 --- .../src/retention/footprint.rs | 43 +++++++++++++++++++ 1 file changed, 43 insertions(+) diff --git a/crates/tracedecay-maintenance/src/retention/footprint.rs b/crates/tracedecay-maintenance/src/retention/footprint.rs index 3fc71f40334..83f74a6d57c 100644 --- a/crates/tracedecay-maintenance/src/retention/footprint.rs +++ b/crates/tracedecay-maintenance/src/retention/footprint.rs @@ -235,6 +235,15 @@ pub fn walk_regular_files_with_budget( WalkInterruptionV1::DeadlineExceeded => WalkTerminationV1::DeadlineExceeded, }; } + // The caller's root can itself be a symlink; DirEntry::file_type + // below only protects children. Never attribute a foreign tree's + // bytes to the requested store. + if !std::fs::symlink_metadata(&directory) + .is_ok_and(|metadata| metadata.file_type().is_dir()) + { + visit(&directory, WalkedEntryV1::Unavailable); + continue; + } let Ok(entries) = std::fs::read_dir(&directory) else { visit(&directory, WalkedEntryV1::Unavailable); continue; @@ -1128,6 +1137,40 @@ mod tests { assert_eq!(remaining, 100 - 1 - 64 - 1 - 1); } + #[cfg(unix)] + #[test] + fn symlinked_walk_root_is_partial_without_traversing_foreign_files() { + let temporary = tempfile::tempdir().expect("walk fixture"); + let outside = temporary.path().join("outside"); + std::fs::create_dir(&outside).expect("outside directory"); + sized(&outside.join("private-a"), 900); + sized(&outside.join("private-b"), 900); + let linked_root = temporary.path().join("linked-profile"); + std::os::unix::fs::symlink(&outside, &linked_root).expect("linked root"); + + let mut remaining = 1; + let mut visited = Vec::new(); + let termination = walk_regular_files_with_budget( + &linked_root, + &mut remaining, + &|| None, + &mut |path, entry| visited.push((path.to_path_buf(), entry)), + ); + assert_eq!(termination, WalkTerminationV1::Complete); + assert_eq!(visited, vec![(linked_root.clone(), WalkedEntryV1::Unavailable)]); + assert_eq!(remaining, 1, "foreign children must not spend the entry budget"); + + let footprint = census_profile_footprint(&linked_root, &BTreeMap::new(), &BTreeMap::new()); + assert_eq!(footprint.total_bytes, 0); + assert_eq!(footprint.unavailable_entry_count, 1); + assert_eq!( + footprint.full_profile_size().state, + ProfileTotalCoverageStateV1::Partial, + "unreadable store roots cannot report healthy complete coverage" + ); + assert_eq!(std::fs::metadata(outside.join("private-a")).unwrap().len(), 900); + } + #[test] fn one_entry_budget_is_shared_across_walk_roots() { let temporary = tempfile::tempdir().expect("walk fixture"); From 7083168ef66f42c7e77bb1176ae1cc1aee6382f6 Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Fri, 6 Nov 2026 17:15:37 -0800 Subject: [PATCH 48/71] style(maintenance): format symlinked footprint regression --- .../src/retention/footprint.rs | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/crates/tracedecay-maintenance/src/retention/footprint.rs b/crates/tracedecay-maintenance/src/retention/footprint.rs index 83f74a6d57c..7423a60e7c3 100644 --- a/crates/tracedecay-maintenance/src/retention/footprint.rs +++ b/crates/tracedecay-maintenance/src/retention/footprint.rs @@ -1157,8 +1157,14 @@ mod tests { &mut |path, entry| visited.push((path.to_path_buf(), entry)), ); assert_eq!(termination, WalkTerminationV1::Complete); - assert_eq!(visited, vec![(linked_root.clone(), WalkedEntryV1::Unavailable)]); - assert_eq!(remaining, 1, "foreign children must not spend the entry budget"); + assert_eq!( + visited, + vec![(linked_root.clone(), WalkedEntryV1::Unavailable)] + ); + assert_eq!( + remaining, 1, + "foreign children must not spend the entry budget" + ); let footprint = census_profile_footprint(&linked_root, &BTreeMap::new(), &BTreeMap::new()); assert_eq!(footprint.total_bytes, 0); @@ -1168,7 +1174,10 @@ mod tests { ProfileTotalCoverageStateV1::Partial, "unreadable store roots cannot report healthy complete coverage" ); - assert_eq!(std::fs::metadata(outside.join("private-a")).unwrap().len(), 900); + assert_eq!( + std::fs::metadata(outside.join("private-a")).unwrap().len(), + 900 + ); } #[test] From fcce64d8359f9a06f2dd7398ebe2c6d9662ac9b5 Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Fri, 6 Nov 2026 18:16:15 -0800 Subject: [PATCH 49/71] fix(maintenance): reject linked cold-store cursor parents Refs #4787 --- .../src/retention/cold_store.rs | 73 +++++++++++++++++++ 1 file changed, 73 insertions(+) diff --git a/crates/tracedecay-maintenance/src/retention/cold_store.rs b/crates/tracedecay-maintenance/src/retention/cold_store.rs index 778b4522295..5250b51443e 100644 --- a/crates/tracedecay-maintenance/src/retention/cold_store.rs +++ b/crates/tracedecay-maintenance/src/retention/cold_store.rs @@ -247,8 +247,37 @@ fn checkpoint_path(profile_root: &Path) -> PathBuf { .join(CHECKPOINT_FILE) } +// A maintenance directory substituted with a symlink must not turn a +// checkpoint read or write into access to an unrelated store. This checks +// the directory before use; it is not a replacement for a descriptor-relative +// open against concurrent path substitution. +fn verified_checkpoint_parent(path: &Path) -> std::io::Result<&Path> { + let parent = path + .parent() + .ok_or_else(|| std::io::Error::other("maintenance cursor has no parent"))?; + let metadata = std::fs::symlink_metadata(parent)?; + if !metadata.file_type().is_dir() { + return Err(std::io::Error::new( + std::io::ErrorKind::InvalidData, + "maintenance cursor directory is not a real directory", + )); + } + Ok(parent) +} + #[tracing::instrument(name = "maintenance.cold_store.load_cursor", level = "trace", skip_all)] fn load_cursor(path: &Path) -> Option { + if let Err(error) = verified_checkpoint_parent(path) { + if error.kind() != std::io::ErrorKind::NotFound { + tracing::warn!( + event = "retention_cold_store_cursor_invalid", + reason = "untrusted_directory", + %error, + "maintenance cursor directory is unsafe; restarting its bounded census" + ); + } + return None; + } let metadata = match std::fs::symlink_metadata(path) { Ok(metadata) => metadata, Err(error) if error.kind() == std::io::ErrorKind::NotFound => return None, @@ -306,6 +335,7 @@ fn persist_cursor(path: &Path, cursor: &ColdStoreCursorV1) -> std::io::Result<() .parent() .ok_or_else(|| std::io::Error::other("maintenance cursor has no parent"))?; std::fs::create_dir_all(parent)?; + verified_checkpoint_parent(path)?; // The old fixed .json.tmp name could be replaced with a symlink and // File::create would truncate its foreign target. Random exclusive staging // keeps the write private; the final rename replaces, never follows, a @@ -314,6 +344,7 @@ fn persist_cursor(path: &Path, cursor: &ColdStoreCursorV1) -> std::io::Result<() let bytes = serde_json::to_vec(cursor).map_err(std::io::Error::other)?; std::io::Write::write_all(temporary.as_file_mut(), &bytes)?; temporary.as_file_mut().sync_all()?; + verified_checkpoint_parent(path)?; temporary.persist(path).map_err(|error| error.error)?; Ok(()) } @@ -408,6 +439,48 @@ mod tests { assert_eq!(load_cursor(&path), Some(expected)); } + #[cfg(unix)] + #[test] + fn a_linked_maintenance_directory_never_claims_foreign_cursors() { + use std::os::unix::fs::symlink; + + let root = tempfile::tempdir().expect("profile root"); + let profile = root.path().join("profile"); + let foreign = root.path().join("foreign"); + std::fs::create_dir(&profile).unwrap(); + std::fs::create_dir(&foreign).unwrap(); + let path = checkpoint_path(&profile); + let foreign_cursor = foreign.join(CHECKPOINT_FILE); + let foreign_bytes = br#"{"after_project_id":"foreign"}"#; + std::fs::write(&foreign_cursor, foreign_bytes).unwrap(); + symlink(&foreign, profile.join(CHECKPOINT_DIRECTORY)).unwrap(); + + assert_eq!( + load_cursor(&path), + None, + "a linked checkpoint parent cannot supply a foreign resume position" + ); + let expected = ColdStoreCursorV1 { + after_project_id: Some("project-local".to_owned()), + after_unregistered_directory: None, + }; + assert!( + persist_cursor(&path, &expected).is_err(), + "publication must not cross a substituted maintenance directory" + ); + assert_eq!(std::fs::read(&foreign_cursor).unwrap(), foreign_bytes); + assert_eq!( + std::fs::read_dir(&foreign).unwrap().count(), + 1, + "no staging file may be created in the foreign directory" + ); + + std::fs::remove_file(profile.join(CHECKPOINT_DIRECTORY)).unwrap(); + persist_cursor(&path, &expected).unwrap(); + assert_eq!(load_cursor(&path), Some(expected)); + assert_eq!(std::fs::read(&foreign_cursor).unwrap(), foreign_bytes); + } + #[test] fn malformed_and_oversized_checkpoints_do_not_authorize_unbounded_reads() { let root = tempfile::tempdir().expect("checkpoint root"); From d4098f48f820ede1a18d16f5061537641c83e7da Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Fri, 6 Nov 2026 19:09:16 -0800 Subject: [PATCH 50/71] fix(maintenance): anchor cold cursor access to directory handles Refs #4787 --- .../src/retention/cold_store.rs | 174 +++++++++++++----- 1 file changed, 130 insertions(+), 44 deletions(-) diff --git a/crates/tracedecay-maintenance/src/retention/cold_store.rs b/crates/tracedecay-maintenance/src/retention/cold_store.rs index 5250b51443e..b516850b6d1 100644 --- a/crates/tracedecay-maintenance/src/retention/cold_store.rs +++ b/crates/tracedecay-maintenance/src/retention/cold_store.rs @@ -4,11 +4,18 @@ //! (`store_maintenance::run_registered_code_generation_retention`). use std::collections::BTreeSet; -use std::io::Read; +use std::ffi::OsStr; +use std::io::{self, Read, Write}; +use std::sync::atomic::{AtomicU64, Ordering}; use std::path::{Path, PathBuf}; use std::time::Instant; +use cap_fs_ext::{FollowSymlinks, OpenOptionsFollowExt}; +use cap_std::fs::{Dir, OpenOptions}; +#[cfg(unix)] +use cap_std::fs::OpenOptionsExt; use serde::{Deserialize, Serialize}; +use tracedecay_private_fs::capability_dir::{replace_file, sync_directory}; use tracedecay_global_db::RegisteredGlobalDb; use tracedecay_runtime_core::cancellation::{CancellationToken, MonotonicDeadline}; @@ -19,6 +26,7 @@ const COLD_STORE_PAGE_LIMIT: usize = 8; const CHECKPOINT_DIRECTORY: &str = "maintenance"; const CHECKPOINT_FILE: &str = "retention-cold-store-cursor-v1.json"; const MAX_CHECKPOINT_BYTES: u64 = 64 * 1024; +static CURSOR_TEMP_NONCE: AtomicU64 = AtomicU64::new(0); #[derive(Debug, Clone, PartialEq, Eq, Deserialize, Serialize)] struct ColdStoreCursorV1 { @@ -247,50 +255,60 @@ fn checkpoint_path(profile_root: &Path) -> PathBuf { .join(CHECKPOINT_FILE) } -// A maintenance directory substituted with a symlink must not turn a -// checkpoint read or write into access to an unrelated store. This checks -// the directory before use; it is not a replacement for a descriptor-relative -// open against concurrent path substitution. -fn verified_checkpoint_parent(path: &Path) -> std::io::Result<&Path> { - let parent = path +// Keep the maintenance directory capability open for the whole cursor read or +// publish. A path check followed by a path-based open or rename can be raced +// with a symlink substitution of "maintenance". +fn checkpoint_directory(path: &Path, create: bool) -> io::Result { + let profile = path .parent() - .ok_or_else(|| std::io::Error::other("maintenance cursor has no parent"))?; - let metadata = std::fs::symlink_metadata(parent)?; - if !metadata.file_type().is_dir() { - return Err(std::io::Error::new( - std::io::ErrorKind::InvalidData, - "maintenance cursor directory is not a real directory", - )); + .and_then(Path::parent) + .ok_or_else(|| io::Error::other("maintenance cursor has no profile root"))?; + let root = Dir::open_ambient_dir(profile, cap_std::ambient_authority())?; + if create { + match root.create_dir(CHECKPOINT_DIRECTORY) { + Ok(()) => {} + Err(error) if error.kind() == io::ErrorKind::AlreadyExists => {} + Err(error) => return Err(error), + } } - Ok(parent) + root.open_dir_nofollow(CHECKPOINT_DIRECTORY) } #[tracing::instrument(name = "maintenance.cold_store.load_cursor", level = "trace", skip_all)] fn load_cursor(path: &Path) -> Option { - if let Err(error) = verified_checkpoint_parent(path) { - if error.kind() != std::io::ErrorKind::NotFound { + let directory = match checkpoint_directory(path, false) { + Ok(directory) => directory, + Err(error) if error.kind() == io::ErrorKind::NotFound => return None, + Err(error) => { tracing::warn!( event = "retention_cold_store_cursor_invalid", reason = "untrusted_directory", %error, "maintenance cursor directory is unsafe; restarting its bounded census" ); + return None; } - return None; - } - let metadata = match std::fs::symlink_metadata(path) { - Ok(metadata) => metadata, - Err(error) if error.kind() == std::io::ErrorKind::NotFound => return None, + }; + load_cursor_in_dir(&directory) +} + +fn load_cursor_in_dir(directory: &Dir) -> Option { + let mut options = OpenOptions::new(); + options.read(true).follow(FollowSymlinks::No); + let file = match directory.open_with(CHECKPOINT_FILE, &options) { + Ok(file) => file, + Err(error) if error.kind() == io::ErrorKind::NotFound => return None, Err(error) => { tracing::warn!( event = "retention_cold_store_cursor_invalid", %error, - "cannot inspect the saved maintenance cursor; restarting its bounded census" + "cannot safely open the saved maintenance cursor" ); return None; } }; - if !metadata.file_type().is_file() || metadata.len() > MAX_CHECKPOINT_BYTES { + let metadata = file.metadata().ok()?; + if !metadata.is_file() || metadata.len() > MAX_CHECKPOINT_BYTES { tracing::warn!( event = "retention_cold_store_cursor_invalid", reason = "nonregular_or_oversized", @@ -298,9 +316,6 @@ fn load_cursor(path: &Path) -> Option { ); return None; } - // Check the byte limit again on the open handle: the path may change after - // metadata inspection, and an invalid checkpoint must never load unbounded data. - let file = std::fs::File::open(path).ok()?; let mut limited = file.take(MAX_CHECKPOINT_BYTES + 1); let mut bytes = Vec::new(); limited.read_to_end(&mut bytes).ok()?; @@ -330,23 +345,50 @@ fn load_cursor(path: &Path) -> Option { level = "trace", skip_all )] -fn persist_cursor(path: &Path, cursor: &ColdStoreCursorV1) -> std::io::Result<()> { - let parent = path - .parent() - .ok_or_else(|| std::io::Error::other("maintenance cursor has no parent"))?; - std::fs::create_dir_all(parent)?; - verified_checkpoint_parent(path)?; - // The old fixed .json.tmp name could be replaced with a symlink and - // File::create would truncate its foreign target. Random exclusive staging - // keeps the write private; the final rename replaces, never follows, a - // substituted destination symlink. - let mut temporary = tempfile::NamedTempFile::new_in(parent)?; - let bytes = serde_json::to_vec(cursor).map_err(std::io::Error::other)?; - std::io::Write::write_all(temporary.as_file_mut(), &bytes)?; - temporary.as_file_mut().sync_all()?; - verified_checkpoint_parent(path)?; - temporary.persist(path).map_err(|error| error.error)?; - Ok(()) +fn persist_cursor(path: &Path, cursor: &ColdStoreCursorV1) -> io::Result<()> { + let directory = checkpoint_directory(path, true)?; + persist_cursor_in_dir(&directory, cursor) +} + +fn persist_cursor_in_dir(directory: &Dir, cursor: &ColdStoreCursorV1) -> io::Result<()> { + let bytes = serde_json::to_vec(cursor).map_err(io::Error::other)?; + // The name supplies no authority. create_new + no-follow guarantees that + // even a collided or attacker-prepared staging name cannot be truncated. + for _ in 0..16 { + let nonce = CURSOR_TEMP_NONCE.fetch_add(1, Ordering::Relaxed); + let temporary_name = format!( + ".retention-cold-store-{}-{nonce}.tmp", + std::process::id() + ); + let mut options = OpenOptions::new(); + options.write(true).create_new(true).follow(FollowSymlinks::No); + #[cfg(unix)] + options.mode(0o600); + let mut temporary = match directory.open_with(&temporary_name, &options) { + Ok(temporary) => temporary, + Err(error) if error.kind() == io::ErrorKind::AlreadyExists => continue, + Err(error) => return Err(error), + }; + let result = (|| { + temporary.write_all(&bytes)?; + temporary.sync_all()?; + drop(temporary); + replace_file( + directory, + OsStr::new(&temporary_name), + OsStr::new(CHECKPOINT_FILE), + )?; + sync_directory(directory) + })(); + if result.is_err() { + let _ = directory.remove_file(&temporary_name); + } + return result; + } + Err(io::Error::new( + io::ErrorKind::AlreadyExists, + "maintenance cursor staging names are occupied", + )) } fn retention_window_secs(days: u64) -> i64 { @@ -481,6 +523,50 @@ mod tests { assert_eq!(std::fs::read(&foreign_cursor).unwrap(), foreign_bytes); } + + #[cfg(unix)] + #[test] + fn a_replaced_parent_cannot_redirect_an_open_checkpoint_capability() { + use std::os::unix::fs::symlink; + + let root = tempfile::tempdir().unwrap(); + let profile = root.path().join("profile"); + let foreign = root.path().join("foreign"); + std::fs::create_dir(&profile).unwrap(); + std::fs::create_dir(&foreign).unwrap(); + let path = checkpoint_path(&profile); + let directory = checkpoint_directory(&path, true).unwrap(); + let expected = ColdStoreCursorV1 { + after_project_id: Some("local".to_owned()), + after_unregistered_directory: None, + }; + persist_cursor_in_dir(&directory, &expected).unwrap(); + std::fs::rename( + profile.join(CHECKPOINT_DIRECTORY), + profile.join("detached"), + ) + .unwrap(); + let foreign_cursor = foreign.join(CHECKPOINT_FILE); + std::fs::write(&foreign_cursor, br#"{"after_project_id":"foreign"}"#).unwrap(); + symlink(&foreign, profile.join(CHECKPOINT_DIRECTORY)).unwrap(); + + assert_eq!(load_cursor_in_dir(&directory), Some(expected.clone())); + let updated = ColdStoreCursorV1 { + after_project_id: Some("local-next".to_owned()), + after_unregistered_directory: None, + }; + persist_cursor_in_dir(&directory, &updated).unwrap(); + assert_eq!(load_cursor_in_dir(&directory), Some(updated)); + assert_eq!( + std::fs::read(&foreign_cursor).unwrap(), + br#"{"after_project_id":"foreign"}"#, + "a path substitution must not grant the foreign directory write authority" + ); + assert_eq!(std::fs::read_dir(&foreign).unwrap().count(), 1); + assert!(persist_cursor(&path, &expected).is_err()); + assert_eq!(load_cursor(&path), None); + } + #[test] fn malformed_and_oversized_checkpoints_do_not_authorize_unbounded_reads() { let root = tempfile::tempdir().expect("checkpoint root"); From 4158e1bee80d1f08d8c70fcf6badc1ed4f8e441c Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Fri, 6 Nov 2026 19:09:58 -0800 Subject: [PATCH 51/71] fix(maintenance): keep cursor capabilities portable Refs #4787 --- crates/tracedecay-maintenance/src/retention/cold_store.rs | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/crates/tracedecay-maintenance/src/retention/cold_store.rs b/crates/tracedecay-maintenance/src/retention/cold_store.rs index b516850b6d1..45eb652c8c0 100644 --- a/crates/tracedecay-maintenance/src/retention/cold_store.rs +++ b/crates/tracedecay-maintenance/src/retention/cold_store.rs @@ -6,17 +6,17 @@ use std::collections::BTreeSet; use std::ffi::OsStr; use std::io::{self, Read, Write}; -use std::sync::atomic::{AtomicU64, Ordering}; use std::path::{Path, PathBuf}; +use std::sync::atomic::{AtomicU64, Ordering}; use std::time::Instant; -use cap_fs_ext::{FollowSymlinks, OpenOptionsFollowExt}; +use cap_fs_ext::{DirExt, FollowSymlinks, OpenOptionsFollowExt}; use cap_std::fs::{Dir, OpenOptions}; #[cfg(unix)] use cap_std::fs::OpenOptionsExt; use serde::{Deserialize, Serialize}; -use tracedecay_private_fs::capability_dir::{replace_file, sync_directory}; use tracedecay_global_db::RegisteredGlobalDb; +use tracedecay_private_fs::capability_dir::{replace_file, sync_directory}; use tracedecay_runtime_core::cancellation::{CancellationToken, MonotonicDeadline}; use super::{incident_debris, orphan_stores}; @@ -523,7 +523,6 @@ mod tests { assert_eq!(std::fs::read(&foreign_cursor).unwrap(), foreign_bytes); } - #[cfg(unix)] #[test] fn a_replaced_parent_cannot_redirect_an_open_checkpoint_capability() { From fae3278e4c7b3cf98f7fdfbd6d93a01b1114b0bd Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Fri, 6 Nov 2026 19:17:30 -0800 Subject: [PATCH 52/71] style(maintenance): format capability-based cold cursor access --- .../src/retention/cold_store.rs | 18 +++++++----------- 1 file changed, 7 insertions(+), 11 deletions(-) diff --git a/crates/tracedecay-maintenance/src/retention/cold_store.rs b/crates/tracedecay-maintenance/src/retention/cold_store.rs index 45eb652c8c0..b1bac25670a 100644 --- a/crates/tracedecay-maintenance/src/retention/cold_store.rs +++ b/crates/tracedecay-maintenance/src/retention/cold_store.rs @@ -11,9 +11,9 @@ use std::sync::atomic::{AtomicU64, Ordering}; use std::time::Instant; use cap_fs_ext::{DirExt, FollowSymlinks, OpenOptionsFollowExt}; -use cap_std::fs::{Dir, OpenOptions}; #[cfg(unix)] use cap_std::fs::OpenOptionsExt; +use cap_std::fs::{Dir, OpenOptions}; use serde::{Deserialize, Serialize}; use tracedecay_global_db::RegisteredGlobalDb; use tracedecay_private_fs::capability_dir::{replace_file, sync_directory}; @@ -356,12 +356,12 @@ fn persist_cursor_in_dir(directory: &Dir, cursor: &ColdStoreCursorV1) -> io::Res // even a collided or attacker-prepared staging name cannot be truncated. for _ in 0..16 { let nonce = CURSOR_TEMP_NONCE.fetch_add(1, Ordering::Relaxed); - let temporary_name = format!( - ".retention-cold-store-{}-{nonce}.tmp", - std::process::id() - ); + let temporary_name = format!(".retention-cold-store-{}-{nonce}.tmp", std::process::id()); let mut options = OpenOptions::new(); - options.write(true).create_new(true).follow(FollowSymlinks::No); + options + .write(true) + .create_new(true) + .follow(FollowSymlinks::No); #[cfg(unix)] options.mode(0o600); let mut temporary = match directory.open_with(&temporary_name, &options) { @@ -540,11 +540,7 @@ mod tests { after_unregistered_directory: None, }; persist_cursor_in_dir(&directory, &expected).unwrap(); - std::fs::rename( - profile.join(CHECKPOINT_DIRECTORY), - profile.join("detached"), - ) - .unwrap(); + std::fs::rename(profile.join(CHECKPOINT_DIRECTORY), profile.join("detached")).unwrap(); let foreign_cursor = foreign.join(CHECKPOINT_FILE); std::fs::write(&foreign_cursor, br#"{"after_project_id":"foreign"}"#).unwrap(); symlink(&foreign, profile.join(CHECKPOINT_DIRECTORY)).unwrap(); From 481fa497080932d316db9d5e463bc9a7bbe4a482 Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Fri, 6 Nov 2026 20:13:43 -0800 Subject: [PATCH 53/71] fix(graph): preserve unowned sealed cleanup artifacts Refs #4616 --- .../tracedecay-graph-db/src/sealed_store.rs | 95 ++++++++++++++++++- 1 file changed, 91 insertions(+), 4 deletions(-) diff --git a/crates/tracedecay-graph-db/src/sealed_store.rs b/crates/tracedecay-graph-db/src/sealed_store.rs index e41aeda64ed..aac3f242c27 100644 --- a/crates/tracedecay-graph-db/src/sealed_store.rs +++ b/crates/tracedecay-graph-db/src/sealed_store.rs @@ -215,6 +215,15 @@ const SEALED_STORE_RECEIPT_FILE: &str = "sealed.json"; /// Digest recorded only after a successful post-reopen proof. A `sealed.json` /// written before that proof is not release authority. const SEALED_STORE_CHECKED_FILE: &str = "sealed.checked"; +const SEALED_STAGING_PREFIX: &str = ".staging-"; + +/// A seal only creates staging directories for a 64-hex physical generation. +/// Prefix matches alone cannot authorize deleting an unrelated directory. +fn sealed_staging_name(name: &str) -> bool { + name.strip_prefix(SEALED_STAGING_PREFIX).is_some_and(|digest| { + digest.len() == 64 && digest.bytes().all(|byte| byte.is_ascii_hexdigit()) + }) +} const SEALED_STORE_DISABLE_ENV: &str = "TRACEDECAY_GRAPH_SEALED_STORE"; /// The one form a sealed store is built in. Every value TraceDecay persists @@ -601,7 +610,7 @@ fn census_sealed_store_entries( if entry .file_name() .to_str() - .is_some_and(|name| name.starts_with(".staging-")) + .is_some_and(sealed_staging_name) { census.abandoned_staging_count += 1; abandoned.push(path); @@ -911,7 +920,7 @@ mod sealed_census_tests { assert_eq!((census.head_count, census.superseded_count), (1, 2)); assert_eq!( (census.abandoned_staging_count, census.unrecognized_count), - (1, 7) + (0, 8) ); assert_eq!(census.head_bytes, 8192 + head_receipt.len() as u64); assert_eq!( @@ -1186,7 +1195,7 @@ pub(crate) fn sweep_abandoned_sealed_staging(database_path: &Path) { let Some(name) = name.to_str() else { continue; }; - if !name.starts_with(".staging-") { + if !sealed_staging_name(name) || !entry.file_type().is_ok_and(|kind| kind.is_dir()) { continue; } let path = entry.path(); @@ -1214,6 +1223,19 @@ pub(crate) fn sweep_abandoned_sealed_staging(database_path: &Path) { /// Prefix of a batch producer's row spill under the sealed root. const ROW_SPILL_PREFIX: &str = ".rows-"; +/// A row-spill owner writes exactly `.rows--`. Unknown entries +/// sharing this prefix do not belong to the spill collector. +fn row_spill_name(name: &str) -> bool { + name.strip_prefix(ROW_SPILL_PREFIX) + .and_then(|suffix| suffix.split_once('-')) + .is_some_and(|(pid, nonce)| { + !pid.is_empty() + && !nonce.is_empty() + && pid.bytes().all(|byte| byte.is_ascii_digit()) + && nonce.bytes().all(|byte| byte.is_ascii_digit()) + }) +} + /// Removes every row spill another process left under the store's sealed /// root. Run at the eager open, whose exclusive store lock means no other /// process is producing rows for this store. Spills named for this process @@ -1239,7 +1261,8 @@ pub(crate) fn sweep_abandoned_row_spills(database_path: &Path) { if entry .file_name() .to_str() - .is_some_and(|name| name.starts_with(ROW_SPILL_PREFIX) && !name.starts_with(&own)) + .is_some_and(|name| row_spill_name(name) && !name.starts_with(&own)) + && entry.file_type().is_ok_and(|kind| kind.is_dir()) { let path = entry.path(); if let Err(error) = std::fs::remove_dir_all(&path) @@ -1256,6 +1279,70 @@ pub(crate) fn sweep_abandoned_row_spills(database_path: &Path) { } } +#[cfg(all(test, unix))] +mod abandoned_sealed_cleanup_tests { + use super::*; + + #[test] + fn recovery_collects_only_owned_generation_staging_and_numeric_row_spills() { + use std::os::unix::fs::symlink; + + let temp = tempfile::tempdir().unwrap(); + let database = temp.path().join("graph.grafeo"); + let root = sealed_store_root(&database); + std::fs::create_dir(&root).unwrap(); + + let owned_staging = format!("{SEALED_STAGING_PREFIX}{}", "a".repeat(64)); + let foreign_pid = u64::from(std::process::id()) + 1; + let owned_spill = format!("{ROW_SPILL_PREFIX}{foreign_pid}-0"); + let live_spill = format!("{ROW_SPILL_PREFIX}{}-0", std::process::id()); + for name in [&owned_staging, &owned_spill, &live_spill] { + std::fs::create_dir(root.join(name)).unwrap(); + std::fs::write(root.join(name).join("payload"), b"owned").unwrap(); + } + + let unknown = [ + ".staging-pending".to_owned(), + format!("{SEALED_STAGING_PREFIX}{}-extra", "b".repeat(64)), + ".rows-foreign".to_owned(), + ".rows-123-extra".to_owned(), + ".rows-123-0-extra".to_owned(), + ]; + for name in &unknown { + std::fs::create_dir(root.join(name)).unwrap(); + std::fs::write(root.join(name).join("foreign"), b"keep").unwrap(); + } + let outside = temp.path().join("outside"); + std::fs::create_dir(&outside).unwrap(); + std::fs::write(outside.join("foreign"), b"external").unwrap(); + let linked_staging = format!("{SEALED_STAGING_PREFIX}{}", "c".repeat(64)); + symlink(&outside, root.join(&linked_staging)).unwrap(); + let linked_spill = format!("{ROW_SPILL_PREFIX}{foreign_pid}-1"); + symlink(&outside, root.join(&linked_spill)).unwrap(); + + let before = census_sealed_store(&database, &Default::default()).unwrap(); + assert_eq!(before.abandoned_staging_count, 1); + assert!(before.unrecognized_count >= unknown.len() as u64 + 3); + + sweep_abandoned_sealed_staging(&database); + sweep_abandoned_row_spills(&database); + + assert!(!root.join(&owned_staging).exists()); + assert!(!root.join(&owned_spill).exists()); + assert!(root.join(&live_spill).is_dir()); + for name in &unknown { + assert_eq!(std::fs::read(root.join(name).join("foreign")).unwrap(), b"keep"); + } + assert!(std::fs::symlink_metadata(root.join(&linked_staging)) + .unwrap().file_type().is_symlink()); + assert!(std::fs::symlink_metadata(root.join(&linked_spill)) + .unwrap().file_type().is_symlink()); + assert_eq!(std::fs::read(outside.join("foreign")).unwrap(), b"external"); + let after = census_sealed_store(&database, &Default::default()).unwrap(); + assert_eq!(after.abandoned_staging_count, 0); + } +} + fn remove_sealed_directory_checked(directory: &Path) -> Result<(), GraphDbError> { let root = directory .parent() From 9f1686e2254004e07f2c32b43b65ae6c5842b4e7 Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Fri, 6 Nov 2026 21:12:41 -0800 Subject: [PATCH 54/71] fix(graph-db): refuse unrecognized sealed receipts before release Require the shared supported form/base pairing in every receipt binding, including direct sealed serving and staging-row release. Preserve existing compact and layered receipts; reject unknown or inconsistent ones. Refs #4616 --- .../tracedecay-graph-db/src/sealed_store.rs | 79 +++++++++++++++++-- 1 file changed, 71 insertions(+), 8 deletions(-) diff --git a/crates/tracedecay-graph-db/src/sealed_store.rs b/crates/tracedecay-graph-db/src/sealed_store.rs index aac3f242c27..ecf2450650e 100644 --- a/crates/tracedecay-graph-db/src/sealed_store.rs +++ b/crates/tracedecay-graph-db/src/sealed_store.rs @@ -120,14 +120,7 @@ pub(crate) fn open_direct_sealed_generation( }; let receipt: SealedStoreReceiptV1 = serde_json::from_slice(&receipt_bytes) .map_err(|error| GraphDbError::unavailable(format!("sealed receipt decode: {error}")))?; - if receipt.version != SEALED_STORE_RECEIPT_VERSION - || receipt.graph_format != GraphFormatVersion::current().get() - || receipt.recovered_digest != expected.as_str() - || receipt.physical_namespace != physical_namespace.as_str() - || receipt.namespace != projection.namespace.as_str() - || receipt.projection != projection.projection.as_str() - || receipt.generation != generation.as_str() - { + if !receipt.binds(&locator, physical_namespace.as_str(), expected.as_str()) { return Err(GraphDbError::unavailable( "sealed generation store receipt does not bind this generation".to_owned(), )); @@ -275,6 +268,7 @@ impl SealedStoreReceiptV1 { ) -> bool { self.version == SEALED_STORE_RECEIPT_VERSION && self.graph_format == GraphFormatVersion::current().get() + && recognized_sealed_form(self) && self.recovered_digest == expected_digest && self.physical_namespace == physical_namespace && self.namespace == locator.projection.namespace.as_str() @@ -295,6 +289,75 @@ fn recognized_sealed_form(receipt: &SealedStoreReceiptV1) -> bool { ) } +#[cfg(test)] +mod sealed_receipt_binding_tests { + use super::*; + use crate::{GraphGenerationId, GraphProjectionId}; + + #[test] + fn sealed_adoption_and_staging_release_refuse_unknown_or_inconsistent_forms() { + let projection = GraphProjectionIdentity::new( + GraphNamespace::new("receipt-binding").unwrap(), + GraphProjectionId::new("code").unwrap(), + ); + let locator = GenerationLocator::new( + projection.clone(), + GraphGenerationId::new("generation:receipt-binding").unwrap(), + ); + let physical_namespace = locator.physical_namespace().unwrap(); + let mut receipt = SealedStoreReceiptV1 { + version: SEALED_STORE_RECEIPT_VERSION, + form: SEALED_STORE_FORM_COMPACT.to_owned(), + namespace: projection.namespace.as_str().to_owned(), + projection: projection.projection.as_str().to_owned(), + generation: locator.generation.as_str().to_owned(), + physical_namespace: physical_namespace.as_str().to_owned(), + recovered_digest: "verified-digest".to_owned(), + entities: 1, + relations: 0, + graph_format: GraphFormatVersion::current().get(), + row_sum: None, + base: None, + source_generation: None, + watermark: None, + }; + let directory = tempfile::tempdir().unwrap(); + let binds_on_disk = |receipt: &SealedStoreReceiptV1| { + std::fs::write( + directory.path().join(SEALED_STORE_RECEIPT_FILE), + serde_json::to_vec(receipt).unwrap(), + ) + .unwrap(); + load_sealed_store_receipt(directory.path()) + .unwrap() + .unwrap() + .binds(&locator, physical_namespace.as_str(), "verified-digest") + }; + + assert!(binds_on_disk(&receipt), "verified compact receipt is accepted"); + receipt.form = "future-container".to_owned(); + assert!(!binds_on_disk(&receipt), "future form must never bind"); + receipt.form = SEALED_STORE_FORM_LAYERED.to_owned(); + assert!(!binds_on_disk(&receipt), "layered form without base must refuse"); + + receipt.base = Some(SealedBaseReceiptV1 { + generation: "generation:base".to_owned(), + source_generation: "source:base".to_owned(), + watermark: "watermark:base".to_owned(), + physical_namespace: "generation:base".to_owned(), + recovered_digest: "base-digest".to_owned(), + entities: 1, + relations: 0, + row_sum: "row-sum".to_owned(), + hidden_row_sum: "hidden-sum".to_owned(), + }); + receipt.form = SEALED_STORE_FORM_COMPACT.to_owned(); + assert!(!binds_on_disk(&receipt), "compact form with base must refuse"); + receipt.form = SEALED_STORE_FORM_LAYERED.to_owned(); + assert!(binds_on_disk(&receipt), "recognized layered form remains eligible"); + } +} + /// Digest and row counts that authorize a staging-row release without opening /// the sealed engine. pub(crate) struct SealedReleaseEvidence { From b03430dab91198fc5d441b5cc1c766cf1f64d62e Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Fri, 6 Nov 2026 21:18:40 -0800 Subject: [PATCH 55/71] style(graph-db): format sealed-store receipt and cleanup regressions Apply only Rust 2024 formatting transformations reported by the exact-head Repository gates Bazel rustfmt target; preserve all assertions and behavior. --- .../tracedecay-graph-db/src/sealed_store.rs | 54 +++++++++++++------ 1 file changed, 37 insertions(+), 17 deletions(-) diff --git a/crates/tracedecay-graph-db/src/sealed_store.rs b/crates/tracedecay-graph-db/src/sealed_store.rs index ecf2450650e..3f5260d5b0a 100644 --- a/crates/tracedecay-graph-db/src/sealed_store.rs +++ b/crates/tracedecay-graph-db/src/sealed_store.rs @@ -213,9 +213,10 @@ const SEALED_STAGING_PREFIX: &str = ".staging-"; /// A seal only creates staging directories for a 64-hex physical generation. /// Prefix matches alone cannot authorize deleting an unrelated directory. fn sealed_staging_name(name: &str) -> bool { - name.strip_prefix(SEALED_STAGING_PREFIX).is_some_and(|digest| { - digest.len() == 64 && digest.bytes().all(|byte| byte.is_ascii_hexdigit()) - }) + name.strip_prefix(SEALED_STAGING_PREFIX) + .is_some_and(|digest| { + digest.len() == 64 && digest.bytes().all(|byte| byte.is_ascii_hexdigit()) + }) } const SEALED_STORE_DISABLE_ENV: &str = "TRACEDECAY_GRAPH_SEALED_STORE"; @@ -334,11 +335,17 @@ mod sealed_receipt_binding_tests { .binds(&locator, physical_namespace.as_str(), "verified-digest") }; - assert!(binds_on_disk(&receipt), "verified compact receipt is accepted"); + assert!( + binds_on_disk(&receipt), + "verified compact receipt is accepted" + ); receipt.form = "future-container".to_owned(); assert!(!binds_on_disk(&receipt), "future form must never bind"); receipt.form = SEALED_STORE_FORM_LAYERED.to_owned(); - assert!(!binds_on_disk(&receipt), "layered form without base must refuse"); + assert!( + !binds_on_disk(&receipt), + "layered form without base must refuse" + ); receipt.base = Some(SealedBaseReceiptV1 { generation: "generation:base".to_owned(), @@ -352,9 +359,15 @@ mod sealed_receipt_binding_tests { hidden_row_sum: "hidden-sum".to_owned(), }); receipt.form = SEALED_STORE_FORM_COMPACT.to_owned(); - assert!(!binds_on_disk(&receipt), "compact form with base must refuse"); + assert!( + !binds_on_disk(&receipt), + "compact form with base must refuse" + ); receipt.form = SEALED_STORE_FORM_LAYERED.to_owned(); - assert!(binds_on_disk(&receipt), "recognized layered form remains eligible"); + assert!( + binds_on_disk(&receipt), + "recognized layered form remains eligible" + ); } } @@ -670,11 +683,7 @@ fn census_sealed_store_entries( continue; } let path = entry.path(); - if entry - .file_name() - .to_str() - .is_some_and(sealed_staging_name) - { + if entry.file_name().to_str().is_some_and(sealed_staging_name) { census.abandoned_staging_count += 1; abandoned.push(path); continue; @@ -1394,12 +1403,23 @@ mod abandoned_sealed_cleanup_tests { assert!(!root.join(&owned_spill).exists()); assert!(root.join(&live_spill).is_dir()); for name in &unknown { - assert_eq!(std::fs::read(root.join(name).join("foreign")).unwrap(), b"keep"); + assert_eq!( + std::fs::read(root.join(name).join("foreign")).unwrap(), + b"keep" + ); } - assert!(std::fs::symlink_metadata(root.join(&linked_staging)) - .unwrap().file_type().is_symlink()); - assert!(std::fs::symlink_metadata(root.join(&linked_spill)) - .unwrap().file_type().is_symlink()); + assert!( + std::fs::symlink_metadata(root.join(&linked_staging)) + .unwrap() + .file_type() + .is_symlink() + ); + assert!( + std::fs::symlink_metadata(root.join(&linked_spill)) + .unwrap() + .file_type() + .is_symlink() + ); assert_eq!(std::fs::read(outside.join("foreign")).unwrap(), b"external"); let after = census_sealed_store(&database, &Default::default()).unwrap(); assert_eq!(after.abandoned_staging_count, 0); From 8bf96c6ba01fcab61c6da43ca4cef514c9c6177d Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Fri, 6 Nov 2026 23:15:43 -0800 Subject: [PATCH 56/71] fix(graph-db): fence sealed cleanup with held directories Refs #4616 --- .../tracedecay-graph-db/src/sealed_store.rs | 188 ++++++------------ .../src/capability_dir.rs | 154 +++++++++++++- 2 files changed, 209 insertions(+), 133 deletions(-) diff --git a/crates/tracedecay-graph-db/src/sealed_store.rs b/crates/tracedecay-graph-db/src/sealed_store.rs index 3f5260d5b0a..dc48f9e449a 100644 --- a/crates/tracedecay-graph-db/src/sealed_store.rs +++ b/crates/tracedecay-graph-db/src/sealed_store.rs @@ -1020,52 +1020,28 @@ pub(crate) fn discard_superseded_sealed_generations( database_path: &Path, ) -> Result<(), GraphDbError> { let root = sealed_store_root(database_path); - let entries = match sealed_root_entries(&root) { - Ok(Some(entries)) => entries, - Ok(None) => return Ok(()), - Err(error) => return Err(sealed_store_io_failure("sealed root read failed", error)), - }; - for entry in entries { - let entry = - entry.map_err(|error| sealed_store_io_failure("sealed root read failed", error))?; - if !entry - .file_type() - .map_err(|error| sealed_store_io_failure("sealed entry type failed", error))? - .is_dir() - { - continue; - } - let name = entry.file_name(); - let Some(name) = name.to_str() else { - continue; - }; - if name.len() != 64 || !name.bytes().all(|byte| byte.is_ascii_hexdigit()) { - continue; - } - let path = entry.path(); - let Some(receipt) = load_sealed_store_receipt(&path).ok().flatten() else { - continue; - }; - // The old graph format alone is not proof that this build owns the - // artifact: a newer or malformed sealed form must survive a downgrade. - if receipt.version != SEALED_STORE_RECEIPT_VERSION - || receipt.physical_namespace != format!("generation:{name}") - || receipt.graph_format >= GraphFormatVersion::current().get() - || !recognized_sealed_form(&receipt) - { - continue; - } - match std::fs::remove_dir_all(&path) { - Ok(()) => {} - Err(error) if error.kind() == std::io::ErrorKind::NotFound => {} - Err(error) => { - return Err(sealed_store_io_failure( - "superseded sealed generation removal failed", - error, - )); + tracedecay_private_fs::capability_dir::remove_selected_child_directories_nofollow( + &root, + Some((SEALED_STORE_RECEIPT_FILE, MAX_SEALED_STORE_RECEIPT_BYTES as usize)), + |name, receipt_bytes| { + let Some(name) = name.to_str() else { + return false; + }; + if name.len() != 64 || !name.bytes().all(|byte| byte.is_ascii_hexdigit()) { + return false; } - } - } + let Some(receipt) = receipt_bytes + .and_then(|bytes| serde_json::from_slice::(bytes).ok()) + else { + return false; + }; + receipt.version == SEALED_STORE_RECEIPT_VERSION + && receipt.physical_namespace == format!("generation:{name}") + && receipt.graph_format < GraphFormatVersion::current().get() + && recognized_sealed_form(&receipt) + }, + ) + .map_err(|error| sealed_store_io_failure("superseded sealed generation removal failed", error))?; Ok(()) } @@ -1248,47 +1224,24 @@ mod superseded_sealed_cleanup_tests { /// this store is in flight. pub(crate) fn sweep_abandoned_sealed_staging(database_path: &Path) { let root = sealed_store_root(database_path); - let entries = match sealed_root_entries(&root) { - Ok(Some(entries)) => entries, - Ok(None) => return, - Err(error) => { - tracing::warn!( - event = "sealed_staging_sweep_unreadable", - root = %root.display(), - error = %error, - "sealed store root could not be enumerated; abandoned staging stays" - ); - return; - } - }; - let mut removed = 0usize; - for entry in entries.flatten() { - let name = entry.file_name(); - let Some(name) = name.to_str() else { - continue; - }; - if !sealed_staging_name(name) || !entry.file_type().is_ok_and(|kind| kind.is_dir()) { - continue; - } - let path = entry.path(); - match std::fs::remove_dir_all(&path) { - Ok(()) => removed += 1, - Err(error) if error.kind() == std::io::ErrorKind::NotFound => {} - Err(error) => tracing::warn!( - event = "sealed_staging_sweep_failed", - path = %path.display(), - error = %error, - "abandoned sealed staging directory could not be removed" - ), - } - } - if removed > 0 { - tracing::info!( + match tracedecay_private_fs::capability_dir::remove_selected_child_directories_nofollow( + &root, + None, + |name, _| name.to_str().is_some_and(sealed_staging_name), + ) { + Ok(Some(removed)) if removed > 0 => tracing::info!( event = "sealed_staging_swept", root = %root.display(), removed, "removed abandoned sealed staging directories left by interrupted seals" - ); + ), + Ok(_) => {} + Err(error) => tracing::warn!( + event = "sealed_staging_sweep_unreadable", + root = %root.display(), + %error, + "sealed store root could not be safely swept; remaining staging stays" + ), } } @@ -1315,39 +1268,23 @@ fn row_spill_name(name: &str) -> bool { /// whose spill removes itself when it drops. pub(crate) fn sweep_abandoned_row_spills(database_path: &Path) { let root = sealed_store_root(database_path); - let entries = match sealed_root_entries(&root) { - Ok(Some(entries)) => entries, - Ok(None) => return, - Err(error) => { - tracing::warn!( - event = "graph_row_spill_sweep_unreadable", - root = %root.display(), - %error, - "sealed root cannot be safely enumerated" - ); - return; - } - }; let own = format!("{ROW_SPILL_PREFIX}{}-", std::process::id()); - for entry in entries.flatten() { - if entry - .file_name() - .to_str() - .is_some_and(|name| row_spill_name(name) && !name.starts_with(&own)) - && entry.file_type().is_ok_and(|kind| kind.is_dir()) - { - let path = entry.path(); - if let Err(error) = std::fs::remove_dir_all(&path) - && error.kind() != std::io::ErrorKind::NotFound - { - tracing::warn!( - event = "graph_row_spill_sweep_failed", - path = %path.display(), - error = %error, - "abandoned graph row spill could not be removed" - ); - } - } + if let Err(error) = + tracedecay_private_fs::capability_dir::remove_selected_child_directories_nofollow( + &root, + None, + |name, _| { + name.to_str() + .is_some_and(|name| row_spill_name(name) && !name.starts_with(&own)) + }, + ) + { + tracing::warn!( + event = "graph_row_spill_sweep_failed", + root = %root.display(), + %error, + "sealed row spills could not be safely swept" + ); } } @@ -1430,24 +1367,11 @@ fn remove_sealed_directory_checked(directory: &Path) -> Result<(), GraphDbError> let root = directory .parent() .ok_or_else(|| GraphDbError::unavailable("sealed artifact has no parent directory"))?; - match sealed_root_is_real_directory(root) { - Ok(true) => {} - Ok(false) => return Ok(()), - Err(error) => { - return Err(sealed_store_io_failure( - "sealed root validation failed", - error, - )); - } - } - match std::fs::remove_dir_all(directory) { - Ok(()) => Ok(()), - Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(()), - Err(error) => Err(sealed_store_io_failure( - "sealed directory removal failed", - error, - )), - } + let name = directory + .file_name() + .ok_or_else(|| GraphDbError::unavailable("sealed artifact has no directory name"))?; + tracedecay_private_fs::capability_dir::remove_named_child_directory_nofollow(root, name) + .map_err(|error| sealed_store_io_failure("sealed directory removal failed", error)) } /// Staging and invalid-artifact cleanup are best effort. Retiring an accepted diff --git a/crates/tracedecay-private-fs/src/capability_dir.rs b/crates/tracedecay-private-fs/src/capability_dir.rs index c0c846fb8f4..77b2e745e6f 100644 --- a/crates/tracedecay-private-fs/src/capability_dir.rs +++ b/crates/tracedecay-private-fs/src/capability_dir.rs @@ -6,7 +6,10 @@ //! swapped for a symlink cannot redirect the operation. use std::ffi::OsStr; -use std::io; +use std::io::{self, Read}; +use std::path::{Component, Path}; + +use cap_fs_ext::{FollowSymlinks, OpenOptionsFollowExt}; #[cfg(windows)] use cap_fs_ext::DirExt; @@ -212,6 +215,116 @@ pub fn sync_directory(directory: &Dir) -> io::Result<()> { } } +/// Open an owned leaf through a held parent, never following a replaced root +/// symlink. Missing roots are normal before the first artifact is sealed. +fn open_owned_directory_nofollow(root: &Path) -> io::Result> { + let name = root + .file_name() + .ok_or_else(|| io::Error::new(io::ErrorKind::InvalidInput, "root has no leaf name"))?; + let parent_path = root + .parent() + .ok_or_else(|| io::Error::new(io::ErrorKind::InvalidInput, "root has no parent"))?; + let parent = Dir::open_ambient_dir(parent_path, cap_std::ambient_authority())?; + match parent.open_dir_nofollow(name) { + Ok(opened) => Ok(Some(opened)), + Err(error) if error.kind() == io::ErrorKind::NotFound => Ok(None), + Err(error) => Err(error), + } +} + +/// Read an optional bounded regular receipt from an already-open child. +/// A replaced symlink or unreadable receipt never authorizes reclamation. +fn read_child_receipt( + child: &Dir, + receipt: &str, + limit: usize, +) -> io::Result>> { + let mut options = OpenOptions::new(); + options.read(true).follow(FollowSymlinks::No); + let mut file = match child.open_with(receipt, &options) { + Ok(file) => file, + Err(error) if error.kind() == io::ErrorKind::NotFound => return Ok(None), + Err(error) => return Err(error), + }; + if !file.metadata()?.is_file() { + return Ok(None); + } + let mut bytes = Vec::new(); + file.by_ref() + .take((limit as u64).saturating_add(1)) + .read_to_end(&mut bytes)?; + if bytes.len() > limit { + return Ok(None); + } + Ok(Some(bytes)) +} + +/// Enumerate and reclaim only selected real child directories under one held +/// no-follow root. Selection (including optional receipt evidence) and removal +/// refer to the *same opened child*: a root or child swap cannot redirect the +/// deletion into a foreign pathname. An unsafe entry is preserved. +pub fn remove_selected_child_directories_nofollow( + root: &Path, + receipt: Option<(&str, usize)>, + mut eligible: impl FnMut(&OsStr, Option<&[u8]>) -> bool, +) -> io::Result> { + let Some(directory) = open_owned_directory_nofollow(root)? else { + return Ok(None); + }; + let mut removed = 0usize; + for entry in directory.read_dir(".")? { + let entry = entry?; + if !entry.file_type()?.is_dir() { + continue; + } + let name = entry.file_name(); + let child = match directory.open_dir_nofollow(&name) { + Ok(child) => child, + Err(error) if error.kind() == io::ErrorKind::NotFound => continue, + Err(error) => { + tracing::warn!( + event = "private_fs.cleanup_unsafe_child", + root = %root.display(), + child = ?name, + %error, + "unverifiable cleanup child preserved" + ); + continue; + } + }; + let receipt_bytes = match receipt { + Some((file, limit)) => read_child_receipt(&child, file, limit).ok().flatten(), + None => None, + }; + if eligible(&name, receipt_bytes.as_deref()) { + remove_open_dir_all_nofollow(child)?; + removed += 1; + } + } + Ok(Some(removed)) +} + +/// Remove an exact child of the held root. Unlike a path-based recursive +/// removal this never follows a replacement of the root or child symlink. +pub fn remove_named_child_directory_nofollow(root: &Path, name: &OsStr) -> io::Result<()> { + let mut components = Path::new(name).components(); + if !matches!(components.next(), Some(Component::Normal(_))) || components.next().is_some() { + return Err(io::Error::new( + io::ErrorKind::InvalidInput, + "removal requires one child directory name", + )); + } + let Some(directory) = open_owned_directory_nofollow(root)? else { + return Ok(()); + }; + let child = match directory.open_dir_nofollow(name) { + Ok(child) => child, + Err(error) if error.kind() == io::ErrorKind::NotFound => return Ok(()), + Err(error) => return Err(error), + }; + remove_open_dir_all_nofollow(child) +} + /// Removes the held directory without following child symlinks. pub fn remove_open_dir_all_nofollow(directory: Dir) -> io::Result<()> { #[cfg(not(windows))] @@ -267,6 +380,45 @@ mod replacement_tests { assert!(!root.path().join("staged").exists()); } + #[cfg(unix)] + #[test] + fn held_cleanup_does_not_follow_a_root_swap_into_foreign_data() { + use std::os::unix::fs::symlink; + + let temp = tempfile::tempdir().unwrap(); + let root = temp.path().join("graph.sealed"); + let former = temp.path().join("former-sealed"); + let foreign = temp.path().join("foreign"); + std::fs::create_dir(&root).unwrap(); + std::fs::create_dir(&foreign).unwrap(); + for parent in [&root, &foreign] { + std::fs::create_dir(parent.join(".staging-owned")).unwrap(); + std::fs::write(parent.join(".staging-owned/keep"), b"sentinel").unwrap(); + } + let mut swapped = false; + let removed = remove_selected_child_directories_nofollow(&root, None, |name, _| { + if !swapped { + std::fs::rename(&root, &former).unwrap(); + symlink(&foreign, &root).unwrap(); + swapped = true; + } + name == OsStr::new(".staging-owned") + }) + .unwrap(); + assert!(swapped); + assert_eq!(removed, Some(1)); + assert!(!former.join(".staging-owned").exists()); + assert_eq!( + std::fs::read(foreign.join(".staging-owned/keep")).unwrap(), + b"sentinel" + ); + assert!(remove_named_child_directory_nofollow(&root, OsStr::new(".staging-owned")).is_err()); + assert_eq!( + std::fs::read(foreign.join(".staging-owned/keep")).unwrap(), + b"sentinel" + ); + } + #[cfg(windows)] #[test] fn no_replace_publication_preserves_an_occupied_entry() { From f8b645cc4c915ac239dbff682aa2ff0769a194da Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Fri, 6 Nov 2026 23:16:36 -0800 Subject: [PATCH 57/71] fix(private-fs): import nofollow directory extensions Refs #4616 --- crates/tracedecay-graph-db/src/sealed_store.rs | 5 ++++- crates/tracedecay-private-fs/src/capability_dir.rs | 9 ++++----- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/crates/tracedecay-graph-db/src/sealed_store.rs b/crates/tracedecay-graph-db/src/sealed_store.rs index dc48f9e449a..a2479e69902 100644 --- a/crates/tracedecay-graph-db/src/sealed_store.rs +++ b/crates/tracedecay-graph-db/src/sealed_store.rs @@ -1022,7 +1022,10 @@ pub(crate) fn discard_superseded_sealed_generations( let root = sealed_store_root(database_path); tracedecay_private_fs::capability_dir::remove_selected_child_directories_nofollow( &root, - Some((SEALED_STORE_RECEIPT_FILE, MAX_SEALED_STORE_RECEIPT_BYTES as usize)), + Some(( + SEALED_STORE_RECEIPT_FILE, + MAX_SEALED_STORE_RECEIPT_BYTES as usize, + )), |name, receipt_bytes| { let Some(name) = name.to_str() else { return false; diff --git a/crates/tracedecay-private-fs/src/capability_dir.rs b/crates/tracedecay-private-fs/src/capability_dir.rs index 77b2e745e6f..e633e210e0f 100644 --- a/crates/tracedecay-private-fs/src/capability_dir.rs +++ b/crates/tracedecay-private-fs/src/capability_dir.rs @@ -9,10 +9,7 @@ use std::ffi::OsStr; use std::io::{self, Read}; use std::path::{Component, Path}; -use cap_fs_ext::{FollowSymlinks, OpenOptionsFollowExt}; - -#[cfg(windows)] -use cap_fs_ext::DirExt; +use cap_fs_ext::{DirExt, FollowSymlinks, OpenOptionsFollowExt}; #[cfg(not(windows))] use cap_fs_ext::OpenOptionsMaybeDirExt; use cap_std::fs::{Dir, File, OpenOptions}; @@ -412,7 +409,9 @@ mod replacement_tests { std::fs::read(foreign.join(".staging-owned/keep")).unwrap(), b"sentinel" ); - assert!(remove_named_child_directory_nofollow(&root, OsStr::new(".staging-owned")).is_err()); + assert!( + remove_named_child_directory_nofollow(&root, OsStr::new(".staging-owned")).is_err() + ); assert_eq!( std::fs::read(foreign.join(".staging-owned/keep")).unwrap(), b"sentinel" From 29cb82118211ecf3df8c48cec8e3f39474da93dd Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Fri, 6 Nov 2026 23:24:18 -0800 Subject: [PATCH 58/71] style(storage): align nofollow helpers with rustfmt 2024 --- crates/tracedecay-graph-db/src/sealed_store.rs | 4 +++- crates/tracedecay-private-fs/src/capability_dir.rs | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/crates/tracedecay-graph-db/src/sealed_store.rs b/crates/tracedecay-graph-db/src/sealed_store.rs index a2479e69902..820b95f6fba 100644 --- a/crates/tracedecay-graph-db/src/sealed_store.rs +++ b/crates/tracedecay-graph-db/src/sealed_store.rs @@ -1044,7 +1044,9 @@ pub(crate) fn discard_superseded_sealed_generations( && recognized_sealed_form(&receipt) }, ) - .map_err(|error| sealed_store_io_failure("superseded sealed generation removal failed", error))?; + .map_err(|error| { + sealed_store_io_failure("superseded sealed generation removal failed", error) + })?; Ok(()) } diff --git a/crates/tracedecay-private-fs/src/capability_dir.rs b/crates/tracedecay-private-fs/src/capability_dir.rs index e633e210e0f..aa66f6c6008 100644 --- a/crates/tracedecay-private-fs/src/capability_dir.rs +++ b/crates/tracedecay-private-fs/src/capability_dir.rs @@ -9,9 +9,9 @@ use std::ffi::OsStr; use std::io::{self, Read}; use std::path::{Component, Path}; -use cap_fs_ext::{DirExt, FollowSymlinks, OpenOptionsFollowExt}; #[cfg(not(windows))] use cap_fs_ext::OpenOptionsMaybeDirExt; +use cap_fs_ext::{DirExt, FollowSymlinks, OpenOptionsFollowExt}; use cap_std::fs::{Dir, File, OpenOptions}; /// How many times a create-or-open that reports the entry missing is asked From d948b4f2024737ded63ed6b909627aaf98af230a Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Sat, 7 Nov 2026 00:22:26 -0800 Subject: [PATCH 59/71] style(private-fs): format bounded receipt reader --- crates/tracedecay-private-fs/src/capability_dir.rs | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/crates/tracedecay-private-fs/src/capability_dir.rs b/crates/tracedecay-private-fs/src/capability_dir.rs index aa66f6c6008..c233708839c 100644 --- a/crates/tracedecay-private-fs/src/capability_dir.rs +++ b/crates/tracedecay-private-fs/src/capability_dir.rs @@ -231,11 +231,7 @@ fn open_owned_directory_nofollow(root: &Path) -> io::Result> { /// Read an optional bounded regular receipt from an already-open child. /// A replaced symlink or unreadable receipt never authorizes reclamation. -fn read_child_receipt( - child: &Dir, - receipt: &str, - limit: usize, -) -> io::Result>> { +fn read_child_receipt(child: &Dir, receipt: &str, limit: usize) -> io::Result>> { let mut options = OpenOptions::new(); options.read(true).follow(FollowSymlinks::No); let mut file = match child.open_with(receipt, &options) { From da05936dba296cc6bb6ae21bc39a3a69b4f1071b Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Sat, 7 Nov 2026 01:19:30 -0800 Subject: [PATCH 60/71] fix(storage): fence sealed census under held directory handles Refs #4616 Keep identity-bound receipt classification, recursive size accounting, and live-head-first hardlink charging under a held no-follow root. Exercise root-symlink substitution and nested foreign links. --- .../tracedecay-graph-db/src/sealed_store.rs | 214 +++++------------- .../src/capability_dir.rs | 153 +++++++++++++ 2 files changed, 209 insertions(+), 158 deletions(-) diff --git a/crates/tracedecay-graph-db/src/sealed_store.rs b/crates/tracedecay-graph-db/src/sealed_store.rs index 820b95f6fba..b61a4bdfc25 100644 --- a/crates/tracedecay-graph-db/src/sealed_store.rs +++ b/crates/tracedecay-graph-db/src/sealed_store.rs @@ -55,6 +55,7 @@ use grafeo_core::graph::compact::IncrementalCompactStoreBuilder; use grafeo_engine::GrafeoDB; use rayon::prelude::*; use serde::{Deserialize, Serialize}; +use tracedecay_private_fs::capability_dir::census_child_directories_nofollow; use tracedecay_store::runtime::GraphRecoveredGenerationDigestV1; use crate::generation::{ @@ -643,139 +644,68 @@ fn sealed_root_is_real_directory(root: &Path) -> std::io::Result { } } -fn sealed_root_entries(root: &Path) -> std::io::Result> { - if sealed_root_is_real_directory(root)? { - std::fs::read_dir(root).map(Some) - } else { - Ok(None) - } -} - -/// Measure the sealed root beside `database_path` against the full -/// namespace, projection, and generation identities of verified heads. -/// -/// Metadata only: every entry is one receipt read plus a directory size walk, -/// never a container open. An unreadable receipt counts as unrecognized so the -/// census stays truthful when a receipt is mid-write or corrupt. +/// Keep census enumeration, receipt identity and byte measurement beneath one +/// already-open, no-follow sealed root. A path rename or symlink replacement +/// cannot substitute a foreign generation after the census begins. pub fn census_sealed_store( database_path: &Path, head_generations: &std::collections::BTreeSet<(String, String, String)>, ) -> std::io::Result { - let root = sealed_store_root(database_path); - let Some(entries) = sealed_root_entries(&root)? else { - return Ok(SealedStoreCensusV1::default()); - }; - census_sealed_store_entries(entries, head_generations) + census_sealed_store_root(&sealed_store_root(database_path), head_generations) } -fn census_sealed_store_entries( - entries: impl IntoIterator>, +fn census_sealed_store_root( + root: &Path, head_generations: &std::collections::BTreeSet<(String, String, String)>, ) -> std::io::Result { - let mut census = SealedStoreCensusV1::default(); - let mut heads = Vec::new(); - let mut superseded = Vec::new(); - let mut abandoned = Vec::new(); - for entry in entries { - let entry = entry?; - if !entry.file_type()?.is_dir() { - census.unrecognized_count += 1; - continue; - } - let path = entry.path(); - if entry.file_name().to_str().is_some_and(sealed_staging_name) { - census.abandoned_staging_count += 1; - abandoned.push(path); - continue; - } - let receipt = read_sealed_receipt_bytes(&path.join(SEALED_STORE_RECEIPT_FILE)) - .ok() - .flatten() - .and_then(|bytes| serde_json::from_slice::(&bytes).ok()); - let Some(receipt) = receipt else { - census.unrecognized_count += 1; - continue; - }; - let name = entry.file_name(); - let Some(name) = name.to_str() else { - census.unrecognized_count += 1; - continue; - }; - if name.len() != 64 - || !name.bytes().all(|byte| byte.is_ascii_hexdigit()) - || receipt.version != SEALED_STORE_RECEIPT_VERSION - || receipt.physical_namespace != format!("generation:{name}") - || !recognized_sealed_form(&receipt) - { - census.unrecognized_count += 1; - continue; - } - if head_generations.contains(&( - receipt.namespace.clone(), - receipt.projection.clone(), - receipt.generation.clone(), - )) { - census.head_count += 1; - heads.push(path); - } else { - census.superseded_count += 1; - superseded.push(path); - } - } - - // A layered generation can hard-link its base across directories. Charge - // that inode to its live head before any superseded or abandoned alias, - // regardless of read_dir order. Unknown receipts have no charging authority. - let mut counted = std::collections::HashSet::new(); - for path in heads { - census.head_bytes = census - .head_bytes - .saturating_add(directory_bytes(&path, &mut counted)?); - } - for path in superseded { - census.superseded_bytes = census - .superseded_bytes - .saturating_add(directory_bytes(&path, &mut counted)?); - } - for path in abandoned { - census.abandoned_staging_bytes = census - .abandoned_staging_bytes - .saturating_add(directory_bytes(&path, &mut counted)?); - } - Ok(census) -} + const HEAD: usize = 0; + const SUPERSEDED: usize = 1; + const ABANDONED: usize = 2; -/// Bytes under `directory` not already counted in `counted`: a layered -/// generation's base container is a hard link to bytes another directory -/// already holds, so each file is charged once per census. -fn directory_bytes( - directory: &Path, - counted: &mut std::collections::HashSet<(u64, u64)>, -) -> std::io::Result { - let mut total = 0u64; - let mut pending = vec![directory.to_path_buf()]; - while let Some(directory) = pending.pop() { - for entry in std::fs::read_dir(&directory)? { - let entry = entry?; - // Sealed census must never follow a nested link outside its root. - let metadata = std::fs::symlink_metadata(entry.path())?; - if metadata.is_dir() { - pending.push(entry.path()); - continue; + let Some(measured) = census_child_directories_nofollow::<3>( + root, + Some(( + SEALED_STORE_RECEIPT_FILE, + MAX_SEALED_STORE_RECEIPT_BYTES as usize, + )), + |name, receipt_bytes| { + if name.to_str().is_some_and(sealed_staging_name) { + return Some(ABANDONED); } - #[cfg(unix)] + let name = name.to_str()?; + let receipt = receipt_bytes + .and_then(|bytes| serde_json::from_slice::(bytes).ok())?; + if name.len() != 64 + || !name.bytes().all(|byte| byte.is_ascii_hexdigit()) + || receipt.version != SEALED_STORE_RECEIPT_VERSION + || receipt.physical_namespace != format!("generation:{name}") + || !recognized_sealed_form(&receipt) { - use std::os::unix::fs::MetadataExt; - if metadata.nlink() > 1 && !counted.insert((metadata.dev(), metadata.ino())) { - continue; - } + return None; } - #[cfg(not(unix))] - let _ = &counted; - total = total.saturating_add(metadata.len()); - } - } - Ok(total) + if head_generations.contains(&( + receipt.namespace, + receipt.projection, + receipt.generation, + )) { + Some(HEAD) + } else { + Some(SUPERSEDED) + } + }, + )? else { + return Ok(SealedStoreCensusV1::default()); + }; + + Ok(SealedStoreCensusV1 { + head_count: measured.counts[HEAD], + head_bytes: measured.bytes[HEAD], + superseded_count: measured.counts[SUPERSEDED], + superseded_bytes: measured.bytes[SUPERSEDED], + abandoned_staging_count: measured.counts[ABANDONED], + abandoned_staging_bytes: measured.bytes[ABANDONED], + unrecognized_count: measured.unrecognized, + }) } #[cfg(all(test, unix))] @@ -784,7 +714,7 @@ mod sealed_census_tests { use std::fs; use super::{ - SEALED_STORE_RECEIPT_FILE, SEALED_STORE_RECEIPT_VERSION, census_sealed_store_entries, + SEALED_STORE_RECEIPT_FILE, SEALED_STORE_RECEIPT_VERSION, census_sealed_store_root, }; #[test] @@ -951,41 +881,9 @@ mod sealed_census_tests { fs::hard_link(&shared, root.join(directory).join("shared")).unwrap(); } - // Put unknown, staging and retired paths first, reproducing the order - // that previously consumed the live inode before its head was visited. - let ordered = || { - let mut entries = fs::read_dir(root) - .unwrap() - .collect::>>() - .unwrap(); - entries.sort_by_key(|entry| { - let name = entry.file_name(); - let name = name.to_str().unwrap(); - if name == "unrecognized" { - 0 - } else if name == ".staging-pending" { - 1 - } else if name == old { - 2 - } else if name == wrong_namespace - || name == future_version - || name == foreign_projection - || name == unknown_form - || name == missing_layer_base - || name == future_graph_format - || name == zero_graph_format - { - 3 - } else if name == head { - 4 - } else { - unreachable!() - } - }); - entries.into_iter().map(Ok).collect::>() - }; - let census = census_sealed_store_entries( - ordered(), + // Head-first class charging must be independent of readdir ordering. + let census = census_sealed_store_root( + root, &BTreeSet::from([("code".to_owned(), "index".to_owned(), "live".to_owned())]), ) .unwrap(); @@ -1001,7 +899,7 @@ mod sealed_census_tests { ); assert_eq!(census.abandoned_staging_bytes, 0); - let no_head = census_sealed_store_entries(ordered(), &BTreeSet::new()).unwrap(); + let no_head = census_sealed_store_root(root, &BTreeSet::new()).unwrap(); assert_eq!(no_head.head_bytes, 0); assert_eq!( no_head.superseded_bytes, diff --git a/crates/tracedecay-private-fs/src/capability_dir.rs b/crates/tracedecay-private-fs/src/capability_dir.rs index c233708839c..2f9d6e2aeb2 100644 --- a/crates/tracedecay-private-fs/src/capability_dir.rs +++ b/crates/tracedecay-private-fs/src/capability_dir.rs @@ -252,6 +252,95 @@ fn read_child_receipt(child: &Dir, receipt: &str, limit: usize) -> io::Result { + pub counts: [u64; N], + pub bytes: [u64; N], + pub unrecognized: u64, +} + +/// Classify direct children under one held, no-follow root, then charge live +/// classes in caller-specified priority order. A root swap cannot redirect +/// receipt reads or recursive size walks; an unreadable/oversized receipt +/// grants no classification. Substituted unsafe children fail closed. +pub fn census_child_directories_nofollow( + root: &Path, + receipt: Option<(&str, usize)>, + mut classify: impl FnMut(&OsStr, Option<&[u8]>) -> Option, +) -> io::Result>> { + let Some(directory) = open_owned_directory_nofollow(root)? else { + return Ok(None); + }; + let mut classes: [Vec; N] = std::array::from_fn(|_| Vec::new()); + let mut result = DirectoryCensus { + counts: [0; N], + bytes: [0; N], + unrecognized: 0, + }; + for entry in directory.read_dir(".")? { + let entry = entry?; + if !entry.file_type()?.is_dir() { + result.unrecognized = result.unrecognized.saturating_add(1); + continue; + } + let name = entry.file_name(); + let child = directory.open_dir_nofollow(&name)?; + let receipt_bytes = receipt + .and_then(|(file, limit)| read_child_receipt(&child, file, limit).ok().flatten()); + match classify(&name, receipt_bytes.as_deref()) { + Some(index) if index < N => { + result.counts[index] = result.counts[index].saturating_add(1); + classes[index].push(child); + } + _ => result.unrecognized = result.unrecognized.saturating_add(1), + } + } + + // A live layered generation may hard-link files from a superseded base. + // The first class to charge a multiply linked inode owns those bytes. + let mut counted = std::collections::HashSet::new(); + for (index, children) in classes.into_iter().enumerate() { + for child in children { + result.bytes[index] = result.bytes[index] + .saturating_add(directory_bytes_nofollow(child, &mut counted)?); + } + } + Ok(Some(result)) +} + +fn directory_bytes_nofollow( + root: Dir, + counted: &mut std::collections::HashSet<(u64, u64)>, +) -> io::Result { + let mut total = 0_u64; + let mut pending = vec![root]; + while let Some(directory) = pending.pop() { + for entry in directory.read_dir(".")? { + let entry = entry?; + let name = entry.file_name(); + if entry.file_type()?.is_dir() { + pending.push(directory.open_dir_nofollow(&name)?); + continue; + } + let metadata = directory.symlink_metadata(&name)?; + #[cfg(unix)] + { + use cap_fs_ext::MetadataExt; + if metadata.nlink() > 1 && !counted.insert((metadata.dev(), metadata.ino())) { + continue; + } + } + #[cfg(not(unix))] + let _ = &counted; + total = total.saturating_add(metadata.len()); + } + } + Ok(total) +} + /// Enumerate and reclaim only selected real child directories under one held /// no-follow root. Selection (including optional receipt evidence) and removal /// refer to the *same opened child*: a root or child swap cannot redirect the @@ -347,6 +436,70 @@ mod replacement_tests { use super::*; + #[cfg(unix)] + #[test] + fn census_uses_held_root_after_foreign_symlink_swap() { + use std::os::unix::fs::symlink; + + let temp = tempfile::tempdir().unwrap(); + let owned = temp.path().join("owned"); + let moved = temp.path().join("moved"); + let foreign = temp.path().join("foreign"); + std::fs::create_dir_all(owned.join("generation")).unwrap(); + std::fs::create_dir_all(foreign.join("generation")).unwrap(); + std::fs::write(owned.join("generation").join("payload"), vec![1_u8; 41]).unwrap(); + std::fs::write(foreign.join("generation").join("payload"), vec![2_u8; 100_041]) + .unwrap(); + + let mut swapped = false; + let measured = census_child_directories_nofollow::<1>(&owned, None, |name, _| { + assert_eq!(name, OsStr::new("generation")); + assert!(!swapped); + std::fs::rename(&owned, &moved).unwrap(); + symlink(&foreign, &owned).unwrap(); + swapped = true; + Some(0) + }) + .unwrap() + .unwrap(); + assert!(swapped); + assert_eq!(measured.counts, [1]); + assert_eq!(measured.bytes, [41]); + assert_eq!( + std::fs::metadata(foreign.join("generation").join("payload")) + .unwrap() + .len(), + 100_041 + ); + assert_eq!( + std::fs::metadata(moved.join("generation").join("payload")) + .unwrap() + .len(), + 41 + ); + } + + #[cfg(unix)] + #[test] + fn census_never_descends_foreign_symlink_children() { + use std::os::unix::fs::symlink; + + let temp = tempfile::tempdir().unwrap(); + let owned = temp.path().join("owned"); + let foreign = temp.path().join("foreign"); + std::fs::create_dir_all(owned.join("generation")).unwrap(); + std::fs::create_dir_all(&foreign).unwrap(); + std::fs::write(foreign.join("payload"), vec![0_u8; 100_041]).unwrap(); + std::fs::write(owned.join("generation").join("local"), vec![0_u8; 41]).unwrap(); + symlink(&foreign, owned.join("generation").join("linked")).unwrap(); + let measured = census_child_directories_nofollow::<1>(&owned, None, |_, _| Some(0)) + .unwrap() + .unwrap(); + assert_eq!(measured.counts, [1]); + assert!(measured.bytes[0] >= 41); + assert!(measured.bytes[0] < 100_041); + } + #[test] fn replacement_publishes_complete_bytes_and_refuses_parent_traversal() { let root = tempfile::tempdir().unwrap(); From d1e26132b7f08557dc7a398923e164d402db0834 Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Sat, 7 Nov 2026 02:11:40 -0800 Subject: [PATCH 61/71] fix(graph): fence sealed proof markers under held directories Refs #4616 Pin sealed-root and generation handles for proof-marker reads and atomic writes, including no-follow file admission and exclusive staging. Test parent substitution, foreign-file preservation, and local recovery. Repository Bazel and native macOS validation pending. --- .../tracedecay-graph-db/src/sealed_store.rs | 43 ++---- .../src/capability_dir.rs | 143 ++++++++++++++++++ 2 files changed, 154 insertions(+), 32 deletions(-) diff --git a/crates/tracedecay-graph-db/src/sealed_store.rs b/crates/tracedecay-graph-db/src/sealed_store.rs index b61a4bdfc25..88510630223 100644 --- a/crates/tracedecay-graph-db/src/sealed_store.rs +++ b/crates/tracedecay-graph-db/src/sealed_store.rs @@ -2965,11 +2965,14 @@ fn sealed_store_check_matches( directory: &Path, expected_digest: &str, ) -> Result { - let path = directory.join(SEALED_STORE_CHECKED_FILE); let limit = expected_digest .len() .min(MAX_SEALED_STORE_RECEIPT_BYTES as usize); - match tracedecay_private_fs::framed_log::read_bounded(&path, limit) { + match tracedecay_private_fs::capability_dir::read_nested_bounded_file_nofollow( + directory, + SEALED_STORE_CHECKED_FILE, + limit, + ) { Ok(Some(bytes)) => Ok(bytes == expected_digest.as_bytes()), Ok(None) => Ok(false), Err(error) @@ -2988,36 +2991,12 @@ fn persist_sealed_store_check(directory: &Path, expected_digest: &str) -> Result if sealed_store_check_matches(directory, expected_digest)? { return Ok(()); } - // The marker is a re-verification optimization, never authority to write - // through a caller-controlled symlink to data outside the sealed store. - let root = directory - .parent() - .ok_or_else(|| GraphDbError::unavailable("sealed marker has no parent root"))?; - let root_is_real = sealed_root_is_real_directory(root) - .map_err(|error| sealed_store_io_failure("sealed marker root inspect failed", error))?; - let directory_is_real = std::fs::symlink_metadata(directory) - .map(|metadata| metadata.file_type().is_dir()) - .map_err(|error| { - sealed_store_io_failure("sealed marker directory inspect failed", error) - })?; - if !root_is_real || !directory_is_real { - return Err(GraphDbError::unavailable( - "sealed marker directory is not a real directory", - )); - } - let mut staged = tempfile::NamedTempFile::new_in(directory) - .map_err(|error| sealed_store_io_failure("sealed check stage failed", error))?; - staged - .write_all(expected_digest.as_bytes()) - .map_err(|error| sealed_store_io_failure("sealed check write failed", error))?; - staged - .as_file() - .sync_all() - .map_err(|error| sealed_store_io_failure("sealed check sync failed", error))?; - staged - .persist(directory.join(SEALED_STORE_CHECKED_FILE)) - .map_err(|error| sealed_store_io_failure("sealed check persist failed", error.error))?; - Ok(()) + tracedecay_private_fs::capability_dir::replace_nested_file_nofollow( + directory, + SEALED_STORE_CHECKED_FILE, + expected_digest.as_bytes(), + ) + .map_err(|error| sealed_store_io_failure("sealed check persist failed", error)) } #[cfg(test)] diff --git a/crates/tracedecay-private-fs/src/capability_dir.rs b/crates/tracedecay-private-fs/src/capability_dir.rs index 2f9d6e2aeb2..af5023c27ef 100644 --- a/crates/tracedecay-private-fs/src/capability_dir.rs +++ b/crates/tracedecay-private-fs/src/capability_dir.rs @@ -8,6 +8,10 @@ use std::ffi::OsStr; use std::io::{self, Read}; use std::path::{Component, Path}; +use std::sync::atomic::{AtomicU64, Ordering}; + +#[cfg(unix)] +use cap_std::fs::OpenOptionsExt; #[cfg(not(windows))] use cap_fs_ext::OpenOptionsMaybeDirExt; @@ -252,6 +256,103 @@ fn read_child_receipt(child: &Dir, receipt: &str, limit: usize) -> io::Result io::Result { + let root = directory + .parent() + .ok_or_else(|| io::Error::new(io::ErrorKind::InvalidInput, "directory has no root"))?; + let name = directory + .file_name() + .ok_or_else(|| io::Error::new(io::ErrorKind::InvalidInput, "directory has no name"))?; + let root = open_owned_directory_nofollow(root)? + .ok_or_else(|| io::Error::from(io::ErrorKind::NotFound))?; + root.open_dir_nofollow(name) +} + +fn checked_leaf_name(name: &str) -> io::Result<&OsStr> { + let mut components = Path::new(name).components(); + if !matches!(components.next(), Some(Component::Normal(_))) || components.next().is_some() { + return Err(io::Error::new( + io::ErrorKind::InvalidInput, + "proof file name must be one leaf", + )); + } + Ok(OsStr::new(name)) +} + +/// Read bounded regular proof bytes beneath a held root and generation. +/// Missing files and static symlinks never provide reusable proof. +pub fn read_nested_bounded_file_nofollow( + directory: &Path, + name: &str, + limit: usize, +) -> io::Result>> { + checked_leaf_name(name)?; + let child = match open_nested_directory_nofollow(directory) { + Ok(child) => child, + Err(error) if error.kind() == io::ErrorKind::NotFound => return Ok(None), + Err(error) => return Err(error), + }; + match child.symlink_metadata(name) { + Ok(metadata) if !metadata.is_file() => return Ok(None), + Ok(_) => {} + Err(error) if error.kind() == io::ErrorKind::NotFound => return Ok(None), + Err(error) => return Err(error), + } + read_child_receipt(&child, name, limit) +} + +static PROOF_TEMP_NONCE: AtomicU64 = AtomicU64::new(0); + +/// Publish a proof atomically relative to the same held generation used for +/// staging. A substituted root/child link cannot redirect or truncate bytes. +pub fn replace_nested_file_nofollow( + directory: &Path, + name: &str, + bytes: &[u8], +) -> io::Result<()> { + let leaf = checked_leaf_name(name)?; + let child = open_nested_directory_nofollow(directory)?; + replace_file_in_open_directory(&child, leaf, bytes) +} + +fn replace_file_in_open_directory(directory: &Dir, leaf: &OsStr, bytes: &[u8]) -> io::Result<()> { + use std::io::Write; + + for _ in 0..16 { + let nonce = PROOF_TEMP_NONCE.fetch_add(1, Ordering::Relaxed); + let temporary_name = format!(".sealed-proof-{}-{nonce}.tmp", std::process::id()); + let mut options = OpenOptions::new(); + options + .write(true) + .create_new(true) + .follow(FollowSymlinks::No); + #[cfg(unix)] + options.mode(0o600); + let mut temporary = match directory.open_with(&temporary_name, &options) { + Ok(file) => file, + Err(error) if error.kind() == io::ErrorKind::AlreadyExists => continue, + Err(error) => return Err(error), + }; + let result = (|| { + temporary.write_all(bytes)?; + temporary.sync_all()?; + drop(temporary); + replace_file(directory, OsStr::new(&temporary_name), leaf)?; + sync_directory(directory) + })(); + if result.is_err() { + let _ = directory.remove_file(&temporary_name); + } + return result; + } + Err(io::Error::new( + io::ErrorKind::AlreadyExists, + "sealed proof staging names are occupied", + )) +} + /// Capability-relative census classes. The caller decides which /// bounded receipt identities are recognizable, while private-fs owns all /// directory traversal, metadata and hardlink accounting. @@ -500,6 +601,48 @@ mod replacement_tests { assert!(measured.bytes[0] < 100_041); } + #[cfg(unix)] + #[test] + fn nested_proof_stays_with_open_generation_when_root_is_swapped() { + use std::os::unix::fs::symlink; + + let temp = tempfile::tempdir().unwrap(); + let root = temp.path().join("graph.sealed"); + let moved = temp.path().join("former-sealed"); + let foreign = temp.path().join("foreign"); + let generation = root.join("generation"); + std::fs::create_dir_all(&generation).unwrap(); + std::fs::create_dir_all(foreign.join("generation")).unwrap(); + std::fs::write(foreign.join("generation/sealed.checked"), b"foreign").unwrap(); + + let held = open_nested_directory_nofollow(&generation).unwrap(); + std::fs::rename(&root, &moved).unwrap(); + symlink(&foreign, &root).unwrap(); + replace_file_in_open_directory(&held, OsStr::new("sealed.checked"), b"verified").unwrap(); + assert_eq!( + std::fs::read(moved.join("generation/sealed.checked")).unwrap(), + b"verified" + ); + assert_eq!( + std::fs::read(foreign.join("generation/sealed.checked")).unwrap(), + b"foreign" + ); + assert!(replace_nested_file_nofollow(&generation, "sealed.checked", b"unsafe").is_err()); + assert!(read_nested_bounded_file_nofollow(&generation, "sealed.checked", 64).is_err()); + + std::fs::remove_file(&root).unwrap(); + std::fs::rename(&moved, &root).unwrap(); + replace_nested_file_nofollow(&generation, "sealed.checked", b"fresh").unwrap(); + assert_eq!( + read_nested_bounded_file_nofollow(&generation, "sealed.checked", 64).unwrap(), + Some(b"fresh".to_vec()) + ); + assert_eq!( + std::fs::read(foreign.join("generation/sealed.checked")).unwrap(), + b"foreign" + ); + } + #[test] fn replacement_publishes_complete_bytes_and_refuses_parent_traversal() { let root = tempfile::tempdir().unwrap(); From 549f161c433d844519f099be58b8003b9549bc9e Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Sat, 7 Nov 2026 02:17:52 -0800 Subject: [PATCH 62/71] fix: handle absent sealed census ancestors and format capability scans Treat a missing sealed-root parent as an absent census rather than returning ENOENT. The existing sealed_census_separates_heads_from_superseded_and_abandoned_bytes Bazel regression exercises this missing-ancestor path. Apply only the Rust 2024 formatting diffs reported by exact-head Repository gates. --- .../tracedecay-graph-db/src/sealed_store.rs | 3 ++- .../src/capability_dir.rs | 23 +++++++++++-------- 2 files changed, 15 insertions(+), 11 deletions(-) diff --git a/crates/tracedecay-graph-db/src/sealed_store.rs b/crates/tracedecay-graph-db/src/sealed_store.rs index 88510630223..9bc6d80f9a0 100644 --- a/crates/tracedecay-graph-db/src/sealed_store.rs +++ b/crates/tracedecay-graph-db/src/sealed_store.rs @@ -693,7 +693,8 @@ fn census_sealed_store_root( Some(SUPERSEDED) } }, - )? else { + )? + else { return Ok(SealedStoreCensusV1::default()); }; diff --git a/crates/tracedecay-private-fs/src/capability_dir.rs b/crates/tracedecay-private-fs/src/capability_dir.rs index af5023c27ef..ccf257eccd1 100644 --- a/crates/tracedecay-private-fs/src/capability_dir.rs +++ b/crates/tracedecay-private-fs/src/capability_dir.rs @@ -225,7 +225,11 @@ fn open_owned_directory_nofollow(root: &Path) -> io::Result> { let parent_path = root .parent() .ok_or_else(|| io::Error::new(io::ErrorKind::InvalidInput, "root has no parent"))?; - let parent = Dir::open_ambient_dir(parent_path, cap_std::ambient_authority())?; + let parent = match Dir::open_ambient_dir(parent_path, cap_std::ambient_authority()) { + Ok(parent) => parent, + Err(error) if error.kind() == io::ErrorKind::NotFound => return Ok(None), + Err(error) => return Err(error), + }; match parent.open_dir_nofollow(name) { Ok(opened) => Ok(Some(opened)), Err(error) if error.kind() == io::ErrorKind::NotFound => Ok(None), @@ -307,11 +311,7 @@ static PROOF_TEMP_NONCE: AtomicU64 = AtomicU64::new(0); /// Publish a proof atomically relative to the same held generation used for /// staging. A substituted root/child link cannot redirect or truncate bytes. -pub fn replace_nested_file_nofollow( - directory: &Path, - name: &str, - bytes: &[u8], -) -> io::Result<()> { +pub fn replace_nested_file_nofollow(directory: &Path, name: &str, bytes: &[u8]) -> io::Result<()> { let leaf = checked_leaf_name(name)?; let child = open_nested_directory_nofollow(directory)?; replace_file_in_open_directory(&child, leaf, bytes) @@ -405,8 +405,8 @@ pub fn census_child_directories_nofollow( let mut counted = std::collections::HashSet::new(); for (index, children) in classes.into_iter().enumerate() { for child in children { - result.bytes[index] = result.bytes[index] - .saturating_add(directory_bytes_nofollow(child, &mut counted)?); + result.bytes[index] = + result.bytes[index].saturating_add(directory_bytes_nofollow(child, &mut counted)?); } } Ok(Some(result)) @@ -549,8 +549,11 @@ mod replacement_tests { std::fs::create_dir_all(owned.join("generation")).unwrap(); std::fs::create_dir_all(foreign.join("generation")).unwrap(); std::fs::write(owned.join("generation").join("payload"), vec![1_u8; 41]).unwrap(); - std::fs::write(foreign.join("generation").join("payload"), vec![2_u8; 100_041]) - .unwrap(); + std::fs::write( + foreign.join("generation").join("payload"), + vec![2_u8; 100_041], + ) + .unwrap(); let mut swapped = false; let measured = census_child_directories_nofollow::<1>(&owned, None, |name, _| { From 0566171ba967173809cc52c4e8cb42d0f43b0d64 Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Sat, 7 Nov 2026 03:23:42 -0800 Subject: [PATCH 63/71] fix(graph-db): remove stale sealed census code after handle cutover Drop the no-longer-used symlink metadata helper and Write import after census moved to capability directory handles. This resolves both exact-head Bazel Clippy errors without weakening the test suite or census behavior. --- crates/tracedecay-graph-db/src/sealed_store.rs | 15 --------------- 1 file changed, 15 deletions(-) diff --git a/crates/tracedecay-graph-db/src/sealed_store.rs b/crates/tracedecay-graph-db/src/sealed_store.rs index 9bc6d80f9a0..9daec8796ca 100644 --- a/crates/tracedecay-graph-db/src/sealed_store.rs +++ b/crates/tracedecay-graph-db/src/sealed_store.rs @@ -44,7 +44,6 @@ //! digest before the store serves a read. use std::collections::{BTreeMap, HashMap}; -use std::io::Write; use std::panic::{AssertUnwindSafe, catch_unwind}; use std::path::{Path, PathBuf}; use std::sync::Arc; @@ -630,20 +629,6 @@ pub struct SealedStoreCensusV1 { pub unrecognized_count: u64, } -// A symlinked sealed root must not redirect cleanup or accounting into -// unrelated data. A missing root is normal before the first seal. -fn sealed_root_is_real_directory(root: &Path) -> std::io::Result { - match std::fs::symlink_metadata(root) { - Ok(metadata) if metadata.is_dir() => Ok(true), - Ok(_) => Err(std::io::Error::new( - std::io::ErrorKind::InvalidData, - "sealed store root is not a real directory", - )), - Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(false), - Err(error) => Err(error), - } -} - /// Keep census enumeration, receipt identity and byte measurement beneath one /// already-open, no-follow sealed root. A path rename or symlink replacement /// cannot substitute a foreign generation after the census begins. From 2f0f1ef63290afa135c89fc3a22e00880f1d2c6c Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Sat, 7 Nov 2026 13:11:55 -0800 Subject: [PATCH 64/71] fix(retention): advance cold-store cursor across filtered projects Page by the registry cursor rather than inferred store census entries so projects without profile-sharded stores cannot pin a maintenance sweep. Add a database-backed two-page regression for nine projects and zero stores. Refs #4787 --- .../src/retention/cold_store.rs | 95 ++++++++++++------- 1 file changed, 60 insertions(+), 35 deletions(-) diff --git a/crates/tracedecay-maintenance/src/retention/cold_store.rs b/crates/tracedecay-maintenance/src/retention/cold_store.rs index b1bac25670a..47221fa9a44 100644 --- a/crates/tracedecay-maintenance/src/retention/cold_store.rs +++ b/crates/tracedecay-maintenance/src/retention/cold_store.rs @@ -198,17 +198,11 @@ pub async fn run_cold_store_page( orphan_stores::UnregisteredSweepCompletionV1::DeadlineExceeded => {} } } - let project_ids = page - .entries - .iter() - .map(|entry| entry.project_id.clone()) - .collect::>(); + // The page boundary belongs to the project registry, not the filtered + // profile-sharded store census. Projects with no matching stores still + // consume a registry page and must advance its durable resume key. let next_cursor = ColdStoreCursorV1 { - after_project_id: next_cold_store_cursor( - cursor.after_project_id.as_deref(), - &project_ids, - page.next_cursor.is_some(), - ), + after_project_id: page.next_cursor, after_unregistered_directory, }; persist_cursor(&checkpoint_path, &next_cursor).map_err(|error| { @@ -219,20 +213,6 @@ pub async fn run_cold_store_page( Ok(report) } -fn next_cold_store_cursor( - previous: Option<&str>, - project_ids: &[String], - has_more: bool, -) -> Option { - if !has_more { - return None; - } - project_ids - .last() - .cloned() - .or_else(|| previous.map(str::to_owned)) -} - fn classify_cold_store_state( cancelled: bool, manifest_readable: bool, @@ -402,18 +382,63 @@ fn retention_window_secs(days: u64) -> i64 { mod tests { use super::*; - #[test] - fn cursor_resumes_after_the_last_complete_project() { - let first = next_cold_store_cursor( - None, - &["project-a".to_owned(), "project-b".to_owned()], - true, - ) - .expect("first page cursor"); - assert_eq!(first, "project-b"); + #[asupersync::test] + async fn cold_store_cursor_advances_across_projects_without_profile_stores() { + use tracedecay_global_db::tests::harness::RegisteredGlobalDbTestRuntime; + + let temporary = tempfile::tempdir().unwrap(); + let profile = temporary.path().join("profile"); + std::fs::create_dir(&profile).unwrap(); + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + std::fs::set_permissions(&profile, std::fs::Permissions::from_mode(0o700)).unwrap(); + } + let runtime = RegisteredGlobalDbTestRuntime::profile(&profile) + .await + .unwrap(); + let database = runtime.profile_database_arc(); + let transaction = database.begin_write_transaction().await.unwrap(); + for index in 0..=COLD_STORE_PAGE_LIMIT { + let project_id = format!("project-{index:02}"); + let root = temporary.path().join(&project_id).to_string_lossy().into_owned(); + transaction + .execute( + "INSERT INTO code_projects ( + project_id, canonical_root, display_root, created_at, last_seen_at + ) VALUES (?1, ?2, ?2, 1, 1)", + tracedecay_runtime_core::db::engine::params![project_id, root], + ) + .await + .unwrap(); + } + transaction.commit().await.unwrap(); + + let cancellation = CancellationToken::new(); + let owners = BTreeSet::new(); + let first = run_cold_store_page(&profile, database.as_ref(), None, &owners, &cancellation) + .await + .unwrap(); + assert_eq!(first.processed_stores, 0); assert_eq!( - next_cold_store_cursor(Some(&first), &["project-c".to_owned()], false), - None + load_cursor(&checkpoint_path(&profile)) + .unwrap() + .after_project_id + .as_deref(), + Some("project-07"), + "the registry's page boundary must persist even when its census is empty" + ); + + let second = run_cold_store_page(&profile, database.as_ref(), None, &owners, &cancellation) + .await + .unwrap(); + assert_eq!(second.processed_stores, 0); + assert_eq!( + load_cursor(&checkpoint_path(&profile)) + .unwrap() + .after_project_id, + None, + "the final registry page must finish instead of repeating its first page" ); } From 453a19b2e644b0bcdc15ff40739002e7849bf111 Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Sat, 7 Nov 2026 13:19:17 -0800 Subject: [PATCH 65/71] style(retention): format cold-store cursor regression --- crates/tracedecay-maintenance/src/retention/cold_store.rs | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/crates/tracedecay-maintenance/src/retention/cold_store.rs b/crates/tracedecay-maintenance/src/retention/cold_store.rs index 47221fa9a44..2cc74daef9c 100644 --- a/crates/tracedecay-maintenance/src/retention/cold_store.rs +++ b/crates/tracedecay-maintenance/src/retention/cold_store.rs @@ -401,7 +401,11 @@ mod tests { let transaction = database.begin_write_transaction().await.unwrap(); for index in 0..=COLD_STORE_PAGE_LIMIT { let project_id = format!("project-{index:02}"); - let root = temporary.path().join(&project_id).to_string_lossy().into_owned(); + let root = temporary + .path() + .join(&project_id) + .to_string_lossy() + .into_owned(); transaction .execute( "INSERT INTO code_projects ( From 04a4aa8e3f83a2da303d6cd09bb44538a2032825 Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Sat, 7 Nov 2026 16:18:54 -0800 Subject: [PATCH 66/71] fix(projection): avoid connection-lifetime TEMP ownership sets Use indexed ownership probes instead of two persistent TEMP tables during projection rebuild activation; retain foreign and staged output protections. Add a same-connection TEMP-schema regression. Refs #4152. Standalone SQLite comparison passed six ownership scenarios. Rust Bazel, native macOS and hosted exact-head CI are not yet verified. --- .../src/observation_projection/rebuild.rs | 114 ++++++++---------- 1 file changed, 53 insertions(+), 61 deletions(-) diff --git a/crates/tracedecay-global-db/src/observation_projection/rebuild.rs b/crates/tracedecay-global-db/src/observation_projection/rebuild.rs index e32499a11b5..bb61d39ead5 100644 --- a/crates/tracedecay-global-db/src/observation_projection/rebuild.rs +++ b/crates/tracedecay-global-db/src/observation_projection/rebuild.rs @@ -2027,78 +2027,44 @@ async fn clear_active_projection( conn: &impl Executor, generation: &str, ) -> ProjectionStoreResult<()> { - conn.execute_batch( - "CREATE TEMP TABLE IF NOT EXISTS observation_projection_rebuild_retained_outputs ( - output_provider TEXT NOT NULL, - output_message_id TEXT NOT NULL, - PRIMARY KEY(output_provider, output_message_id) - ) WITHOUT ROWID; - CREATE TEMP TABLE IF NOT EXISTS observation_projection_rebuild_cleared_outputs ( - output_provider TEXT NOT NULL, - output_message_id TEXT NOT NULL, - PRIMARY KEY(output_provider, output_message_id) - ) WITHOUT ROWID; - DELETE FROM temp.observation_projection_rebuild_retained_outputs; - DELETE FROM temp.observation_projection_rebuild_cleared_outputs;", - ) - .await - .map_err(|error| storage("prepare projection rebuild activation", error))?; + // Projector ownership is indexed by output identity. Query that authority + // directly instead of materializing two large TEMP tables on the long-lived + // SQLite writer; their unlinked temp files can survive DELETE and DROP. conn.execute( - "INSERT INTO temp.observation_projection_rebuild_retained_outputs ( - output_provider, output_message_id - ) - SELECT DISTINCT current.output_provider, current.output_message_id - FROM observation_projection_provenance AS current - WHERE current.projector_version = ?1 - AND EXISTS ( - SELECT 1 FROM observation_projection_provenance AS retained - WHERE retained.output_provider = current.output_provider - AND retained.output_message_id = current.output_message_id - AND retained.projector_version <> current.projector_version - )", - params![SESSION_MESSAGE_PROJECTOR_VERSION], - ) - .await - .map_err(|error| storage("materialize retained projection outputs", error))?; - conn.execute( - "INSERT INTO temp.observation_projection_rebuild_cleared_outputs ( - output_provider, output_message_id + "DELETE FROM lcm_raw_messages + WHERE EXISTS ( + SELECT 1 FROM observation_projection_provenance AS current + WHERE current.projector_version = ?1 + AND current.message_created = 1 + AND current.output_provider = lcm_raw_messages.provider + AND current.output_message_id = lcm_raw_messages.message_id ) - SELECT DISTINCT provenance.output_provider, provenance.output_message_id - FROM observation_projection_provenance AS provenance - WHERE provenance.projector_version = ?1 AND provenance.message_created = 1 AND NOT EXISTS ( - SELECT 1 FROM temp.observation_projection_rebuild_retained_outputs AS retained - WHERE retained.output_provider = provenance.output_provider - AND retained.output_message_id = provenance.output_message_id - ) + SELECT 1 FROM observation_projection_provenance AS foreign_owner + WHERE foreign_owner.projector_version <> ?1 + AND foreign_owner.output_provider = lcm_raw_messages.provider + AND foreign_owner.output_message_id = lcm_raw_messages.message_id + ) AND NOT EXISTS ( SELECT 1 FROM observation_projection_rebuild_messages AS staged WHERE staged.projector_version = ?1 AND staged.generation = ?2 - AND staged.output_provider = provenance.output_provider - AND staged.output_message_id = provenance.output_message_id - )", - params![SESSION_MESSAGE_PROJECTOR_VERSION, generation], - ) - .await - .map_err(|error| storage("materialize cleared projection outputs", error))?; - conn.execute( - "DELETE FROM lcm_raw_messages - WHERE EXISTS ( - SELECT 1 FROM temp.observation_projection_rebuild_cleared_outputs AS cleared - WHERE cleared.output_provider = lcm_raw_messages.provider - AND cleared.output_message_id = lcm_raw_messages.message_id + AND staged.output_provider = lcm_raw_messages.provider + AND staged.output_message_id = lcm_raw_messages.message_id )", - (), + params![SESSION_MESSAGE_PROJECTOR_VERSION, generation], ) .await .map_err(|error| storage("clear projected message rows for rebuild", error))?; + // Foreign projector versions are never deleted. The predicate reads only + // those other versions, which remain stable while this version is cleared. conn.execute( "DELETE FROM observation_projection_provenance - WHERE projector_version = ?1 AND NOT EXISTS ( - SELECT 1 FROM temp.observation_projection_rebuild_retained_outputs AS retained - WHERE retained.output_provider = observation_projection_provenance.output_provider - AND retained.output_message_id = observation_projection_provenance.output_message_id + WHERE projector_version = ?1 + AND NOT EXISTS ( + SELECT 1 FROM observation_projection_provenance AS foreign_owner + WHERE foreign_owner.projector_version <> ?1 + AND foreign_owner.output_provider = observation_projection_provenance.output_provider + AND foreign_owner.output_message_id = observation_projection_provenance.output_message_id )", params![SESSION_MESSAGE_PROJECTOR_VERSION], ) @@ -2609,7 +2575,7 @@ mod activation_tests { use super::{ REBUILD_PAGE_SIZE, SESSION_MESSAGE_PROJECTOR_VERSION, activate_projection_rebuild_transaction, activate_rebuild_sessions, - reconcile_overlapping_rebuild_sessions, write_staged_message, + clear_active_projection, reconcile_overlapping_rebuild_sessions, write_staged_message, }; use crate::tests::harness::RegisteredGlobalDbHarness; use tracedecay_runtime_core::db::engine::{Executor, params}; @@ -2733,6 +2699,32 @@ mod activation_tests { ); } + #[asupersync::test] + async fn clearing_projection_does_not_retain_temp_output_tables() { + let harness = RegisteredGlobalDbHarness::open("projection-clear-no-temp-cache").await; + let transaction = harness.registered.begin_write_transaction().await.unwrap(); + + clear_active_projection(&transaction, "generation.no-temp") + .await + .unwrap(); + + let mut rows = transaction + .query( + "SELECT name FROM sqlite_temp_master + WHERE name IN ( + 'observation_projection_rebuild_retained_outputs', + 'observation_projection_rebuild_cleared_outputs' + ) LIMIT 1", + (), + ) + .await + .unwrap(); + assert!( + rows.next().await.unwrap().is_none(), + "clearing projector ownership must not leave output TEMP tables on the writer" + ); + } + #[asupersync::test] async fn activation_refuses_an_unowned_message_and_rolls_back() { const GENERATION: &str = "generation.foreign-output"; From f7b605e2be428ea6e82b88c36adabbca6c065f2b Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Sat, 7 Nov 2026 16:19:31 -0800 Subject: [PATCH 67/71] test(retention): cover multi-generation sealed replay retirement Refs #4616. Exercise five durable sealed publications and four exact retirements while preserving the latest live serving snapshot. --- .../sealed_store.rs | 69 +++++++++++++++++++ 1 file changed, 69 insertions(+) diff --git a/crates/tracedecay-graph-db/tests/graph_db_suite/verified_generation_contract/sealed_store.rs b/crates/tracedecay-graph-db/tests/graph_db_suite/verified_generation_contract/sealed_store.rs index 2d91adfcc9e..e68178bb760 100644 --- a/crates/tracedecay-graph-db/tests/graph_db_suite/verified_generation_contract/sealed_store.rs +++ b/crates/tracedecay-graph-db/tests/graph_db_suite/verified_generation_contract/sealed_store.rs @@ -1553,6 +1553,75 @@ fn retirement_deletes_the_superseded_sealed_artifact() { ); } +/// Repeated retention must converge after more than three sealed publications. +/// Older code generations become collectable without disturbing the latest +/// verified head or its still-live serving snapshot. +#[test] +fn replay_retirement_reclaims_four_sealed_generations_and_preserves_live_head() { + let temp = TempDir::new().unwrap(); + let registered = RegisteredGraph::new_mounted(temp.path()).unwrap(); + let mut authority = RelationalAuthority::default(); + let identity = projection("sealed-store:retire-many", "code"); + let mut previous_head = None; + let mut latest = None; + + for sequence in 0..5_u8 { + let generation = format!("retire-many-g{sequence}"); + let marker = format!("marker-{sequence}"); + let manifest = rich_manifest(identity.clone(), &generation, &marker); + let input = char::from(b'1' + sequence); + let record = stage_sealed_manifest( + &mut authority, + ®istered.binding, + &manifest, + &format!("publish:retire-many-{sequence}"), + previous_head, + input, + ); + let committed = publish_sealed(®istered, temp.path(), &mut authority, &record, &manifest); + previous_head = Some(committed.head.clone()); + if sequence == 4 { + assert_snapshot_reads(&committed.snapshot, &identity, &marker); + latest = Some(committed); + } + } + + assert_eq!(sealed_receipts(temp.path()).len(), 5); + let (control, probe) = control_and_probe(); + let context = GraphPublicationOperationContextV1::new(&control, &probe).unwrap(); + for sequence in 0..4_u8 { + let generation = format!("retire-many-g{sequence}"); + let input = char::from(b'1' + sequence); + let source_generation = + CodeGenerationId::new(format!("code-generation.{generation}")).unwrap(); + let source_digest = + SealedGraphStateDigest::try_from(format!("sha256:{}", input.to_string().repeat(64))) + .unwrap(); + assert!(matches!( + registered + .registry + .retire_one_code_generation_replay( + registration(registered.binding.clone(), temp.path()), + &mut authority, + &context, + &source_generation, + &source_digest, + ) + .unwrap(), + GraphReplayCollectionOutcome::Retired(_) + )); + assert!( + receipt_for_generation(temp.path(), &generation).is_none(), + "retired sealed artifact survived: {generation}" + ); + } + + let live = latest.expect("the last publication retains a live snapshot"); + assert_snapshot_reads(&live.snapshot, &identity, "marker-4"); + assert_eq!(sealed_receipts(temp.path()).len(), 1); + assert!(receipt_for_generation(temp.path(), "retire-many-g4").is_some()); +} + /// A live direct-sealed reader (recovered through /// `recover_verified_sealed_snapshot`, which bypasses the staging database's /// verified-generation state) must gate retirement of its generation exactly From c9ef27635e484111b77411a563cef44cc2eb0c19 Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Sat, 7 Nov 2026 16:26:47 -0800 Subject: [PATCH 68/71] style(graph-db): format sealed replay retirement test --- .../verified_generation_contract/sealed_store.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/crates/tracedecay-graph-db/tests/graph_db_suite/verified_generation_contract/sealed_store.rs b/crates/tracedecay-graph-db/tests/graph_db_suite/verified_generation_contract/sealed_store.rs index e68178bb760..2491f99b9b0 100644 --- a/crates/tracedecay-graph-db/tests/graph_db_suite/verified_generation_contract/sealed_store.rs +++ b/crates/tracedecay-graph-db/tests/graph_db_suite/verified_generation_contract/sealed_store.rs @@ -1578,7 +1578,8 @@ fn replay_retirement_reclaims_four_sealed_generations_and_preserves_live_head() previous_head, input, ); - let committed = publish_sealed(®istered, temp.path(), &mut authority, &record, &manifest); + let committed = + publish_sealed(®istered, temp.path(), &mut authority, &record, &manifest); previous_head = Some(committed.head.clone()); if sequence == 4 { assert_snapshot_reads(&committed.snapshot, &identity, &marker); From 805381956232d674b9594ce176975ddf0dee36f3 Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Sun, 8 Nov 2026 12:37:20 -0800 Subject: [PATCH 69/71] fix(maintenance): bind cursors to the admitted profile directory --- crates/tracedecay-global-db/src/registered.rs | 7 + .../src/retention/cold_store.rs | 176 +++++++++--------- .../src/capability_dir.rs | 12 +- .../tracedecay-runtime-core/src/db/access.rs | 45 +++++ 4 files changed, 151 insertions(+), 89 deletions(-) diff --git a/crates/tracedecay-global-db/src/registered.rs b/crates/tracedecay-global-db/src/registered.rs index 4402e809fbf..d86d3e77241 100644 --- a/crates/tracedecay-global-db/src/registered.rs +++ b/crates/tracedecay-global-db/src/registered.rs @@ -966,6 +966,13 @@ impl RegisteredGlobalDb { .await } + /// Directory retained by the registered profile sessions writer at admission. + pub fn profile_sessions_directory(&self) -> tracedecay_domain::errors::Result { + self.database + .write_authority()? + .profile_sessions_directory() + } + pub fn db_path(&self) -> &Path { self.database.canonical_database_path() } diff --git a/crates/tracedecay-maintenance/src/retention/cold_store.rs b/crates/tracedecay-maintenance/src/retention/cold_store.rs index 2cc74daef9c..e8bd0ccd19c 100644 --- a/crates/tracedecay-maintenance/src/retention/cold_store.rs +++ b/crates/tracedecay-maintenance/src/retention/cold_store.rs @@ -4,19 +4,15 @@ //! (`store_maintenance::run_registered_code_generation_retention`). use std::collections::BTreeSet; -use std::ffi::OsStr; -use std::io::{self, Read, Write}; +use std::io::{self, Read}; use std::path::{Path, PathBuf}; -use std::sync::atomic::{AtomicU64, Ordering}; use std::time::Instant; -use cap_fs_ext::{DirExt, FollowSymlinks, OpenOptionsFollowExt}; -#[cfg(unix)] -use cap_std::fs::OpenOptionsExt; +use cap_fs_ext::{DirExt, FollowSymlinks, MetadataExt, OpenOptionsFollowExt, OpenOptionsSyncExt}; use cap_std::fs::{Dir, OpenOptions}; use serde::{Deserialize, Serialize}; use tracedecay_global_db::RegisteredGlobalDb; -use tracedecay_private_fs::capability_dir::{replace_file, sync_directory}; +use tracedecay_private_fs::capability_dir::replace_file_in_open_directory; use tracedecay_runtime_core::cancellation::{CancellationToken, MonotonicDeadline}; use super::{incident_debris, orphan_stores}; @@ -26,7 +22,6 @@ const COLD_STORE_PAGE_LIMIT: usize = 8; const CHECKPOINT_DIRECTORY: &str = "maintenance"; const CHECKPOINT_FILE: &str = "retention-cold-store-cursor-v1.json"; const MAX_CHECKPOINT_BYTES: u64 = 64 * 1024; -static CURSOR_TEMP_NONCE: AtomicU64 = AtomicU64::new(0); #[derive(Debug, Clone, PartialEq, Eq, Deserialize, Serialize)] struct ColdStoreCursorV1 { @@ -85,8 +80,13 @@ pub async fn run_cold_store_page( owner_roots: &BTreeSet, cancellation: &CancellationToken, ) -> tracedecay_domain::errors::Result { - let checkpoint_path = checkpoint_path(profile_root); - let cursor = load_cursor(&checkpoint_path).unwrap_or(ColdStoreCursorV1 { + let admitted = Dir::from_std_file(profile_database.profile_sessions_directory()?); + let directory = admitted_checkpoint_directory(&admitted, profile_root).map_err(|error| { + tracedecay_domain::errors::TraceDecayError::Config { + message: format!("open maintenance cold-store cursor directory: {error}"), + } + })?; + let cursor = load_cursor_in_dir(&directory).unwrap_or(ColdStoreCursorV1 { after_project_id: None, after_unregistered_directory: None, }); @@ -205,7 +205,7 @@ pub async fn run_cold_store_page( after_project_id: page.next_cursor, after_unregistered_directory, }; - persist_cursor(&checkpoint_path, &next_cursor).map_err(|error| { + persist_cursor_in_dir(&directory, &next_cursor).map_err(|error| { tracedecay_domain::errors::TraceDecayError::Config { message: format!("persist maintenance cold-store cursor: {error}"), } @@ -229,52 +229,41 @@ fn classify_cold_store_state( } } +#[cfg(test)] fn checkpoint_path(profile_root: &Path) -> PathBuf { profile_root .join(CHECKPOINT_DIRECTORY) .join(CHECKPOINT_FILE) } +fn admitted_checkpoint_directory(admitted: &Dir, profile_root: &Path) -> io::Result { + let supplied = + Dir::open_ambient_dir(profile_root, cap_std::ambient_authority())?.dir_metadata()?; + let owner = admitted.dir_metadata()?; + if (supplied.dev(), supplied.ino()) != (owner.dev(), owner.ino()) { + return Err(io::Error::other( + "profile directory differs from the admitted owner", + )); + } + checkpoint_directory(admitted) +} + // Keep the maintenance directory capability open for the whole cursor read or // publish. A path check followed by a path-based open or rename can be raced // with a symlink substitution of "maintenance". -fn checkpoint_directory(path: &Path, create: bool) -> io::Result { - let profile = path - .parent() - .and_then(Path::parent) - .ok_or_else(|| io::Error::other("maintenance cursor has no profile root"))?; - let root = Dir::open_ambient_dir(profile, cap_std::ambient_authority())?; - if create { - match root.create_dir(CHECKPOINT_DIRECTORY) { - Ok(()) => {} - Err(error) if error.kind() == io::ErrorKind::AlreadyExists => {} - Err(error) => return Err(error), - } +fn checkpoint_directory(root: &Dir) -> io::Result { + match root.create_dir(CHECKPOINT_DIRECTORY) { + Ok(()) => {} + Err(error) if error.kind() == io::ErrorKind::AlreadyExists => {} + Err(error) => return Err(error), } root.open_dir_nofollow(CHECKPOINT_DIRECTORY) } #[tracing::instrument(name = "maintenance.cold_store.load_cursor", level = "trace", skip_all)] -fn load_cursor(path: &Path) -> Option { - let directory = match checkpoint_directory(path, false) { - Ok(directory) => directory, - Err(error) if error.kind() == io::ErrorKind::NotFound => return None, - Err(error) => { - tracing::warn!( - event = "retention_cold_store_cursor_invalid", - reason = "untrusted_directory", - %error, - "maintenance cursor directory is unsafe; restarting its bounded census" - ); - return None; - } - }; - load_cursor_in_dir(&directory) -} - fn load_cursor_in_dir(directory: &Dir) -> Option { let mut options = OpenOptions::new(); - options.read(true).follow(FollowSymlinks::No); + options.read(true).follow(FollowSymlinks::No).nonblock(true); let file = match directory.open_with(CHECKPOINT_FILE, &options) { Ok(file) => file, Err(error) if error.kind() == io::ErrorKind::NotFound => return None, @@ -325,50 +314,9 @@ fn load_cursor_in_dir(directory: &Dir) -> Option { level = "trace", skip_all )] -fn persist_cursor(path: &Path, cursor: &ColdStoreCursorV1) -> io::Result<()> { - let directory = checkpoint_directory(path, true)?; - persist_cursor_in_dir(&directory, cursor) -} - fn persist_cursor_in_dir(directory: &Dir, cursor: &ColdStoreCursorV1) -> io::Result<()> { let bytes = serde_json::to_vec(cursor).map_err(io::Error::other)?; - // The name supplies no authority. create_new + no-follow guarantees that - // even a collided or attacker-prepared staging name cannot be truncated. - for _ in 0..16 { - let nonce = CURSOR_TEMP_NONCE.fetch_add(1, Ordering::Relaxed); - let temporary_name = format!(".retention-cold-store-{}-{nonce}.tmp", std::process::id()); - let mut options = OpenOptions::new(); - options - .write(true) - .create_new(true) - .follow(FollowSymlinks::No); - #[cfg(unix)] - options.mode(0o600); - let mut temporary = match directory.open_with(&temporary_name, &options) { - Ok(temporary) => temporary, - Err(error) if error.kind() == io::ErrorKind::AlreadyExists => continue, - Err(error) => return Err(error), - }; - let result = (|| { - temporary.write_all(&bytes)?; - temporary.sync_all()?; - drop(temporary); - replace_file( - directory, - OsStr::new(&temporary_name), - OsStr::new(CHECKPOINT_FILE), - )?; - sync_directory(directory) - })(); - if result.is_err() { - let _ = directory.remove_file(&temporary_name); - } - return result; - } - Err(io::Error::new( - io::ErrorKind::AlreadyExists, - "maintenance cursor staging names are occupied", - )) + replace_file_in_open_directory(directory, CHECKPOINT_FILE, &bytes) } fn retention_window_secs(days: u64) -> i64 { @@ -382,6 +330,22 @@ fn retention_window_secs(days: u64) -> i64 { mod tests { use super::*; + fn open_test_directory(path: &Path) -> io::Result { + let root = Dir::open_ambient_dir( + path.parent().unwrap().parent().unwrap(), + cap_std::ambient_authority(), + )?; + checkpoint_directory(&root) + } + + fn load_cursor(path: &Path) -> Option { + load_cursor_in_dir(&open_test_directory(path).ok()?) + } + + fn persist_cursor(path: &Path, cursor: &ColdStoreCursorV1) -> io::Result<()> { + persist_cursor_in_dir(&open_test_directory(path)?, cursor) + } + #[asupersync::test] async fn cold_store_cursor_advances_across_projects_without_profile_stores() { use tracedecay_global_db::tests::harness::RegisteredGlobalDbTestRuntime; @@ -446,6 +410,50 @@ mod tests { ); } + #[cfg(unix)] + #[asupersync::test] + async fn cold_store_cursor_uses_the_directory_its_profile_database_admitted() { + use std::os::unix::fs::symlink; + use tracedecay_global_db::tests::harness::RegisteredGlobalDbTestRuntime; + + let temporary = tempfile::tempdir().unwrap(); + let owner = temporary.path().join("owner"); + let supplied = temporary.path().join("profile"); + let foreign = temporary.path().join("other"); + let foreign_cursor = checkpoint_path(&foreign); + std::fs::create_dir_all(foreign_cursor.parent().unwrap()).unwrap(); + std::fs::write(&foreign_cursor, b"another profile's resume point").unwrap(); + let runtime = RegisteredGlobalDbTestRuntime::profile(&owner) + .await + .unwrap(); + let database = runtime.profile_database_arc(); + symlink(&owner, &supplied).unwrap(); + let owners = BTreeSet::new(); + let cancellation = CancellationToken::new(); + let run = + || run_cold_store_page(&supplied, database.as_ref(), None, &owners, &cancellation); + run().await.unwrap(); + assert!(checkpoint_path(&owner).is_file()); + + std::fs::remove_file(&supplied).unwrap(); + symlink(&foreign, &supplied).unwrap(); + assert!(run().await.is_err()); + std::fs::remove_file(&supplied).unwrap(); + std::fs::rename(&foreign, &supplied).unwrap(); + assert!(run().await.is_err()); + let foreign_cursor = checkpoint_path(&supplied); + assert_eq!( + std::fs::read(&foreign_cursor).unwrap(), + b"another profile's resume point" + ); + assert_eq!( + std::fs::read_dir(foreign_cursor.parent().unwrap()) + .unwrap() + .count(), + 1 + ); + } + #[test] fn checkpoint_survives_restart() { let root = tempfile::tempdir().expect("checkpoint root"); @@ -563,7 +571,7 @@ mod tests { std::fs::create_dir(&profile).unwrap(); std::fs::create_dir(&foreign).unwrap(); let path = checkpoint_path(&profile); - let directory = checkpoint_directory(&path, true).unwrap(); + let directory = open_test_directory(&path).unwrap(); let expected = ColdStoreCursorV1 { after_project_id: Some("local".to_owned()), after_unregistered_directory: None, diff --git a/crates/tracedecay-private-fs/src/capability_dir.rs b/crates/tracedecay-private-fs/src/capability_dir.rs index ccf257eccd1..b4b4630ad83 100644 --- a/crates/tracedecay-private-fs/src/capability_dir.rs +++ b/crates/tracedecay-private-fs/src/capability_dir.rs @@ -218,7 +218,7 @@ pub fn sync_directory(directory: &Dir) -> io::Result<()> { /// Open an owned leaf through a held parent, never following a replaced root /// symlink. Missing roots are normal before the first artifact is sealed. -fn open_owned_directory_nofollow(root: &Path) -> io::Result> { +pub fn open_owned_directory_nofollow(root: &Path) -> io::Result> { let name = root .file_name() .ok_or_else(|| io::Error::new(io::ErrorKind::InvalidInput, "root has no leaf name"))?; @@ -312,14 +312,16 @@ static PROOF_TEMP_NONCE: AtomicU64 = AtomicU64::new(0); /// Publish a proof atomically relative to the same held generation used for /// staging. A substituted root/child link cannot redirect or truncate bytes. pub fn replace_nested_file_nofollow(directory: &Path, name: &str, bytes: &[u8]) -> io::Result<()> { - let leaf = checked_leaf_name(name)?; let child = open_nested_directory_nofollow(directory)?; - replace_file_in_open_directory(&child, leaf, bytes) + replace_file_in_open_directory(&child, name, bytes) } -fn replace_file_in_open_directory(directory: &Dir, leaf: &OsStr, bytes: &[u8]) -> io::Result<()> { +/// Atomically replaces a single file beneath a directory held by its owner. +pub fn replace_file_in_open_directory(directory: &Dir, name: &str, bytes: &[u8]) -> io::Result<()> { use std::io::Write; + let leaf = checked_leaf_name(name)?; + for _ in 0..16 { let nonce = PROOF_TEMP_NONCE.fetch_add(1, Ordering::Relaxed); let temporary_name = format!(".sealed-proof-{}-{nonce}.tmp", std::process::id()); @@ -621,7 +623,7 @@ mod replacement_tests { let held = open_nested_directory_nofollow(&generation).unwrap(); std::fs::rename(&root, &moved).unwrap(); symlink(&foreign, &root).unwrap(); - replace_file_in_open_directory(&held, OsStr::new("sealed.checked"), b"verified").unwrap(); + replace_file_in_open_directory(&held, "sealed.checked", b"verified").unwrap(); assert_eq!( std::fs::read(moved.join("generation/sealed.checked")).unwrap(), b"verified" diff --git a/crates/tracedecay-runtime-core/src/db/access.rs b/crates/tracedecay-runtime-core/src/db/access.rs index f9bf5182162..625ac3e16f8 100644 --- a/crates/tracedecay-runtime-core/src/db/access.rs +++ b/crates/tracedecay-runtime-core/src/db/access.rs @@ -139,6 +139,7 @@ struct MaintenanceScopeState { #[derive(Debug)] struct AuthorityInner { identity: DatabaseIdentity, + profile_directory: Option, role: DatabaseAuthorityRole, token: String, } @@ -353,10 +354,37 @@ impl DatabaseAuthority { role: DatabaseAuthorityRole, intent: &str, ) -> Result { + // Cold-store checkpoints belong to the profile that admitted its sessions + // database. Retain that directory before publishing its writer authority. + let profile_directory = if identity.database_path + == identity + .profile_root + .join(crate::storage::USER_SESSIONS_DB_FILENAME) + { + Some( + tracedecay_private_fs::capability_dir::open_owned_directory_nofollow( + &identity.profile_root, + ) + .map_err(|error| { + access_io_error("open profile directory", &identity.profile_root, &error) + })? + .ok_or_else(|| { + access_error( + "open profile directory", + &identity.profile_root, + "profile directory disappeared", + ) + })? + .into_std_file(), + ) + } else { + None + }; let token = acquire_process_lease(&identity, role, intent)?; Ok(Self { inner: Arc::new(AuthorityInner { identity, + profile_directory, role, token, }), @@ -375,6 +403,23 @@ impl DatabaseAuthority { Ok(self.clone()) } + pub fn profile_sessions_directory(&self) -> Result { + let operation = "access admitted profile directory"; + self.require_active_write_scope(operation)?; + self.inner + .profile_directory + .as_ref() + .ok_or_else(|| { + access_error( + operation, + &self.inner.identity.database_path, + "a profile sessions writer is required", + ) + })? + .try_clone() + .map_err(|error| access_io_error(operation, &self.inner.identity.profile_root, &error)) + } + pub fn canonical_database_path(&self) -> &Path { &self.inner.identity.database_path } From f538fd0d6cf017c474f0c18966c3946445c3351b Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Sun, 8 Nov 2026 12:37:28 -0800 Subject: [PATCH 70/71] test(projection): verify populated cleanup and rollback --- .../src/observation_collision_tests.rs | 2 +- .../src/observation_projection/rebuild.rs | 161 ++++++++++++++++-- 2 files changed, 146 insertions(+), 17 deletions(-) diff --git a/crates/tracedecay-global-db/src/observation_collision_tests.rs b/crates/tracedecay-global-db/src/observation_collision_tests.rs index 49c60e2c3df..330594dba6b 100644 --- a/crates/tracedecay-global-db/src/observation_collision_tests.rs +++ b/crates/tracedecay-global-db/src/observation_collision_tests.rs @@ -491,7 +491,7 @@ fn collision_candidate_at( (observation, anchored) } -fn collision_candidate( +pub(super) fn collision_candidate( session_id: &SessionId, record_id: &str, generation: u64, diff --git a/crates/tracedecay-global-db/src/observation_projection/rebuild.rs b/crates/tracedecay-global-db/src/observation_projection/rebuild.rs index bb61d39ead5..92dc86887ac 100644 --- a/crates/tracedecay-global-db/src/observation_projection/rebuild.rs +++ b/crates/tracedecay-global-db/src/observation_projection/rebuild.rs @@ -2700,28 +2700,157 @@ mod activation_tests { } #[asupersync::test] - async fn clearing_projection_does_not_retain_temp_output_tables() { - let harness = RegisteredGlobalDbHarness::open("projection-clear-no-temp-cache").await; - let transaction = harness.registered.begin_write_transaction().await.unwrap(); + async fn clearing_projection_preserves_shared_staged_and_preexisting_outputs() { + use crate::observation_collision_tests::collision_candidate; + use crate::tests::harness::{HostAdmissionScope, HostAdmissionTestRuntimeV1}; + use std::collections::BTreeMap; + use tracedecay_domain::SessionId; + use tracedecay_runtime_core::db::engine::QueryExecutor; + use tracedecay_store::{ObservationProjectionStore, ObservationStore}; + + async fn outputs( + conn: &impl QueryExecutor, + ) -> BTreeMap)> { + let mut result = BTreeMap::new(); + let mut rows = conn + .query( + "SELECT session_id, message_id, content, location_path FROM lcm_raw_messages", + (), + ) + .await + .unwrap(); + while let Some(row) = rows.next().await.unwrap() { + assert!( + result + .insert( + row.get(0).unwrap(), + ( + row.get(1).unwrap(), + row.get(2).unwrap(), + row.get(3).unwrap() + ) + ) + .is_none() + ); + } + result + } + async fn owners(conn: &impl QueryExecutor) -> Vec<(String, String)> { + let mut rows = conn.query("SELECT observation_id, projector_version FROM observation_projection_provenance ORDER BY observation_id, projector_version", ()).await.unwrap(); + let mut result = Vec::new(); + while let Some(row) = rows.next().await.unwrap() { + result.push((row.get(0).unwrap(), row.get(1).unwrap())); + } + result + } - clear_active_projection(&transaction, "generation.no-temp") + let temporary = tempfile::tempdir().unwrap(); + let runtime = HostAdmissionTestRuntimeV1::profile(temporary.path()) + .await + .unwrap(); + let store = runtime + .observation_store(HostAdmissionScope::Profile) + .unwrap(); + let database = runtime + .registered_database(HostAdmissionScope::Profile) + .unwrap(); + let mut ids = BTreeMap::new(); + for kind in ["exclusive", "shared", "staged", "existing"] { + let session = SessionId::new(kind).unwrap(); + let (observation, write) = collision_candidate( + &session, + &format!("record.{kind}"), + 1, + kind, + &format!("receipt.{kind}"), + None, + ); + store.persist_observation(write).await.unwrap(); + store + .project_observation(observation.observation_id()) + .await + .unwrap(); + ids.insert(kind, observation.observation_id().as_str().to_owned()); + } + const GENERATION: &str = "generation.clear"; + let transaction = database.begin_write_transaction().await.unwrap(); + let first = outputs(&transaction).await; + assert_eq!(first.len(), 4); + assert_eq!(transaction.execute( + "INSERT INTO observation_projection_provenance + (projector_version, observation_id, output_ordinal, receipt_id, + output_provider, output_message_id, output_digest, message_created, retrieval_anchor_id) + SELECT 'another-projector', observation_id, output_ordinal, receipt_id, + output_provider, output_message_id, output_digest, message_created, retrieval_anchor_id + FROM observation_projection_provenance WHERE observation_id = ?1", + params![ids["shared"].as_str()]).await.unwrap(), 1); + assert_eq!(transaction.execute( + "UPDATE observation_projection_provenance SET message_created = 0 WHERE observation_id = ?1", + params![ids["existing"].as_str()]).await.unwrap(), 1); + + let staged_session = SessionRecord { + provider: "collision-test".to_owned(), + session_id: "staged".to_owned(), + ..session(None, None) + }; + stage(&transaction, GENERATION, &staged_session).await; + let (message_id, text, _) = &first["staged"]; + let message = SessionMessageRecord { + provider: staged_session.provider, + session_id: staged_session.session_id, + message_id: message_id.clone(), + text: text.clone(), + role: "assistant".to_owned(), + ordinal: 1, + timestamp: None, + kind: None, + model: None, + tool_names: None, + source_path: None, + source_offset: None, + metadata_json: None, + }; + write_staged_message(&transaction, GENERATION, &message) .await .unwrap(); + let initial_owners = owners(&transaction).await; + assert_eq!(initial_owners.len(), 5); + transaction.commit().await.unwrap(); - let mut rows = transaction - .query( - "SELECT name FROM sqlite_temp_master - WHERE name IN ( - 'observation_projection_rebuild_retained_outputs', - 'observation_projection_rebuild_cleared_outputs' - ) LIMIT 1", - (), - ) + let transaction = database.begin_write_transaction().await.unwrap(); + clear_active_projection(&transaction, GENERATION) .await .unwrap(); - assert!( - rows.next().await.unwrap().is_none(), - "clearing projector ownership must not leave output TEMP tables on the writer" + let mut expected = first.clone(); + assert!(expected.remove("exclusive").is_some()); + assert_eq!(outputs(&transaction).await, expected); + assert_eq!( + owners(&transaction).await, + initial_owners + .iter() + .filter(|(id, _)| id == &ids["shared"]) + .cloned() + .collect::>() + ); + let mut rows = transaction.query( + "SELECT name FROM sqlite_temp_master WHERE name IN + ('observation_projection_rebuild_retained_outputs', 'observation_projection_rebuild_cleared_outputs')", + ()).await.unwrap(); + assert!(rows.next().await.unwrap().is_none()); + drop(rows); + transaction.rollback().await.unwrap(); + let snapshot = database.read_snapshot().await.unwrap(); + assert_eq!(outputs(&snapshot).await, first); + assert_eq!(owners(&snapshot).await, initial_owners); + let mut rows = snapshot.query("PRAGMA integrity_check", ()).await.unwrap(); + assert_eq!( + rows.next() + .await + .unwrap() + .unwrap() + .get::(0) + .unwrap(), + "ok" ); } From 79b22c31126a1f67c4704ae5e620b8dec0617c66 Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Sun, 8 Nov 2026 12:48:40 -0800 Subject: [PATCH 71/71] perf(projection): clear output through bounded ownership pages With one projected output in 100, 10,000, and 100,000 raw messages, SQLite VM steps for the ownership/delete query grew from 1,890 to 1,800,090. The paged query executes 264 steps at each size and deletes only the owned output. --- .../src/observation_projection/rebuild.rs | 134 +++++++++++------- 1 file changed, 85 insertions(+), 49 deletions(-) diff --git a/crates/tracedecay-global-db/src/observation_projection/rebuild.rs b/crates/tracedecay-global-db/src/observation_projection/rebuild.rs index 92dc86887ac..441b2da7d16 100644 --- a/crates/tracedecay-global-db/src/observation_projection/rebuild.rs +++ b/crates/tracedecay-global-db/src/observation_projection/rebuild.rs @@ -2023,38 +2023,86 @@ async fn activate_rebuild_provider_usage( .map_err(|error| storage("activate rebuilt provider usage", error)) } +/// Only this projector's provenance can authorize deleting a message. Bound +/// SQLite's transient list of output keys and seek directly into the message +/// identity index, including when most messages came from another source. +async fn clear_created_projection_messages( + conn: &impl Executor, + generation: &str, +) -> ProjectionStoreResult<()> { + // All observation IDs are TEXT and all stored output ordinals are >= 0. + let mut after_id = String::new(); + let mut after_ordinal = -1_i64; + loop { + const BOUNDARY: &str = "read projection cleanup page boundary"; + let mut rows = conn + .query( + "SELECT observation_id, output_ordinal FROM ( + SELECT observation_id, output_ordinal FROM observation_projection_provenance + WHERE projector_version = ?1 AND (observation_id, output_ordinal) > (?2, ?3) + ORDER BY observation_id, output_ordinal LIMIT ?4 + ) ORDER BY observation_id DESC, output_ordinal DESC LIMIT 1", + params![ + SESSION_MESSAGE_PROJECTOR_VERSION, + after_id.as_str(), + after_ordinal, + REBUILD_PAGE_SIZE + ], + ) + .await + .map_err(|error| storage(BOUNDARY, error))?; + let Some(end) = rows + .next() + .await + .map_err(|error| storage(BOUNDARY, error))? + else { + return Ok(()); + }; + let end_id: String = end.get(0).map_err(|error| storage(BOUNDARY, error))?; + let end_ordinal: i64 = end.get(1).map_err(|error| storage(BOUNDARY, error))?; + drop(rows); + + conn.execute( + "DELETE FROM lcm_raw_messages WHERE (provider, message_id) IN ( + SELECT current.output_provider, current.output_message_id + FROM observation_projection_provenance AS current + WHERE current.projector_version = ?1 AND current.message_created = 1 + AND (current.observation_id, current.output_ordinal) > (?3, ?4) + AND (current.observation_id, current.output_ordinal) <= (?5, ?6) + AND NOT EXISTS ( + SELECT 1 FROM observation_projection_provenance AS foreign_owner + WHERE foreign_owner.projector_version <> ?1 + AND foreign_owner.output_provider = current.output_provider + AND foreign_owner.output_message_id = current.output_message_id + ) + AND NOT EXISTS ( + SELECT 1 FROM observation_projection_rebuild_messages AS staged + WHERE staged.projector_version = ?1 AND staged.generation = ?2 + AND staged.output_provider = current.output_provider + AND staged.output_message_id = current.output_message_id + ) + )", + params![ + SESSION_MESSAGE_PROJECTOR_VERSION, + generation, + after_id.as_str(), + after_ordinal, + end_id.as_str(), + end_ordinal + ], + ) + .await + .map_err(|error| storage("clear projected message rows for rebuild", error))?; + after_id = end_id; + after_ordinal = end_ordinal; + } +} + async fn clear_active_projection( conn: &impl Executor, generation: &str, ) -> ProjectionStoreResult<()> { - // Projector ownership is indexed by output identity. Query that authority - // directly instead of materializing two large TEMP tables on the long-lived - // SQLite writer; their unlinked temp files can survive DELETE and DROP. - conn.execute( - "DELETE FROM lcm_raw_messages - WHERE EXISTS ( - SELECT 1 FROM observation_projection_provenance AS current - WHERE current.projector_version = ?1 - AND current.message_created = 1 - AND current.output_provider = lcm_raw_messages.provider - AND current.output_message_id = lcm_raw_messages.message_id - ) - AND NOT EXISTS ( - SELECT 1 FROM observation_projection_provenance AS foreign_owner - WHERE foreign_owner.projector_version <> ?1 - AND foreign_owner.output_provider = lcm_raw_messages.provider - AND foreign_owner.output_message_id = lcm_raw_messages.message_id - ) - AND NOT EXISTS ( - SELECT 1 FROM observation_projection_rebuild_messages AS staged - WHERE staged.projector_version = ?1 AND staged.generation = ?2 - AND staged.output_provider = lcm_raw_messages.provider - AND staged.output_message_id = lcm_raw_messages.message_id - )", - params![SESSION_MESSAGE_PROJECTOR_VERSION, generation], - ) - .await - .map_err(|error| storage("clear projected message rows for rebuild", error))?; + clear_created_projection_messages(conn, generation).await?; // Foreign projector versions are never deleted. The predicate reads only // those other versions, which remain stable while this version is cleared. conn.execute( @@ -2788,28 +2836,16 @@ mod activation_tests { "UPDATE observation_projection_provenance SET message_created = 0 WHERE observation_id = ?1", params![ids["existing"].as_str()]).await.unwrap(), 1); - let staged_session = SessionRecord { - provider: "collision-test".to_owned(), - session_id: "staged".to_owned(), - ..session(None, None) - }; + let staged_session = super::read_session(&transaction, "collision-test", "staged") + .await + .unwrap() + .unwrap(); stage(&transaction, GENERATION, &staged_session).await; - let (message_id, text, _) = &first["staged"]; - let message = SessionMessageRecord { - provider: staged_session.provider, - session_id: staged_session.session_id, - message_id: message_id.clone(), - text: text.clone(), - role: "assistant".to_owned(), - ordinal: 1, - timestamp: None, - kind: None, - model: None, - tool_names: None, - source_path: None, - source_offset: None, - metadata_json: None, - }; + let message = + super::read_message(&transaction, &staged_session.provider, &first["staged"].0) + .await + .unwrap() + .unwrap(); write_staged_message(&transaction, GENERATION, &message) .await .unwrap();