From 061436a53ff9d0a68134627f73cb32f8858c5ded Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Fri, 13 Nov 2026 00:02:42 -0800 Subject: [PATCH 1/7] fix(novelty): a lexical-only run no longer stores a placeholder retention verdict With no adjudicator, an ambiguous-band atom was memoized and written to the disk cache as absent. The key names no oracle, so a later LLM run replayed it as a final cache verdict and never adjudicated. The placeholder is now neither memoized nor stored, and the cache version moves to v5 so old entries are not read. Signed-off-by: Nikolay Petrov --- ...261113_120000_napetrov_fsm687-retention.md | 8 ++++++++ dokimi/docs/skill-novelty-walkthrough.md | 2 ++ dokimi/docs/skill-novelty.md | 4 ++-- dokimi/dokimi/novelty/retention.py | 12 +++++++----- dokimi/tests/test_novelty_silent_defaults.py | 19 +++++++++++++++++++ 5 files changed, 38 insertions(+), 7 deletions(-) create mode 100644 dokimi/changelog.d/20261113_120000_napetrov_fsm687-retention.md diff --git a/dokimi/changelog.d/20261113_120000_napetrov_fsm687-retention.md b/dokimi/changelog.d/20261113_120000_napetrov_fsm687-retention.md new file mode 100644 index 000000000..d69f1ff70 --- /dev/null +++ b/dokimi/changelog.d/20261113_120000_napetrov_fsm687-retention.md @@ -0,0 +1,8 @@ + + +### Fixed + +- **`--lexical-only` no longer poisons the retention cache** — an ambiguous-band atom answered without an adjudicator is a placeholder, so it is no longer stored. A later run with an LLM adjudicator used to replay it as a final verdict. The cache version moves to `retention-cache-v5`, so entries already on disk are not read; the old files can be deleted. diff --git a/dokimi/docs/skill-novelty-walkthrough.md b/dokimi/docs/skill-novelty-walkthrough.md index 52efdaed0..e9bcf4ba0 100644 --- a/dokimi/docs/skill-novelty-walkthrough.md +++ b/dokimi/docs/skill-novelty-walkthrough.md @@ -185,6 +185,8 @@ python cli.py skill shorten --artifact skill:skills/my-skill --atoms atoms.json `--lexical-only` runs the retention oracle without an LLM adjudicator: cheap, reproducible, and blind to contradictions, so `knowledge` is a floor. Drop it (and pass `--model`) when you want the adjudicated figure, and expect up to one call per atom per candidate. +Its ambiguous-band atoms are placeholders, not verdicts: they are not cached, so an +adjudicated run after it asks them again. A candidate that drops a protected span is reported but never offered: it is not smaller, it is different. Shortening rewrites one file, so with a bundle-derived atom set it scores diff --git a/dokimi/docs/skill-novelty.md b/dokimi/docs/skill-novelty.md index 045a114eb..f3ed4b12d 100644 --- a/dokimi/docs/skill-novelty.md +++ b/dokimi/docs/skill-novelty.md @@ -1275,11 +1275,11 @@ And the retention oracle runs retrieval → NLI locally, escalating to an LLM on ambiguous band, with a disk cache (`--cache-dir`) keyed on (atom, text). Full sweeps stay opt-in (`--all-atoms`). -The cache key carries a version (`retention-cache-v4`). Entries written before failed +The cache key carries a version (`retention-cache-v5`). Entries written before failed adjudications were kept out of the cache can hold a failure stored as `absent`, entries written before a hedged verdict (`retained?`) counted as no verdict can hold it stored as `retained`, and entries written before a reply cut at the output ceiling was an error can hold a verdict parsed -from a truncated reply; all of them look like real ones, so none of them is read. The old files stay in `--cache-dir` and can be deleted. The +from a truncated reply, and entries written by a `--lexical-only` run can hold its ambiguous-band placeholder stored as `absent`; all of them look like real ones, so none of them is read. The old files stay in `--cache-dir` and can be deleted. The first run after upgrading therefore pays one adjudication call per pair the old cache held, unless `DOKIMI_RESPONSE_CACHE` replays it, but the response-cache key carries `max_tokens`, and novelty calls now send 16384 where they sent none, so those entries miss too. diff --git a/dokimi/dokimi/novelty/retention.py b/dokimi/dokimi/novelty/retention.py index 7c6f331e2..6bd4028bf 100644 --- a/dokimi/dokimi/novelty/retention.py +++ b/dokimi/dokimi/novelty/retention.py @@ -31,7 +31,7 @@ ABSENT = "absent" CONTRADICTED = "contradicted" _VERDICTS = (RETAINED, ABSENT, CONTRADICTED) -_CACHE_VERSION = b"retention-cache-v4\0" +_CACHE_VERSION = b"retention-cache-v5\0" _STOP = frozenset(""" a an the and or but if then than that this these those is are was were be been being @@ -111,11 +111,13 @@ def verdict(self, atom: Atom, text: str) -> RetentionVerdict: score = lexical_overlap(atom.statement, text) if score >= self.high: result = RetentionVerdict(RETAINED, "lexical", round(score, 4)) - elif score <= self.low or self.client is None: - # Without an adjudicator the ambiguous band resolves to `absent`: the - # conservative direction, which under-reports retention rather than - # claiming an atom survived that may have been reversed. + elif score <= self.low: result = RetentionVerdict(ABSENT, "lexical", round(score, 4)) + elif self.client is None: + # No adjudicator: a placeholder `absent`, not a verdict, so it is not memoized + # or stored; the cache key names no oracle, so an adjudicated run would replay it. + self.decisions += 1 + return RetentionVerdict(ABSENT, "lexical", round(score, 4)) else: self.escalations += 1 verdict = self._adjudicate(atom, text) diff --git a/dokimi/tests/test_novelty_silent_defaults.py b/dokimi/tests/test_novelty_silent_defaults.py index d5df5ffbc..6e10be23b 100644 --- a/dokimi/tests/test_novelty_silent_defaults.py +++ b/dokimi/tests/test_novelty_silent_defaults.py @@ -360,6 +360,25 @@ def test_a_cache_entry_from_before_the_error_fix_is_not_replayed(tmp_path): assert (result.verdict, result.method) == (RETAINED, "llm") +def test_a_lexical_only_run_does_not_store_a_placeholder_for_a_later_llm_run(tmp_path): + atom, text = _AMBIGUOUS + # --lexical-only has no adjudicator: its ambiguous-band answer is a placeholder, + # not a verdict, so it must not reach the disk cache under the same key. + lexical = RetentionOracle(client=None, cache_dir=tmp_path) + assert lexical.verdict(atom, text).verdict == ABSENT + assert not list(tmp_path.iterdir()) + calls: list[int] = [] + + def transport(prompt, system): + calls.append(1) + return '{"verdict": "retained"}', None + + llm = RetentionOracle(client=LLMClient(model="x", transport=transport), cache_dir=tmp_path) + result = llm.verdict(atom, text) + assert (result.verdict, result.method) == (RETAINED, "llm") + assert len(calls) == 1 + + @pytest.mark.parametrize("stored", ['{"verdict": "maybe", "score": 0.5}', "{not json", '{"verdict": ["absent"]}']) def test_an_unusable_disk_cache_entry_is_asked_again(tmp_path, stored): From bbef8edaeaca60b9222bf65e2941ab36dc21ffb5 Mon Sep 17 00:00:00 2001 From: Alexander Rybkin Date: Fri, 13 Nov 2026 22:00:53 -0800 Subject: [PATCH 2/7] fix(novelty): a lexical-only oracle no longer reads the retention cache An adjudicated run stores its verdict under the same key a --lexical-only run computes, so a shared --cache-dir made lexical output depend on the cache (contradicted/cache instead of absent/lexical). A lexical oracle gains nothing from the disk cache, so it skips the load. Signed-off-by: Alexander Rybkin --- .../20261113_120000_napetrov_fsm687-retention.md | 2 +- dokimi/dokimi/novelty/retention.py | 3 ++- dokimi/tests/test_novelty_silent_defaults.py | 16 +++++++++++++++- 3 files changed, 18 insertions(+), 3 deletions(-) diff --git a/dokimi/changelog.d/20261113_120000_napetrov_fsm687-retention.md b/dokimi/changelog.d/20261113_120000_napetrov_fsm687-retention.md index d69f1ff70..bf062c7e8 100644 --- a/dokimi/changelog.d/20261113_120000_napetrov_fsm687-retention.md +++ b/dokimi/changelog.d/20261113_120000_napetrov_fsm687-retention.md @@ -5,4 +5,4 @@ SPDX-License-Identifier: Apache-2.0 ### Fixed -- **`--lexical-only` no longer poisons the retention cache** — an ambiguous-band atom answered without an adjudicator is a placeholder, so it is no longer stored. A later run with an LLM adjudicator used to replay it as a final verdict. The cache version moves to `retention-cache-v5`, so entries already on disk are not read; the old files can be deleted. +- **`--lexical-only` no longer poisons the retention cache** — an ambiguous-band atom answered without an adjudicator is a placeholder, so it is no longer stored. A later run with an LLM adjudicator used to replay it as a final verdict. The cache version moves to `retention-cache-v5`, so entries already on disk are not read; the old files can be deleted. In the other direction, a `--lexical-only` run no longer reads the cache, so it does not replay a verdict an adjudicated run stored and its output no longer depends on the cache directory. diff --git a/dokimi/dokimi/novelty/retention.py b/dokimi/dokimi/novelty/retention.py index 6bd4028bf..e3b1b8942 100644 --- a/dokimi/dokimi/novelty/retention.py +++ b/dokimi/dokimi/novelty/retention.py @@ -103,7 +103,8 @@ def __init__( def verdict(self, atom: Atom, text: str) -> RetentionVerdict: """Decide whether ``atom`` survives in ``text``.""" key = _key(atom, text) - cached = self._memo.get(key) or self._load(key) + # A lexical oracle skips the disk cache: it would replay an adjudicated verdict. + cached = self._memo.get(key) or (self._load(key) if self.client is not None else None) if cached is not None: self.decisions += 1 return cached diff --git a/dokimi/tests/test_novelty_silent_defaults.py b/dokimi/tests/test_novelty_silent_defaults.py index 6e10be23b..e94d54f66 100644 --- a/dokimi/tests/test_novelty_silent_defaults.py +++ b/dokimi/tests/test_novelty_silent_defaults.py @@ -28,7 +28,7 @@ judge_value, ) from dokimi.novelty.client import LLMClient -from dokimi.novelty.retention import ABSENT, RETAINED, RetentionOracle, _key +from dokimi.novelty.retention import ABSENT, CONTRADICTED, RETAINED, RetentionOracle, _key def _atoms(n: int) -> list[Atom]: @@ -379,6 +379,20 @@ def transport(prompt, system): assert len(calls) == 1 +def test_a_lexical_only_run_does_not_replay_a_verdict_an_llm_run_stored(tmp_path): + atom, text = _AMBIGUOUS + llm = RetentionOracle( + client=LLMClient(model="x", transport=lambda p, s: ('{"verdict": "contradicted"}', None)), + cache_dir=tmp_path, + ) + assert llm.verdict(atom, text).verdict == CONTRADICTED + result = RetentionOracle(client=None, cache_dir=tmp_path).verdict(atom, text) + assert (result.verdict, result.method) == (ABSENT, "lexical") + unused = LLMClient(model="x", transport=lambda p, s: (_ for _ in ()).throw(AssertionError)) + replay = RetentionOracle(client=unused, cache_dir=tmp_path).verdict(atom, text) + assert (replay.verdict, replay.method) == (CONTRADICTED, "cache") + + @pytest.mark.parametrize("stored", ['{"verdict": "maybe", "score": 0.5}', "{not json", '{"verdict": ["absent"]}']) def test_an_unusable_disk_cache_entry_is_asked_again(tmp_path, stored): From 1179367dd7e6a4983bd3a83214be7a9612925757 Mon Sep 17 00:00:00 2001 From: Alexander Rybkin Date: Fri, 13 Nov 2026 22:00:53 -0800 Subject: [PATCH 3/7] test(novelty): a v4 retention cache placeholder is re-adjudicated Reverting _CACHE_VERSION to v4 passed the suite, so placeholders a --lexical-only run left on disk could be replayed again unnoticed. Signed-off-by: Alexander Rybkin --- dokimi/tests/test_novelty_silent_defaults.py | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/dokimi/tests/test_novelty_silent_defaults.py b/dokimi/tests/test_novelty_silent_defaults.py index e94d54f66..68673e8dd 100644 --- a/dokimi/tests/test_novelty_silent_defaults.py +++ b/dokimi/tests/test_novelty_silent_defaults.py @@ -580,6 +580,24 @@ def test_a_verdict_cached_by_the_v3_cache_is_re_adjudicated(tmp_path, monkeypatc assert oracle.verdict(atom, text).method == "llm" +def test_a_placeholder_cached_by_the_v4_cache_is_re_adjudicated(tmp_path, monkeypatch): + from dokimi.novelty import retention + + atom, text = _AMBIGUOUS + monkeypatch.setattr(retention, "_CACHE_VERSION", b"retention-cache-v4\0") + old_key = _key(atom, text) + monkeypatch.undo() + # A v4 `--lexical-only` placeholder: the v5 cache must not replay it as a verdict. + (tmp_path / f"{old_key}.json").write_text('{"verdict": "absent", "score": 0.5}', + encoding="utf-8") + oracle = RetentionOracle( + client=LLMClient(model="x", transport=lambda p, s: ('{"verdict": "retained"}', None)), + cache_dir=tmp_path, + ) + result = oracle.verdict(atom, text) + assert (result.verdict, result.method) == (RETAINED, "llm") + + def test_a_cut_answer_logs_the_truncation_on_every_run(monkeypatch, tmp_path, caplog): from dokimi import llm From beb9edad33b0fc002bc9fffbecb106599ba63366 Mon Sep 17 00:00:00 2001 From: Anton Volkov Date: Sat, 14 Nov 2026 16:10:20 +0000 Subject: [PATCH 4/7] test(novelty): pin the decisions count of a lexical-only placeholder Signed-off-by: Anton Volkov --- dokimi/tests/test_novelty_silent_defaults.py | 1 + 1 file changed, 1 insertion(+) diff --git a/dokimi/tests/test_novelty_silent_defaults.py b/dokimi/tests/test_novelty_silent_defaults.py index 68673e8dd..84bb87c57 100644 --- a/dokimi/tests/test_novelty_silent_defaults.py +++ b/dokimi/tests/test_novelty_silent_defaults.py @@ -366,6 +366,7 @@ def test_a_lexical_only_run_does_not_store_a_placeholder_for_a_later_llm_run(tmp # not a verdict, so it must not reach the disk cache under the same key. lexical = RetentionOracle(client=None, cache_dir=tmp_path) assert lexical.verdict(atom, text).verdict == ABSENT + assert lexical.stats()["decisions"] == 1 assert not list(tmp_path.iterdir()) calls: list[int] = [] From f739954bba31aba56e8b763934b2c6242e55991b Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Sat, 14 Nov 2026 11:27:47 -0800 Subject: [PATCH 5/7] chore(review): already fixed: a lexical-only oracle skips the disk cache Thread: PRRT_kwDOVBg7g863EqiK: already fixed in bbef8ed Thread: IC_kwDOVBg7g88AAAABmckl9Q: already fixed in bbef8ed and 1179367 Signed-off-by: Nikolay Petrov From 58b97dccd64ac163335fcf47887e85e94970e596 Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Sat, 14 Nov 2026 11:27:47 -0800 Subject: [PATCH 6/7] chore(review): already fixed: a v4 placeholder is re-adjudicated under v5 Thread: PRRT_kwDOVBg7g863EqiS: already fixed in 1179367 Thread: IC_kwDOVBg7g88AAAABminTgw: already fixed in bbef8ed and 1179367 Signed-off-by: Nikolay Petrov From 67770eaf6f0d35456bdce71f948b0265e2924afe Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Sat, 14 Nov 2026 11:28:48 -0800 Subject: [PATCH 7/7] chore(review): already fixed: the open threads and the reviews that point at them Thread: PRR_kwDOVBg7g88AAAABXxf0ow: already fixed in bbef8ed and 1179367 Thread: PRR_kwDOVBg7g88AAAABX67tvg: already fixed in bbef8ed and 1179367 Thread: PRR_kwDOVBg7g88AAAABYAHPsg: already fixed in bbef8ed and 1179367 Signed-off-by: Nikolay Petrov