diff --git a/dokimi/changelog.d/20261113_120000_napetrov_fsm687-retention.md b/dokimi/changelog.d/20261113_120000_napetrov_fsm687-retention.md new file mode 100644 index 000000000..bf062c7e8 --- /dev/null +++ b/dokimi/changelog.d/20261113_120000_napetrov_fsm687-retention.md @@ -0,0 +1,8 @@ + + +### Fixed + +- **`--lexical-only` no longer poisons the retention cache** — an ambiguous-band atom answered without an adjudicator is a placeholder, so it is no longer stored. A later run with an LLM adjudicator used to replay it as a final verdict. The cache version moves to `retention-cache-v5`, so entries already on disk are not read; the old files can be deleted. In the other direction, a `--lexical-only` run no longer reads the cache, so it does not replay a verdict an adjudicated run stored and its output no longer depends on the cache directory. diff --git a/dokimi/docs/skill-novelty-walkthrough.md b/dokimi/docs/skill-novelty-walkthrough.md index 521c43988..e6a7187f0 100644 --- a/dokimi/docs/skill-novelty-walkthrough.md +++ b/dokimi/docs/skill-novelty-walkthrough.md @@ -190,6 +190,8 @@ python cli.py skill shorten --artifact skill:skills/my-skill --atoms atoms.json `--lexical-only` runs the retention oracle without an LLM adjudicator: cheap, reproducible, and blind to contradictions, so `knowledge` is a floor. Drop it (and pass `--model`) when you want the adjudicated figure, and expect up to one call per atom per candidate. +Its ambiguous-band atoms are placeholders, not verdicts: they are not cached, so an +adjudicated run after it asks them again. A candidate that drops a protected span is reported but never offered: it is not smaller, it is different. Shortening rewrites one file, so with a bundle-derived atom set it scores diff --git a/dokimi/docs/skill-novelty.md b/dokimi/docs/skill-novelty.md index 6bfd6bd21..c1eacfc57 100644 --- a/dokimi/docs/skill-novelty.md +++ b/dokimi/docs/skill-novelty.md @@ -1319,11 +1319,11 @@ And the retention oracle runs retrieval → NLI locally, escalating to an LLM on ambiguous band, with a disk cache (`--cache-dir`) keyed on (atom, text). Full sweeps stay opt-in (`--all-atoms`). -The cache key carries a version (`retention-cache-v4`). Entries written before failed +The cache key carries a version (`retention-cache-v5`). Entries written before failed adjudications were kept out of the cache can hold a failure stored as `absent`, entries written before a hedged verdict (`retained?`) counted as no verdict can hold it stored as `retained`, and entries written before a reply cut at the output ceiling was an error can hold a verdict parsed -from a truncated reply; all of them look like real ones, so none of them is read. The old files stay in `--cache-dir` and can be deleted. The +from a truncated reply, and entries written by a `--lexical-only` run can hold its ambiguous-band placeholder stored as `absent`; all of them look like real ones, so none of them is read. The old files stay in `--cache-dir` and can be deleted. The first run after upgrading therefore pays one adjudication call per pair the old cache held, unless `DOKIMI_RESPONSE_CACHE` replays it, but the response-cache key carries `max_tokens`, and novelty calls now send 16384 where they sent none, so those entries miss too. diff --git a/dokimi/dokimi/novelty/retention.py b/dokimi/dokimi/novelty/retention.py index 7c6f331e2..e3b1b8942 100644 --- a/dokimi/dokimi/novelty/retention.py +++ b/dokimi/dokimi/novelty/retention.py @@ -31,7 +31,7 @@ ABSENT = "absent" CONTRADICTED = "contradicted" _VERDICTS = (RETAINED, ABSENT, CONTRADICTED) -_CACHE_VERSION = b"retention-cache-v4\0" +_CACHE_VERSION = b"retention-cache-v5\0" _STOP = frozenset(""" a an the and or but if then than that this these those is are was were be been being @@ -103,7 +103,8 @@ def __init__( def verdict(self, atom: Atom, text: str) -> RetentionVerdict: """Decide whether ``atom`` survives in ``text``.""" key = _key(atom, text) - cached = self._memo.get(key) or self._load(key) + # A lexical oracle skips the disk cache: it would replay an adjudicated verdict. + cached = self._memo.get(key) or (self._load(key) if self.client is not None else None) if cached is not None: self.decisions += 1 return cached @@ -111,11 +112,13 @@ def verdict(self, atom: Atom, text: str) -> RetentionVerdict: score = lexical_overlap(atom.statement, text) if score >= self.high: result = RetentionVerdict(RETAINED, "lexical", round(score, 4)) - elif score <= self.low or self.client is None: - # Without an adjudicator the ambiguous band resolves to `absent`: the - # conservative direction, which under-reports retention rather than - # claiming an atom survived that may have been reversed. + elif score <= self.low: result = RetentionVerdict(ABSENT, "lexical", round(score, 4)) + elif self.client is None: + # No adjudicator: a placeholder `absent`, not a verdict, so it is not memoized + # or stored; the cache key names no oracle, so an adjudicated run would replay it. + self.decisions += 1 + return RetentionVerdict(ABSENT, "lexical", round(score, 4)) else: self.escalations += 1 verdict = self._adjudicate(atom, text) diff --git a/dokimi/tests/test_novelty_silent_defaults.py b/dokimi/tests/test_novelty_silent_defaults.py index d5df5ffbc..84bb87c57 100644 --- a/dokimi/tests/test_novelty_silent_defaults.py +++ b/dokimi/tests/test_novelty_silent_defaults.py @@ -28,7 +28,7 @@ judge_value, ) from dokimi.novelty.client import LLMClient -from dokimi.novelty.retention import ABSENT, RETAINED, RetentionOracle, _key +from dokimi.novelty.retention import ABSENT, CONTRADICTED, RETAINED, RetentionOracle, _key def _atoms(n: int) -> list[Atom]: @@ -360,6 +360,40 @@ def test_a_cache_entry_from_before_the_error_fix_is_not_replayed(tmp_path): assert (result.verdict, result.method) == (RETAINED, "llm") +def test_a_lexical_only_run_does_not_store_a_placeholder_for_a_later_llm_run(tmp_path): + atom, text = _AMBIGUOUS + # --lexical-only has no adjudicator: its ambiguous-band answer is a placeholder, + # not a verdict, so it must not reach the disk cache under the same key. + lexical = RetentionOracle(client=None, cache_dir=tmp_path) + assert lexical.verdict(atom, text).verdict == ABSENT + assert lexical.stats()["decisions"] == 1 + assert not list(tmp_path.iterdir()) + calls: list[int] = [] + + def transport(prompt, system): + calls.append(1) + return '{"verdict": "retained"}', None + + llm = RetentionOracle(client=LLMClient(model="x", transport=transport), cache_dir=tmp_path) + result = llm.verdict(atom, text) + assert (result.verdict, result.method) == (RETAINED, "llm") + assert len(calls) == 1 + + +def test_a_lexical_only_run_does_not_replay_a_verdict_an_llm_run_stored(tmp_path): + atom, text = _AMBIGUOUS + llm = RetentionOracle( + client=LLMClient(model="x", transport=lambda p, s: ('{"verdict": "contradicted"}', None)), + cache_dir=tmp_path, + ) + assert llm.verdict(atom, text).verdict == CONTRADICTED + result = RetentionOracle(client=None, cache_dir=tmp_path).verdict(atom, text) + assert (result.verdict, result.method) == (ABSENT, "lexical") + unused = LLMClient(model="x", transport=lambda p, s: (_ for _ in ()).throw(AssertionError)) + replay = RetentionOracle(client=unused, cache_dir=tmp_path).verdict(atom, text) + assert (replay.verdict, replay.method) == (CONTRADICTED, "cache") + + @pytest.mark.parametrize("stored", ['{"verdict": "maybe", "score": 0.5}', "{not json", '{"verdict": ["absent"]}']) def test_an_unusable_disk_cache_entry_is_asked_again(tmp_path, stored): @@ -547,6 +581,24 @@ def test_a_verdict_cached_by_the_v3_cache_is_re_adjudicated(tmp_path, monkeypatc assert oracle.verdict(atom, text).method == "llm" +def test_a_placeholder_cached_by_the_v4_cache_is_re_adjudicated(tmp_path, monkeypatch): + from dokimi.novelty import retention + + atom, text = _AMBIGUOUS + monkeypatch.setattr(retention, "_CACHE_VERSION", b"retention-cache-v4\0") + old_key = _key(atom, text) + monkeypatch.undo() + # A v4 `--lexical-only` placeholder: the v5 cache must not replay it as a verdict. + (tmp_path / f"{old_key}.json").write_text('{"verdict": "absent", "score": 0.5}', + encoding="utf-8") + oracle = RetentionOracle( + client=LLMClient(model="x", transport=lambda p, s: ('{"verdict": "retained"}', None)), + cache_dir=tmp_path, + ) + result = oracle.verdict(atom, text) + assert (result.verdict, result.method) == (RETAINED, "llm") + + def test_a_cut_answer_logs_the_truncation_on_every_run(monkeypatch, tmp_path, caplog): from dokimi import llm