Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
120 changes: 120 additions & 0 deletions tests/test_distiller_payload_types.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,120 @@
"""Malformed proposer payloads: wrongly-typed fields are dropped, never crash the distiller."""
import json

import pytest

import tokeneconomizer.microskills.distiller as dist
from tokeneconomizer.microskills import Distiller
from tokeneconomizer.microskills.distiller import parse_payload
from fixtures import write_claude_transcript


def _runner(payload):
return lambda prompt, model="agent": (payload, 1.0)


def _payload(**body):
return json.dumps(body)


MALFORMED = [
'{"extractors": null, "facts": null}',
_payload(extractors=[{"name": ["a"], "pattern": "x"}], facts=[]),
_payload(extractors=[{"name": {"x": 1}, "pattern": "x"}], facts=[]),
'{"extractors": true, "facts": []}',
'{"extractors": [], "facts": 7}',
_payload(extractors=[{"name": "a", "pattern": ""}], facts=[{"id": "f", "detector": {}}]),
_payload(extractors=[], facts=[{"id": ["a"], "detector": {"scope": "ran_tests"}}]),
_payload(extractors=[], facts=[{"id": {"x": 1}, "detector": {"scope": "ran_tests"}}]),
_payload(extractors=[], facts=[{"id": "f", "detector": {"scope": ["bad"]}}]),
_payload(extractors=[], facts=[{"id": "f", "detector": {"scope": {"a": 1}}}]),
_payload(extractors=[], facts=[{"id": "f", "detector": {"scope": "ran_tests", "fired": 5}}]),
_payload(extractors=[], facts=[{"id": "f", "detector": {"scope": "ran_tests", "confirm": ["x"]}}]),
_payload(extractors=[], facts=[{"id": "f", "detector": {"scope": "ran_tests", "contradict": {"x": 1}}}]),
_payload(extractors=[], facts=[{"id": "f", "detector": {"scope": "ran_tests"}, "cite": 3}]),
]


@pytest.mark.parametrize("raw", MALFORMED)
def test_parse_payload_drops_wrongly_typed_fields(raw):
exts, facts = parse_payload(raw)
assert exts == []
assert facts == []


def test_parse_payload_keeps_valid_entries_next_to_malformed():
raw = _payload(
extractors=[{"name": ["a"], "pattern": "x"}, {"name": "ok", "pattern": "x"}],
facts=[{"id": {"x": 1}, "detector": {"scope": "ran_tests"}},
{"id": "good", "detector": {"scope": "ran_tests", "confirm": "passed"}}])
exts, facts = parse_payload(raw)
assert [e["name"] for e in exts] == ["ok"]
assert [f["id"] for f in facts] == ["good"]


def test_ground_treats_non_list_cite_as_empty(tmp_path):
d = Distiller("acme/widget")
tr = str(tmp_path / "t.jsonl")
write_claude_transcript(tr, [("bash", {"command": "pytest"}, "1 passed")])
d.update(tr, iid="w-0", resolved=True)
for cite in (3, "w-0", {}, {"w-0": 1}, [["w-0"], {"x": 1}, 7]):
spec = {"id": "f", "detector": {"scope": "ran_tests", "confirm": "passed"}, "cite": cite}
assert d._ground(spec) == 0


@pytest.mark.parametrize("raw", MALFORMED)
def test_update_survives_malformed_proposer_reply(tmp_path, raw):
d = Distiller("acme/widget", runner=_runner(raw))
d.last_propose = -1
saved, dist.PROPOSE_EVERY = dist.PROPOSE_EVERY, 1
try:
for i in range(3):
tr = str(tmp_path / f"t{i}.jsonl")
write_claude_transcript(tr, [("bash", {"command": "pytest"}, "1 passed")])
d.update(tr, iid=f"w-{i}", resolved=True)
finally:
dist.PROPOSE_EVERY = saved
assert d.facts == {}
assert d.extractors == {}


def test_valid_fact_admitted_alongside_malformed_ones(tmp_path):
raw = _payload(
extractors=[{"name": ["bad"], "pattern": "x"}],
facts=[{"id": ["bad"], "detector": {"scope": "ran_tests"}},
{"id": "use-pytest", "claim": "pytest is canonical", "injection": "run pytest",
"detector": {"scope": "ran_tests", "confirm": "passed", "fired": "ran_tests"},
"cite": ["w-0", "w-1"]}])
d = Distiller("acme/widget", runner=_runner(raw))
d.last_propose = -1
saved, dist.PROPOSE_EVERY = dist.PROPOSE_EVERY, 1
try:
for i in range(2):
tr = str(tmp_path / f"t{i}.jsonl")
write_claude_transcript(tr, [("bash", {"command": "pytest"}, "1 passed")])
d.update(tr, iid=f"w-{i}", resolved=True)
finally:
dist.PROPOSE_EVERY = saved
assert "use-pytest" in d.facts
assert set(d.facts) == {"use-pytest"}


def test_empty_detectors_do_not_crowd_out_a_valid_fact(tmp_path):
"""An empty detector never grounds; it must not take one of the MAX_CANDS slots."""
raw = _payload(
extractors=[],
facts=[*({"id": f"empty-{i}", "detector": {}} for i in range(4)),
{"id": "use-pytest", "claim": "pytest is canonical", "injection": "run pytest",
"detector": {"scope": "ran_tests", "confirm": "passed", "fired": "ran_tests"},
"cite": ["w-0", "w-1"]}])
d = Distiller("acme/widget", runner=_runner(raw))
d.last_propose = -1
saved, dist.PROPOSE_EVERY = dist.PROPOSE_EVERY, 1
try:
for i in range(2):
tr = str(tmp_path / f"t{i}.jsonl")
write_claude_transcript(tr, [("bash", {"command": "pytest"}, "1 passed")])
d.update(tr, iid=f"w-{i}", resolved=True)
finally:
dist.PROPOSE_EVERY = saved
assert set(d.facts) == {"use-pytest"}
25 changes: 19 additions & 6 deletions tokeneconomizer/microskills/distiller.py
Original file line number Diff line number Diff line change
Expand Up @@ -270,7 +270,7 @@ def proposer_view(dg):


def _rx(pat):
if not pat:
if not pat or not isinstance(pat, str):
return None
try:
return re.compile(pat)
Expand Down Expand Up @@ -535,13 +535,23 @@ def parse_payload(raw):
obj = json.loads(m.group(0))
except Exception:
return [], []
exts = [e for e in obj.get("extractors", [])
if isinstance(e, dict) and e.get("name") and e.get("pattern")]
facts = [f for f in obj.get("facts", [])
if isinstance(f, dict) and f.get("id") and f.get("detector")]
exts, facts = obj.get("extractors"), obj.get("facts")
exts = [e for e in (exts if isinstance(exts, list) else [])
if isinstance(e, dict) and isinstance(e.get("name"), str) and e.get("name")
and isinstance(e.get("pattern"), str) and e.get("pattern")]
facts = [f for f in (facts if isinstance(facts, list) else [])
if isinstance(f, dict) and isinstance(f.get("id"), str) and f.get("id")
and isinstance(f.get("detector"), dict) and f.get("detector")
and _detector_fields_ok(f["detector"])
and (f.get("cite") is None or isinstance(f.get("cite"), list))]
return exts, facts


def _detector_fields_ok(det):
return all(det.get(k) is None or isinstance(det.get(k), str)
for k in ("scope", "fired", "confirm", "contradict"))


def _reparse_cmds(dg):
"""Recover [(cmd,out)] from a digest's stashed blobs (for retro-featurize). The blobs no
longer say which output came from which command, so the whole output rides on the first
Expand Down Expand Up @@ -672,7 +682,10 @@ def _ground(self, spec):
"""#window cites whose confirm regex fires under the CURRENT features."""
by_iid = {d["iid"]: d for d in self.window}
hits = 0
for iid in (spec.get("cite") or []):
cites = spec.get("cite")
for iid in (cites if isinstance(cites, list) else []):
if not isinstance(iid, str):
continue
dg = by_iid.get(iid)
if dg is None:
continue
Expand Down
Loading