diff --git a/plugin/README.md b/plugin/README.md index 8005a8ef..d5510b37 100644 --- a/plugin/README.md +++ b/plugin/README.md @@ -708,6 +708,8 @@ repository is not used by Codex there), while settings are shared by every agent |---|---| | `status` | what it has learned in this repository, what it did, and what your sessions cost | | `microskills` | the facts it injects at session start, and those still forming | +| `microskills forget ID …` | stop injecting the facts with these ids (as `microskills` lists them), and never learn them again under that id or from the same evidence; an unknown id changes nothing | +| `microskills unforget ID …` | bring forgotten facts back, with the evidence they had | | `toolprune` | the deny list: what it left out, what it would leave out next, and the evidence | | `toolprune apply [ITEM …]` | apply a waiting proposal (with `toolprune=propose`), or the named items; from the next session | | `toolprune restore [ITEM …]` | give back the plugin's entries (all, or the named ones); they are not removed again | @@ -737,8 +739,9 @@ How to run them: | Antigravity CLI | none (run from a terminal) | `~/.tokeneconomizer/bin/te --host antigravity ` | | Factory Droid | `/tokeneconomizer-` | `~/.tokeneconomizer/bin/te --host droid ` | -`doctor`, `history` and `repomap` run from a terminal (or as `/te …` on Pi); there is no slash command or -Codex skill for them. +`doctor`, `history`, `repomap` and `microskills forget`/`unforget` run from a terminal (or as `/te …` on +Pi); there is no slash command or Codex skill for them. Quote a fact id that holds spaces or shell +characters: `te microskills forget 'form_repair:run_tests:pytest -q'`. Inside an agent, each command is one model turn; from a terminal it costs nothing. Codex asks before a command that changes something (`config set`, `toolprune restore`, ...), since it diff --git a/plugin/pi/tokeneconomizer.ts b/plugin/pi/tokeneconomizer.ts index 89141375..6ca2debf 100644 --- a/plugin/pi/tokeneconomizer.ts +++ b/plugin/pi/tokeneconomizer.ts @@ -225,7 +225,8 @@ export default function tokeneconomizer(pi: ExtensionAPI) { pi.registerCommand("te", { description: "TokenEconomizer: status | microskills | config | repos (what it learned here, what it did)", handler: async (args, ctx) => { - const argv = (args || "status").trim().split(/\s+/).filter(Boolean); + // `te` splits the line itself (shlex), so a quoted fact id with spaces stays one argument. + const argv = ["--argline", (args || "").trim() || "status"]; // The session id lets `te status` say when this session is recorded under the repository // it started in rather than the one `ctx.cwd` is in now. const out = await run("te", ["--host", "pi", "--cwd", ctx.cwd, ...argv], "", ctx.mode, { diff --git a/tests/test_generic_microskills.py b/tests/test_generic_microskills.py index 30da1baf..57b9fab1 100644 --- a/tests/test_generic_microskills.py +++ b/tests/test_generic_microskills.py @@ -337,6 +337,108 @@ def test_the_generic_digest_keeps_line_breaks_of_list_tool_results(tmp_path): {"cmd": "npm run lint", "error": "error: missing script: lint"}] +BUILD_TEST = [("Bash", {"command": "make build"}, "built"), + ("Bash", {"command": "make test"}, "ok ./... 0.01s")] +BUILD_LINT = [("Bash", {"command": "make build"}, "built"), + ("Bash", {"command": "make lint"}, "0 issues")] + + +def _fold_with(tmp_path, monkeypatch, propose, root, sessions, start=0): + monkeypatch.setattr(dist, "PROPOSE_EVERY", 3) + for i, calls in enumerate(sessions, start): + d = dist.Distiller.load("acme/tool", root, runner=propose, vocab=GENERIC) + d.update(_session(tmp_path, i, calls), iid=f"s{i}") + d.save(root) + return dist.Distiller.load("acme/tool", root, vocab=GENERIC) + + +def _fact(fid, scope=r"make test", confirm=r"make build", cite=("s0", "s1")): + return {"id": fid, "claim": f"{fid} claim", "injection": f"`{fid}`: run make build first.", + "detector": {"scope": scope, "confirm": confirm}, "cite": list(cite)} + + +def test_a_forgotten_fact_is_not_admitted_again_under_a_new_id(tmp_path, monkeypatch): + """The window still holds the sessions that grounded a forgotten fact; a new id that + confirms on that evidence and nothing else is the same fact restated, and is refused.""" + root = str(tmp_path / "state") + first = lambda prompt, model: (json.dumps({"extractors": [], "facts": [ + _fact("build-before-test")]}), 1.0) + d = _fold_with(tmp_path, monkeypatch, first, root, [BUILD_TEST, BUILD_TEST]) + assert "build-before-test" in d.facts + d.forget("build-before-test") + d.save(root) + d = _fold_with(tmp_path, monkeypatch, None, root, [BUILD_LINT, BUILD_LINT], start=2) + + # same evidence, new id and wording: refused + assert d._admit(_fact("make-build-first")) == (False, "restates-forgotten") + # a narrower restatement (one of the same sessions) is refused at grounding already + assert d._admit(_fact("make-build-first", cite=("s0",)))[0] is False + # a fact whose evidence reaches past the forgotten one's gets in, citing the same sessions + wider = _fact("build-first-everywhere", scope=r"make (test|lint)", cite=("s0", "s1")) + assert d._admit(wider) == (True, "admitted") + # and so does one grounded on other sessions + lint = _fact("build-before-lint", scope=r"make lint", cite=("s2", "s3")) + assert d._admit(lint) == (True, "admitted") + + +def test_a_forgotten_id_takes_no_candidate_slot(tmp_path, monkeypatch): + """Up to MAX_CANDS candidates are admitted per paid turn; a forgotten id the proposer + returns does not use one of them up.""" + root = str(tmp_path / "state") + first = lambda prompt, model: (json.dumps({"extractors": [], "facts": [ + _fact("build-before-test")]}), 1.0) + d = _fold_with(tmp_path, monkeypatch, first, root, [BUILD_TEST, BUILD_TEST]) + d.forget("build-before-test") + d.save(root) + again = lambda prompt, model: (json.dumps({"extractors": [], "facts": ( + [_fact("build-before-test")] * dist.MAX_CANDS + + [_fact("build-before-lint", scope=r"make lint", cite=("s2", "s3"))])}), 1.0) + d = _fold_with(tmp_path, monkeypatch, again, root, [BUILD_LINT, BUILD_LINT, BUILD_LINT], + start=2) + assert "build-before-lint" in d.facts and "build-before-test" not in d.facts + + +def test_a_forget_refuses_only_restatements_of_its_own_evidence(tmp_path, monkeypatch): + """A broad forgotten detector confirms on every later session of the same workflow. The + check compares a candidate with the sessions the fact stood on when it was forgotten, so a + corrected fact, or another fact about the same commands, can form from later sessions.""" + root = str(tmp_path / "state") + broad = _fact("make-does-it-all", scope=r"make test", confirm=r"make") + first = lambda prompt, model: (json.dumps({"extractors": [], "facts": [broad]}), 1.0) + d = _fold_with(tmp_path, monkeypatch, first, root, [BUILD_TEST, BUILD_TEST]) + assert "make-does-it-all" in d.facts + d.forget("make-does-it-all") + assert d.forgotten["make-does-it-all"]["evidence"] == ["s0", "s1"] + d.save(root) + + corrected = lambda cite: _fact("make-test-builds-itself", confirm=r"make test", cite=cite) + # on the evidence it was forgotten with, a restatement is refused + assert d._admit(corrected(("s0", "s1"))) == (False, "restates-forgotten") + # one later session that shows it is enough, while the old evidence is still in the window + d = _fold_with(tmp_path, monkeypatch, None, root, [BUILD_TEST], start=2) + assert d._admit(corrected(("s1", "s2"))) == (True, "admitted") + + # and once the old evidence has left the window, nothing about `make test` is kept out + d = _fold_with(tmp_path, monkeypatch, None, root, [BUILD_TEST] * 9, start=3) + assert {dg["iid"] for dg in d.window}.isdisjoint({"s0", "s1"}) + other = _fact("go-test-passes-quietly", confirm=r"ok\s+\./\.\.\. 0\.0", cite=("s10", "s11")) + assert d._admit(other) == (True, "admitted") + + +def test_a_forgotten_entry_without_evidence_falls_back_to_its_cites(tmp_path, monkeypatch): + """An entry forgotten before the evidence was recorded is checked against the sessions the + fact cited when it was admitted.""" + root = str(tmp_path / "state") + first = lambda prompt, model: (json.dumps({"extractors": [], "facts": [ + _fact("build-before-test")]}), 1.0) + d = _fold_with(tmp_path, monkeypatch, first, root, [BUILD_TEST, BUILD_TEST]) + d.forget("build-before-test") + del d.forgotten["build-before-test"]["evidence"] + d.save(root) + d = dist.Distiller.load("acme/tool", root, vocab=GENERIC) + assert d._admit(_fact("make-build-first")) == (False, "restates-forgotten") + + def test_an_output_extractor_authored_on_a_tick_reads_every_output_line(tmp_path, monkeypatch): """An extractor the proposer authors is computed over the window it was authored from, so a fact of the same tick can name it. Over the window's saved blobs it must read what it reads diff --git a/tests/test_microskills_forget.py b/tests/test_microskills_forget.py new file mode 100644 index 00000000..d463206c --- /dev/null +++ b/tests/test_microskills_forget.py @@ -0,0 +1,204 @@ +"""`te microskills forget` / `unforget`: a person retires a learned fact by hand. + +A forgotten fact leaves the injected block at once, stays out however often the sessions that +taught it recur, and comes back as it was with `unforget`. An unknown id changes nothing. +""" +import glob +import json +import os + +import pytest + +from tokeneconomizer.microskills import Distiller, EnvFactsDistiller, forget_facts +from tokeneconomizer.microskills.distiller import Fact +from tokeneconomizer.microskills.lifecycle import write_json_atomic +from tokeneconomizer.plugin import cli + +from test_live_hooks import env, repo, run_session, store_for, trap_session # noqa: F401 + +FID = "form_repair:run_tests:pytest" +CLAIM = "`pytest …` fails here with command not found; `python -m pytest …` works." + + +@pytest.fixture +def te(env, repo, monkeypatch, capsys): # noqa: F811 + """`te --host claude --cwd REPO ...` in the hooks' environment: (exit code, output).""" + for k, v in env.items(): + monkeypatch.setenv(k, v) + + def run(*argv): + capsys.readouterr() + code = cli.main(["--host", "claude", "--cwd", repo, *argv]) + return code, capsys.readouterr().out + return run + + +def learn(env, repo, tmp_path, names): # noqa: F811 + """Fold one trap session per name (the same transcript for a name seen before), and return + the last session start's injected context.""" + start = {} + for name in names: + path = str(tmp_path / f"{name.split('-')[0]}.jsonl") + if not os.path.exists(path): + trap_session(path, repo, int(name.split("-")[0][1:])) + start = run_session(env, repo, name, path) + return (start.get("hookSpecificOutput") or {}).get("additionalContext", "") + + +def env_state(env, repo): # noqa: F811 + (path,) = glob.glob(store_for(env, repo).path("mechanisms", "envfacts_state", "*.json")) + return path + + +def test_forget_takes_a_fact_out_of_the_block_and_it_is_not_learned_again( + env, repo, tmp_path, te): # noqa: F811 + assert CLAIM in learn(env, repo, tmp_path, [f"s{i}" for i in range(6)]) + code, out = te("microskills") + assert code == 0 and FID in out + + code, out = te("microskills", "forget", FID) + assert code == 0 and f"forgotten: {FID}" in out + inj = store_for(env, repo).read("injection.json") + assert CLAIM not in inj["rendered"] and FID not in inj["rendered_ids"] + assert FID not in inj["facts"] + + # The same sessions again, folded as new ones: the evidence recurs, the fact does not. + again = learn(env, repo, tmp_path, [f"s{i}-again" for i in range(6)]) + assert CLAIM not in again + assert CLAIM not in store_for(env, repo).read("injection.json")["rendered"] + saved = json.load(open(env_state(env, repo))) + assert FID not in saved["facts"] and FID in saved["forgotten"] + code, out = te("microskills") + assert "forgotten (" in out and FID in out.split("forgotten (")[1] + + +def test_forget_is_idempotent_and_unforget_brings_the_fact_back( + env, repo, tmp_path, te): # noqa: F811 + learn(env, repo, tmp_path, [f"s{i}" for i in range(6)]) + life = json.load(open(env_state(env, repo)))["facts"][FID]["life"] + assert te("microskills", "forget", FID)[0] == 0 + code, out = te("microskills", "forget", FID) + assert code == 0 and f"already forgotten: {FID}" in out + + code, out = te("microskills", "unforget", FID) + assert code == 0 and f"brought back: {FID}" in out + assert CLAIM in store_for(env, repo).read("injection.json")["rendered"] + saved = json.load(open(env_state(env, repo))) + assert saved["facts"][FID]["life"] == life and saved["forgotten"] == {} + code, out = te("microskills", "unforget", FID) + assert code == 0 and "not forgotten" in out + + +def test_unforget_without_a_record_says_nothing_came_back( + env, repo, tmp_path, te): # noqa: F811 + # Only a hand-edited state file has a forgotten entry with no record. + learn(env, repo, tmp_path, [f"s{i}" for i in range(6)]) + assert te("microskills", "forget", FID)[0] == 0 + path = env_state(env, repo) + state = json.load(open(path)) + del state["forgotten"][FID]["record"] + json.dump(state, open(path, "w")) + + code, out = te("microskills", "unforget", FID) + assert code == 0 and "nothing to bring back" in out and "brought back" not in out + saved = json.load(open(path)) + assert FID not in saved["forgotten"] and FID not in saved["facts"] + + +def test_an_unknown_id_is_an_error_and_changes_nothing(env, repo, tmp_path, te): # noqa: F811 + learn(env, repo, tmp_path, [f"s{i}" for i in range(6)]) + before = open(env_state(env, repo)).read() + inj = store_for(env, repo).read("injection.json") + code, out = te("microskills", "forget", FID, "no-such-fact") + assert code == 1 and "no-such-fact" in out and "not changed" in out + assert open(env_state(env, repo)).read() == before + assert store_for(env, repo).read("injection.json") == inj + assert te("microskills", "unforget", "no-such-fact")[0] == 1 + assert te("microskills", "forget")[0] == 2 # no id at all + + +def test_unknown_id_with_nothing_learned(repo, te): # noqa: F811 + code, out = te("microskills", "forget", "anything") + assert code == 1 and "anything" in out + + +def test_state_from_before_forgetting_still_loads(env, repo, tmp_path, te): # noqa: F811 + learn(env, repo, tmp_path, [f"s{i}" for i in range(6)]) + path = env_state(env, repo) + d = json.load(open(path)) + del d["forgotten"] + json.dump(d, open(path, "w")) + loaded = EnvFactsDistiller.load(d["repo"], os.path.dirname(os.path.dirname(path))) + assert loaded.forgotten == {} and FID in loaded.facts + assert te("microskills", "forget", FID)[0] == 0 + assert FID in json.load(open(path))["forgotten"] + + +# ---- the paid proposer's facts -------------------------------------------------------------- +def _fact(fid): + return Fact({"id": fid, "claim": f"claim of {fid}", "injection": f"say {fid}", + "detector": {}}) + + +def test_a_forgotten_proposed_fact_is_not_admitted_again(tmp_path): + d = Distiller("acme/widget") + d.facts["entrypoint"] = _fact("entrypoint") + d.facts["entrypoint"].life.state = "ACTIVE" + d.save(str(tmp_path)) + + [(fid, changed, claim)] = forget_facts("acme/widget", str(tmp_path), ["entrypoint"]) + assert (fid, changed, claim) == ("entrypoint", True, "claim of entrypoint") + d = Distiller.load("acme/widget", str(tmp_path)) + assert "entrypoint" not in d.facts and "entrypoint" in d.forgotten + assert d._admit({"id": "entrypoint", "claim": "again", "cite": []}) == (False, "forgotten") + assert "say entrypoint" not in d.render() + + with pytest.raises(KeyError): + forget_facts("acme/widget", str(tmp_path), ["entrypoint", "unknown"]) + assert forget_facts("acme/widget", str(tmp_path), ["entrypoint"])[0][1] is False + + +def test_proposer_state_without_the_forgotten_key_loads(tmp_path): + d = Distiller("acme/widget") + d.facts["x"] = _fact("x") + d.save(str(tmp_path)) + (path,) = glob.glob(str(tmp_path / "manual_state" / "*.json")) + saved = json.load(open(path)) + del saved["forgotten"] + json.dump(saved, open(path, "w")) + d = Distiller.load("acme/widget", str(tmp_path)) + assert d.forgotten == {} and set(d.facts) == {"x"} + + +def test_a_failed_write_leaves_the_state_as_it_was(tmp_path): + path = str(tmp_path / "state.json") + write_json_atomic(path, {"a": 1}) + with pytest.raises(TypeError): + write_json_atomic(path, {"a": object()}) + assert json.load(open(path)) == {"a": 1} + assert os.listdir(tmp_path) == ["state.json"] + + +def test_argline_keeps_a_quoted_id_with_spaces_as_one_argument(): + """Pi's `/te ...` hands `te` the line as typed (`--argline`); `te` splits it as a shell would.""" + from tokeneconomizer.plugin.cli import _expand_argline + line = 'microskills forget "form_repair:run_tests:python -m pytest" other-id' + assert _expand_argline(["--host", "pi", "--argline", line]) == [ + "--host", "pi", "microskills", "forget", "form_repair:run_tests:python -m pytest", "other-id"] + assert _expand_argline(["--argline=status"]) == ["status"] + assert _expand_argline(["status"]) == ["status"] + + +def test_argline_keeps_backslashes_as_typed(): + """A Windows path typed after `/te` on Pi is a value, not shell escapes.""" + from tokeneconomizer.plugin.cli import _expand_argline + line = r"config set repomap_python=C:\Python312\python.exe" + assert _expand_argline(["--argline", line]) == [ + "config", "set", r"repomap_python=C:\Python312\python.exe"] + assert _expand_argline(["--argline", r'microskills forget "C:\a b" x']) == [ + "microskills", "forget", r"C:\a b", "x"] + + +def test_an_unclosed_quote_in_argline_changes_nothing(te): # noqa: F811 + from tokeneconomizer.plugin import cli + assert cli.main(["--argline", 'microskills forget "unclosed']) == 2 diff --git a/tokeneconomizer/microskills/README.md b/tokeneconomizer/microskills/README.md index 7c3d2138..b24e60ff 100644 --- a/tokeneconomizer/microskills/README.md +++ b/tokeneconomizer/microskills/README.md @@ -22,6 +22,10 @@ Each finished session goes through three steps: retires on contradiction (for a command-spelling fact, two contradicting sessions in a row), and is injected only while the failure it prevents keeps occurring, or keeps being prevented, in at least 40% of the sessions that exercise it (over at least four). + A person can also retire a fact by hand (`stage.forget_facts`; the plugin's + `te microskills forget`): it leaves the block, and its id goes into its layer's + `forgotten` set, so the fact is never formed again under that id; the proposer's + admission also refuses a new id that confirms only on the forgotten fact's evidence. `stage.PrefixStage` ties these together for a caller: render the block before a session, fold the session in after it. The block is capped (`MAX_FACTS`, `MAX_TOKENS`), and a lint refuses any @@ -33,4 +37,4 @@ fact that reads as completion or verification advice. | `envfacts.py` | `EnvFactsDistiller`: the five LLM-free fact families, rendering, caps | | `distiller.py` | `Distiller`: the model-backed proposer (part of `microskills=on`) and its detector DSL | | `lifecycle.py` | `Lifecycle`: formation, retirement, the firing gate | -| `stage.py` | `PrefixStage`, `ComposedDistiller` | +| `stage.py` | `PrefixStage`, `ComposedDistiller`, `forget_facts` | diff --git a/tokeneconomizer/microskills/__init__.py b/tokeneconomizer/microskills/__init__.py index 70ea1db4..37d98dff 100644 --- a/tokeneconomizer/microskills/__init__.py +++ b/tokeneconomizer/microskills/__init__.py @@ -13,8 +13,8 @@ from .envfacts import EnvFactsDistiller from .lifecycle import Lifecycle from .observe import GENERIC, NODE_WEB, PYTHON_GIT, Vocabulary, classify_command, observe_task -from .stage import ComposedDistiller, PrefixStage +from .stage import ComposedDistiller, PrefixStage, forget_facts, forgotten_facts __all__ = ["Distiller", "EnvFactsDistiller", "GENERIC", "Lifecycle", "NODE_WEB", "PYTHON_GIT", "Vocabulary", "classify_command", "observe_task", "ComposedDistiller", - "PrefixStage"] + "PrefixStage", "forget_facts", "forgotten_facts"] diff --git a/tokeneconomizer/microskills/distiller.py b/tokeneconomizer/microskills/distiller.py index 3005dda0..dc5dd61a 100644 --- a/tokeneconomizer/microskills/distiller.py +++ b/tokeneconomizer/microskills/distiller.py @@ -23,7 +23,7 @@ from ..names import repo_filename from ..trace import call_cwds from ..trace.claude import CONVERSATION -from .lifecycle import Lifecycle, edited_files, edited_pyfiles +from .lifecycle import Forgetting, Lifecycle, edited_files, edited_pyfiles, write_json_atomic # ---- proposer cadence / caps / gates --------------------------------------- PROPOSE_EVERY = 8 # one LLM proposer call per this many tasks (PrefixStage rebinds @@ -552,7 +552,16 @@ def _reparse_cmds(dg): # ---- distiller ------------------------------------------------------------- -class Distiller: +def _forgotten_evidence(rec): + """The tasks a forgotten fact stood on when it was forgotten. An entry saved before they + were recorded falls back to the tasks the fact cited when it was admitted.""" + ev = rec.get("evidence") + if ev is None: + ev = ((rec.get("record") or {}).get("spec") or {}).get("cite") + return [i for i in (ev if isinstance(ev, list) else []) if isinstance(i, str)] + + +class Distiller(Forgetting): """Learns and injects a per-repo operating manual over the task stream. Public surface (the injection Stage calls these): @@ -562,6 +571,8 @@ class Distiller: d.update(transcript_path, iid=..., resolved=...) # fold in the finished task d.save(root) """ + FACT_CLS = Fact + def __init__(self, repo, runner=None, model="agent", external_facts=None, vocab=None): self.repo = repo # The plugin's generic reading (`Vocabulary.generic`): the generic digest and prompt. @@ -578,6 +589,7 @@ def __init__(self, repo, runner=None, model="agent", external_facts=None, vocab= self.last_propose = -1 self.extractors = {} # name -> Extractor self.raw_window = [] # bounded recent [(cmd,out)] per task + self.forgotten = {} # id -> {claim, record}: never admitted again # ---- featurization ----------------------------------------------------- def _observe_fact(self, fact, now, dg, files): @@ -650,6 +662,8 @@ def _maybe_propose(self): exts, facts = parse_payload(raw) for e in exts: self._admit_extractor(e) # append-only + # A forgotten id is refused at admission anyway: keep it from taking a candidate slot. + facts = [f for f in facts if f.get("id") not in self.forgotten] for spec in facts[:MAX_CANDS]: self._admit(spec) self.last_propose = self.now @@ -668,6 +682,36 @@ def _admit_extractor(self, spec): return True # ---- fact admission: new id = append; existing id = contradict-only revise -- + def _confirms(self, detector): + """The window's task ids on which `detector` confirms, under the CURRENT features.""" + out = set() + for dg in self.window: + det = DetectorV3Scope(detector or {}, dg.get("features", {})) + if det.verdict(dg)[0] == "confirm": + out.add(dg["iid"]) + return out + + def _evidence(self, fact): + """The window's tasks the fact's detector confirms on as it is forgotten: the evidence + a restatement under a new id would be grounded on (see _restates_forgotten).""" + return sorted(self._confirms(fact.spec.get("detector"))) + + def _restates_forgotten(self, spec): + """The forgotten fact a new-id candidate restates, if any: the candidate confirms on at + least MIN_CITE of the tasks that fact stood on when it was forgotten, and on no other + task in the window. Only that evidence counts, not what the forgotten detector would + confirm on now: it leaves the window as tasks come in, so a fact that later tasks + show (a corrected one, or another about the same commands) still gets in, where a + broad forgotten detector would otherwise keep it out for good.""" + mine = self._confirms(spec.get("detector")) + if len(mine) < MIN_CITE: + return None + for fid, rec in self.forgotten.items(): + theirs = set(_forgotten_evidence(rec)) + if len(mine & theirs) >= MIN_CITE and mine <= theirs: + return fid + return None + def _ground(self, spec): """#window cites whose confirm regex fires under the CURRENT features.""" by_iid = {d["iid"]: d for d in self.window} @@ -704,10 +748,14 @@ def _frozen_view(self, spec): def _admit(self, spec): fid = spec.get("id") + if fid in self.forgotten: + return False, "forgotten" if fid not in self.facts: hits = self._ground(spec) if hits < MIN_CITE: return False, f"grounding-fail({hits}/{MIN_CITE})" + if self.forgotten and self._restates_forgotten(spec): + return False, "restates-forgotten" spec.setdefault("rev", 1) self.facts[fid] = self._build_fact(spec) return True, "admitted" @@ -780,13 +828,14 @@ def save(self, root): window = [dict(dg, _cmd_blob=dg.get("_cmd_blob", "")[-BLOB_CAP:], _out_blob=dg.get("_out_blob", "")[-2 * BLOB_CAP:]) for dg in self.window[-WINDOW:]] - json.dump({"repo": self.repo, "now": self.now, "opt_tokens": self.opt_tokens, - "last_propose": self.last_propose, - "extractors": {k: e.to_dict() for k, e in self.extractors.items()}, - "facts": {k: f.to_dict() for k, f in self.facts.items()}, - "window": window, - "raw_cmds": [[c for c, _ in cmds] for cmds in self.raw_window[-WINDOW:]]}, - open(self._path(root, self.repo), "w", encoding="utf-8"), indent=2) + write_json_atomic(self._path(root, self.repo), { + "repo": self.repo, "now": self.now, "opt_tokens": self.opt_tokens, + "last_propose": self.last_propose, + "extractors": {k: e.to_dict() for k, e in self.extractors.items()}, + "facts": {k: f.to_dict() for k, f in self.facts.items()}, + "forgotten": self.forgotten, + "window": window, + "raw_cmds": [[c for c, _ in cmds] for cmds in self.raw_window[-WINDOW:]]}) @classmethod def load(cls, repo, root, runner=None, model="agent", external_facts=None, vocab=None): @@ -802,6 +851,8 @@ def load(cls, repo, root, runner=None, model="agent", external_facts=None, vocab self.facts = {k: Fact.from_dict(v) for k, v in d.get("facts", {}).items()} self.window = list(d.get("window") or [])[-WINDOW:] self.raw_window = [[(c, "") for c in cmds] for cmds in d.get("raw_cmds") or []] + self.forgotten = dict(d.get("forgotten") or {}) + self.drop_forgotten() return self def debug_row(self): diff --git a/tokeneconomizer/microskills/envfacts.py b/tokeneconomizer/microskills/envfacts.py index 1fb079dd..60260735 100644 --- a/tokeneconomizer/microskills/envfacts.py +++ b/tokeneconomizer/microskills/envfacts.py @@ -41,7 +41,7 @@ from collections import Counter from ..names import repo_filename -from .lifecycle import Lifecycle +from .lifecycle import Forgetting, Lifecycle, write_json_atomic from .observe import (DEFAULT_VOCAB, NETWORK_ERRORS, form_failed, form_worked, is_bare, label_rule, observe_task) @@ -243,7 +243,7 @@ def lint_ok(text): # -------------------------------------------------------------------------------------- # the distiller # -------------------------------------------------------------------------------------- -class EnvFactsDistiller: +class EnvFactsDistiller(Forgetting): """Learns and injects environment facts over a repository's task stream. Interface parity with `injection.Distiller`, so `PrefixStage` drives both the same way: @@ -256,6 +256,8 @@ class EnvFactsDistiller: on top is told not to restate. `max_facts` / `max_tokens` cap the rendered block (default MAX_FACTS / MAX_TOKENS). """ + FACT_CLS = EnvFact + def __init__(self, repo, runner=None, model="agent", vocab=None, max_facts=None, max_tokens=None): self.max_facts = int(max_facts or MAX_FACTS) @@ -264,6 +266,7 @@ def __init__(self, repo, runner=None, model="agent", vocab=None, max_facts=None, self.vocab = vocab or DEFAULT_VOCAB self.now = 0 self.facts = {} # id -> EnvFact + self.forgotten = {} # id -> {claim, record}: never learned again self.opt_tokens = 0.0 # invariant: LLM-free self.last_injected = set() # fact ids rendered for the task now running self.last_digest = None # the most recent per-task verdicts (telemetry) @@ -577,6 +580,7 @@ def update(self, transcript_path, iid="", resolved=None, edited=None): self.last_digest = {"iid": iid, "resolved": resolved, "injected": sorted(injected), "verdicts": verdicts, "n_calls": digest["n_calls"]} + self.drop_forgotten() self.last_injected = set() self.now += 1 @@ -588,10 +592,10 @@ def _path(root, repo): return os.path.join(d, repo_filename(repo) + ".json") def save(self, root): - with open(self._path(root, self.repo), "w", encoding="utf-8") as fh: - json.dump({"repo": self.repo, "now": self.now, "opt_tokens": 0.0, - "vocab": self.vocab.name, - "facts": {k: f.to_dict() for k, f in self.facts.items()}}, fh, indent=2) + write_json_atomic(self._path(root, self.repo), { + "repo": self.repo, "now": self.now, "opt_tokens": 0.0, "vocab": self.vocab.name, + "facts": {k: f.to_dict() for k, f in self.facts.items()}, + "forgotten": self.forgotten}) @classmethod def load(cls, repo, root, runner=None, model="agent", vocab=None, max_facts=None, @@ -604,6 +608,8 @@ def load(cls, repo, root, runner=None, model="agent", vocab=None, max_facts=None d = json.load(fh) self.now = d.get("now", 0) self.facts = {k: EnvFact.from_dict(v) for k, v in d.get("facts", {}).items()} + self.forgotten = dict(d.get("forgotten") or {}) + self.drop_forgotten() return self # ---- telemetry --------------------------------------------------------------- diff --git a/tokeneconomizer/microskills/lifecycle.py b/tokeneconomizer/microskills/lifecycle.py index 63cf4a86..85465db2 100644 --- a/tokeneconomizer/microskills/lifecycle.py +++ b/tokeneconomizer/microskills/lifecycle.py @@ -9,7 +9,9 @@ injected only while it clears the FIRING GATE: the waste it prevents occurred (or, once injected, kept being prevented) on at least FIRE_GATE of the tasks that exercised it. """ +import json import os +import time # ---- lifecycle knobs (the validated replay constants) ---------------------- K_PROMOTE = 3 # confirmations (drifted, where drift applies) to promote -> ACTIVE @@ -115,3 +117,87 @@ def update(self, now, verdict, fired, exercised, files): hard = self.high and verdict == "contradict" and HARD_KILL if hard or self.score(now) < SCORE_FLOOR: self.state, self.retired_at = "RETIRED", now + + +# ---- forgetting: a person retires a fact by hand ----------------------------- +# Seconds slept between tries of a Windows `os.replace`, which fails while another process has the +# target open; POSIX renames over an open file at once. +_REPLACE_BACKOFF = (0.01, 0.02, 0.05, 0.1, 0.2) + + +def write_json_atomic(path, obj): + """Write `obj` to `path` through a temporary file and `os.replace`: a reader, or a write cut + short, never leaves half a document behind.""" + tmp = f"{path}.{os.getpid()}.tmp" + try: + with open(tmp, "w", encoding="utf-8") as fh: + json.dump(obj, fh, indent=2) + for delay in _REPLACE_BACKOFF if os.name == "nt" else (): + try: + os.replace(tmp, path) + return + except PermissionError: + time.sleep(delay) + os.replace(tmp, path) + finally: + try: + os.unlink(tmp) + except FileNotFoundError: + pass + + +NO_RECORD = "no record" # truthy: the state changed and is saved + + +class Forgetting: + """The forgotten facts of a distiller (`self.facts`: id -> fact). A forgotten fact leaves + `facts`, so it is neither rendered nor folded; its id is kept in `forgotten`, with the claim + it made and its record, so that it is never learned again under that id and `unforget` can + put it back as it was. `FACT_CLS` reads a record back.""" + FACT_CLS = None + + @staticmethod + def _claim_of(fact): + claim = fact.claim + return claim() if callable(claim) else claim + + def known_ids(self): + return set(self.facts) | set(self.forgotten) + + def _evidence(self, fact): + """The sessions `fact` stands on as it is forgotten, for a layer that refuses a + restatement of it (the proposed facts); None for one that does not.""" + return None + + def forget(self, fid): + """True when `fid` is forgotten now, False when it already was; KeyError when no fact + has that id.""" + if fid in self.forgotten: + return False + fact = self.facts[fid] + entry = {"claim": self._claim_of(fact), "record": fact.to_dict()} + evidence = self._evidence(fact) + if evidence is not None: + entry["evidence"] = evidence + del self.facts[fid] + self.forgotten[fid] = entry + return True + + def unforget(self, fid): + """True when `fid` is back among the facts, False when it was not forgotten; KeyError + when no fact has that id. `NO_RECORD` when it was forgotten without a record (a + hand-edited state file): the id is no longer forgotten, but nothing is restored.""" + if fid not in self.forgotten: + if fid in self.facts: + return False + raise KeyError(fid) + rec = self.forgotten.pop(fid).get("record") + if not rec: + return NO_RECORD + self.facts[fid] = self.FACT_CLS.from_dict(rec) + return True + + def drop_forgotten(self): + """Take out any forgotten id the last fold learned again.""" + for fid in self.forgotten: + self.facts.pop(fid, None) diff --git a/tokeneconomizer/microskills/stage.py b/tokeneconomizer/microskills/stage.py index 430a4ef0..19578900 100644 --- a/tokeneconomizer/microskills/stage.py +++ b/tokeneconomizer/microskills/stage.py @@ -7,6 +7,7 @@ """ import os +from ..names import repo_filename from ..stage import Stage from . import distiller as _distiller from .distiller import Distiller @@ -130,3 +131,52 @@ def after_task(self, task, row, run_dir_path, srow): opt_tokens = self.distiller.opt_tokens - before srow["manual_facts"] = self.distiller.debug_row() return opt_tokens, self.reason + + +# ---- forgetting a fact by hand --------------------------------------------- +def _saved_layers(repo, root, vocab=None): + """The distillers with state saved under `root`, loaded: the environment layer first.""" + from . import envfacts + out = [] + for cls, sub in ((envfacts.EnvFactsDistiller, envfacts.STATE_SUBDIR), + (Distiller, _distiller.STATE_SUBDIR)): + if os.path.exists(os.path.join(root, sub, repo_filename(repo) + ".json")): + out.append(cls.load(repo, root, vocab=vocab)) + return out + + +def forget_facts(repo, root, ids, vocab=None, undo=False): + """Forget the facts `ids` (or, with `undo`, bring them back) in whichever saved layer holds + each, and save the layers that changed. A forgotten fact is no longer rendered, and is not + learned again under its id. Returns [(id, changed, claim)] in the order given. When an id + is unknown to every layer nothing is changed, and KeyError carries the unknown ids.""" + layers = _saved_layers(repo, root, vocab) + owner = {} + for layer in layers: + for fid in layer.known_ids(): + owner.setdefault(fid, layer) + unknown = [fid for fid in ids if fid not in owner] + if unknown: + raise KeyError(unknown) + out, changed = [], [] + for fid in dict.fromkeys(ids): + layer = owner[fid] + did = layer.unforget(fid) if undo else layer.forget(fid) + if did and layer not in changed: + changed.append(layer) + fact = layer.facts.get(fid) + claim = (layer._claim_of(fact) if fact is not None + else layer.forgotten.get(fid, {}).get("claim", "")) + out.append((fid, did, claim)) + for layer in changed: + layer.save(root) + return out + + +def forgotten_facts(repo, root, vocab=None): + """{id: the claim it made} for every forgotten fact saved under `root`.""" + out = {} + for layer in _saved_layers(repo, root, vocab): + for fid, rec in layer.forgotten.items(): + out.setdefault(fid, rec.get("claim", "")) + return out diff --git a/tokeneconomizer/plugin/cli.py b/tokeneconomizer/plugin/cli.py index 1a23f870..f0a66f7c 100644 --- a/tokeneconomizer/plugin/cli.py +++ b/tokeneconomizer/plugin/cli.py @@ -3,6 +3,10 @@ te status what it has learned for this repository, and what it did te status --explain explain the current settings and saved learning state te microskills the facts it injects, and the ones still forming + te microskills forget ID ... stop injecting the facts with these ids (as listed by + `te microskills`); they are not learned again + under that id or from the same evidence + te microskills unforget ID ... bring forgotten facts back, as they were te toolprune show the proposed deny list, and the evidence behind it te toolprune apply [ITEM ...] write the proposal (or the named items) to the repository's settings: tools (Claude Code), capabilities (Codex) or @@ -45,6 +49,7 @@ import glob import json import os +import shlex import sys import time @@ -294,6 +299,12 @@ def earlier(n): def cmd_microskills(args): + if args.action in ("forget", "unforget"): + return _microskills_forget(args) + if args.ids: + print(f"`microskills {args.action}` takes no ids; did you mean `microskills forget " + f"{' '.join(args.ids)}`?") + return 2 ctx = _ctx(args) inj = ctx.store.read("injection.json") or {} if ctx.config.microskills == "off": @@ -311,6 +322,55 @@ def cmd_microskills(args): for k, v in sorted(rows): print(f" {v.get('state', '?'):9s} fire={v.get('fire', '?'):>5s} " f"{'INJECTED' if v.get('inject') else ' '} {k}") + from tokeneconomizer.microskills import forgotten_facts + gone = forgotten_facts(ctx.repo.label, os.path.join(ctx.store.base, "mechanisms")) + if gone: + print("\nforgotten (`microskills unforget ID` brings one back):") + for k, claim in sorted(gone.items()): + print(f" {k}" + (f" ({_short(claim)})" if claim else "")) + return 0 + + +def _short(text, n=80): + text = " ".join(str(text).split()) + return text if len(text) <= n else text[:n - 1] + "\u2026" + + +def _microskills_forget(args): + from tokeneconomizer.microskills.lifecycle import NO_RECORD + undo = args.action == "unforget" + if not args.ids: + print(f"`microskills {args.action}` needs the id of at least one fact; " + f"`microskills` lists them") + return 2 + ctx = _ctx(args) + refused = _refuse_unwritable(ctx.root) + if refused: + return refused + try: + done = fold.forget_microskills(ctx, args.ids, undo=undo) + except KeyError as e: + unknown = e.args[0] if e.args and isinstance(e.args[0], list) else list(e.args) + print(f"not changed: no learned fact here has the id {', '.join(unknown)} " + f"({ctx.host.name}, {ctx.repo.label}); `microskills` lists the ids") + return 1 + except fold.Busy as e: + print(f"not changed: {e}; try again in a moment") + return 1 + for fid, changed, claim in done: + if changed == NO_RECORD: + print(f"no longer forgotten, but nothing to bring back: {fid} had no saved record; " + f"a later fold may learn it again") + continue + what = (("brought back" if changed else "not forgotten, nothing to bring back") if undo + else ("forgotten" if changed else "already forgotten")) + print(f"{what}: {fid}" + (f" ({_short(claim)})" if claim else "")) + if any(changed for _, changed, _ in done): + print("from the next session, " + ( + "these facts are kept and injected under the same rules as any other fact" if undo + else "these facts are not injected, and are not learned again under the same id " + "or from the same evidence " + "(`microskills unforget ID` undoes this)")) return 0 @@ -936,7 +996,12 @@ def add(name, *aliases): p.add_argument("--explain", action="store_true", help="briefly explain current settings and saved state (no model calls)") p.set_defaults(fn=cmd_status) - add("microskills", "facts").set_defaults(fn=cmd_microskills) + p = add("microskills", "facts") + p.add_argument("action", nargs="?", default="show", choices=("show", "forget", "unforget"), + help="show the facts (default); forget: stop injecting them and never learn " + "them again; unforget: bring forgotten facts back") + p.add_argument("ids", nargs="*", metavar="ID", help="fact ids, as `te microskills` lists them") + p.set_defaults(fn=cmd_microskills) p = add("toolprune", "untool") p.add_argument("action", nargs="?", default="show", choices=("show", "apply", "restore", "refresh-catalog", "measure", @@ -998,7 +1063,35 @@ def _recent_host(cwd): return max(seen, key=seen.get) if seen else None +def _split_argline(line): + """Split on whitespace and quotes as a POSIX shell would, but keep a backslash as typed: + on Windows `C:\\Python312\\python.exe` is a path, not two escapes. An unclosed quote + raises ValueError.""" + lex = shlex.shlex(line, posix=True) + lex.whitespace_split = True + lex.escape = "" + lex.commenters = "" + return list(lex) + + +def _expand_argline(argv): + """`--argline TEXT` stands for TEXT split by `_split_argline`: a host's slash command + (Pi's `/te ...`) hands over the line as typed, and a fact id may hold spaces and quotes.""" + argv = list(sys.argv[1:] if argv is None else argv) + for i, a in enumerate(argv): + if a == "--argline" and i + 1 < len(argv): + return argv[:i] + _split_argline(argv[i + 1]) + argv[i + 2:] + if a.startswith("--argline="): + return argv[:i] + _split_argline(a.split("=", 1)[1]) + argv[i + 1:] + return argv + + def main(argv=None): + try: + argv = _expand_argline(argv) + except ValueError as e: + print(f"te: could not read the arguments: {e}") + return 2 args = build_parser().parse_args(argv) if args.host is None: args.host = hosts.default_host(recent=_recent_host(args.cwd)) diff --git a/tokeneconomizer/plugin/fold.py b/tokeneconomizer/plugin/fold.py index 9c565f29..efe98b79 100644 --- a/tokeneconomizer/plugin/fold.py +++ b/tokeneconomizer/plugin/fold.py @@ -64,16 +64,20 @@ def _run_dir(workdir, trace_path): return run +def _microskills_config(ctx, profile): + from tokeneconomizer.config import InjectionConfig + return InjectionConfig(enabled=True, profile=profile, + propose_every=ctx.config.microskills_propose_every, + max_facts=ctx.config.microskills_max_facts, + max_tokens=ctx.config.microskills_max_tokens) + + def _fold_microskills(ctx, adapter, run_dir, session_id, injected=None, imported=False): profile = ctx.config.microskills if profile == "off": return None - from tokeneconomizer.config import InjectionConfig from tokeneconomizer.microskills import PrefixStage - icfg = InjectionConfig(enabled=True, profile=profile, - propose_every=ctx.config.microskills_propose_every, - max_facts=ctx.config.microskills_max_facts, - max_tokens=ctx.config.microskills_max_tokens) + icfg = _microskills_config(ctx, profile) # Paused once the host's CLI said it is not signed in (`sidecall.paused`): the free layer # still learns. A session read from the agent's history never calls a model: the free # layer learns from it, and the proposer waits for sessions the plugin sees end. @@ -100,6 +104,42 @@ def _fold_microskills(ctx, adapter, run_dir, session_id, injected=None, imported return {"optimizer_tokens": float(opt_tokens or 0), "rendered_chars": len(rendered)} +class Busy(RuntimeError): + """Another fold holds this repository's learning state.""" + + +def forget_microskills(ctx, ids, undo=False): + """`te microskills forget` (`unforget` with `undo`): forget the facts `ids` in this + repository's learning state, so that no later fold learns them again, and render the cached + block again without them (with them, for `unforget`). Returns [(id, changed, claim)]; + raises KeyError with the unknown ids, changing nothing, and `Busy` when a fold holds the + state longer than the lock waits.""" + from tokeneconomizer.microskills import PrefixStage, forget_facts + adapter = LiveAdapter(ctx.host, ctx.repo.root, vocabulary=ctx.config.vocabulary, + catalog={}, protect=ctx.config.toolprune_protect) + vocab = adapter.env_vocabulary() + with ctx.store.locked("mechanisms") as got: + if not got: + raise Busy("a fold of this repository's sessions holds its state") + out = forget_facts(ctx.repo.label, ctx.mech_dir, ids, vocab=vocab, undo=undo) + prior = ctx.store.read("injection.json") + profile = (ctx.config.microskills if ctx.config.microskills != "off" + else (prior or {}).get("profile")) + if not any(changed for _, changed, _ in out) or prior is None or not profile: + return out + stage = PrefixStage(_microskills_config(ctx, profile), ctx.repo.label, ctx.mech_dir, + False, adapter=adapter, agent=ctx.host.name) + rendered = stage.render() or "" + facts = dict(stage.distiller.debug_row()) if stage.distiller else {} + # What only a fold knows (the last session's verdicts) stays as the fold left it. + facts.update({k: v for k, v in (prior.get("facts") or {}).items() + if k.startswith("_") and k not in facts}) + ctx.store.write("injection.json", dict( + prior, rendered=rendered, rendered_ids=stage.injected_ids(), facts=facts, + updated=time.time())) + return out + + def enough_evidence(sessions, prompts, cfg, high_risk=False): """Has a never-used tool gone unused long enough to be left out? Over `toolprune_evidence` prompts spread across at least `toolprune_min_sessions` sessions, or over