diff --git a/src/agent_env/bundle/ledger.py b/src/agent_env/bundle/ledger.py index be1407ceb..3a144e990 100644 --- a/src/agent_env/bundle/ledger.py +++ b/src/agent_env/bundle/ledger.py @@ -104,12 +104,14 @@ def check(self, write: Write, needs: Mapping[tuple[str, str], int]) -> Check: """Compare ``write`` with the latest version the ledger records for it. ``needs`` holds, by (store, id), the version each earlier write it needs left or, in a dry run, would leave; an env or agent is hashed with them, so one written anew rewrites it too.""" - stored = self._stored_version(write) + stored_doc = self._stored(write) + stored = stored_doc["version"] if stored_doc else None digest = self.digest(write, needs) if digest is None: untracked = ("its inputs aren't tracked yet, so it is written every run",) return Check(write, None, None, untracked, stored, needs) - orphan = self._orphan(write, stored, digest) + stale = _predates_entity_ids(write, stored_doc) + orphan = None if stale else self._orphan(write, stored, digest) if orphan is not None: return Check(write, digest, orphan, ("written by an interrupted run that didn't record it",), stored, needs, adopted=True) @@ -122,6 +124,8 @@ def check(self, write: Write, needs: Mapping[tuple[str, str], int]) -> Check: reasons.append("new" if row is None else f"its v{recorded} is not in the store") elif stored != recorded: reasons.append(f"the store's latest, v{stored}, wasn't recorded by this bundle") + elif stale: + reasons.append("its stored version doesn't record the ids the bundle gave what it names") if row is not None: reasons.extend(_changes(row, digest)) return Check(write, digest, None if reasons else recorded, tuple(reasons), stored, needs) @@ -214,9 +218,9 @@ def _bundle(self, write: Write) -> str: def _latest(self, store: str, id: str) -> dict | None: return self._find(id, self.collection, Filter.of(store=store, id=id, status="done")) - def _stored_version(self, write: Write) -> int | None: - doc = self._find(write.id, _COLLECTIONS[write.kind.store], Filter.of(id=write.id)) - return doc["version"] if doc else None + def _stored(self, write: Write) -> dict | None: + """The store's latest version of ``write``'s entity.""" + return self._find(write.id, _COLLECTIONS[write.kind.store], Filter.of(id=write.id)) def _find(self, entity_id: str, collection: str, filter: Filter) -> dict | None: """The highest version matching ``filter`` in ``entity_id``'s store, read without creating a local store.""" @@ -350,3 +354,9 @@ def _file_sha256(path: Path) -> str: def _now() -> str: return datetime.now(timezone.utc).isoformat() + + +def _predates_entity_ids(write: Write, stored: dict | None) -> bool: + """Whether the store's latest version of task ``write``, written before tasks recorded them, lacks the entity_ids + its bundle gives it now, so reusing it would leave its runs naming those entities as written.""" + return write.kind is BundleKind.TASK and stored is not None and (stored.get("entity_ids") or {}) != write.source.entity_ids diff --git a/src/agent_env/bundle/materialize.py b/src/agent_env/bundle/materialize.py index 60cd26daf..ef3990380 100644 --- a/src/agent_env/bundle/materialize.py +++ b/src/agent_env/bundle/materialize.py @@ -164,7 +164,8 @@ def through_ledger(write: Write, write_fn: Callable[[], int]) -> None: raise BundleError(problems) for write in tasks: through_ledger(write, lambda: Task.put(id=write.id, steps=built[write.id].steps, - requires=write.requires).version) + requires=write.requires, + entity_ids=built[write.id].entity_ids).version) for write in evals: through_ledger(write, lambda: _write_eval(plan, write, written)) return Materialization(plan, tuple(done[_key(write)] for write in plan.writes), tuple(unchecked), tuple(left)) @@ -418,7 +419,8 @@ def _build(write: Write, plan: Plan, written: Mapping[tuple[str, str], Materiali """The task ``write`` puts: as resolved, or given what was ``written``, with each reference that takes a version naming the one it reads.""" steps = write.source.config if written is None else pinned_steps(plan, write, written) - return Task(id=write.id, version=None, steps=[build_step(step) for step in steps]) + return Task(id=write.id, version=None, steps=[build_step(step) for step in steps], + entity_ids=write.source.entity_ids) def pinned_steps(plan: Plan, write: Write, written: Mapping[tuple[str, str], Materialized]) -> list[dict]: diff --git a/src/agent_env/bundle/resolve.py b/src/agent_env/bundle/resolve.py index 71ff5c527..686330e62 100644 --- a/src/agent_env/bundle/resolve.py +++ b/src/agent_env/bundle/resolve.py @@ -102,6 +102,14 @@ class ResolvedEntry: references: tuple[Reference, ...] outputs: tuple[Output, ...] + @property + def entity_ids(self) -> dict[str, str]: + """For a task, each id a field of its steps names as written that ``config`` names by another id, to that id + (``Task.entity_ids``); a reference the plan adds without rewriting a field, as ``--agent``'s, adds nothing.""" + if self.entry.kind is not BundleKind.TASK: + return {} + return _rewritten_ids(self.entry.config, self.config) + @dataclass(frozen=True) class ResolvedBundle: @@ -561,3 +569,18 @@ def _strings(value: Any, path: str = "") -> Iterator[tuple[str, str]]: elif isinstance(value, list): for i, item in enumerate(value): yield from _strings(item, f"{path}[{i}]") + + +def _rewritten_ids(written: list[dict], resolved: list[dict]) -> dict[str, str]: + """Each id a reference field of ``written``'s steps holds that the same field of ``resolved``'s holds another id + in place of, to that id.""" + registry = get_task_step_registry() + renamed = {} + for before, after in zip(written, resolved): + cls = registry.get(before.get("type")) + if cls is None or not cls.entity_refs: + continue + now = {site.path: site.value for site in ref_sites(cls.entity_refs, after)} + renamed.update((site.value, now[site.path]) for site in ref_sites(cls.entity_refs, before) + if site.path in now and now[site.path] != site.value) + return renamed diff --git a/src/agent_env/task/task.py b/src/agent_env/task/task.py index 69d002d74..823a8ae1b 100644 --- a/src/agent_env/task/task.py +++ b/src/agent_env/task/task.py @@ -13,7 +13,7 @@ import uuid from dataclasses import dataclass from datetime import datetime, timezone -from collections.abc import Sequence +from collections.abc import Mapping, Sequence from typing import TYPE_CHECKING, Callable, ClassVar, Optional from agentenv_protocol import client as protocol_v1 @@ -25,7 +25,7 @@ from agent_env.store.routing import run_scope from agent_env.task_step.preflight import pinned from agent_env.task.teardown import env_record_ids -from agent_env.task_step.context import TaskStepContext, regraft_redacted_keys +from agent_env.task_step.context import ENTITY_IDS_KEY, TaskStepContext, regraft_redacted_keys from agent_env.task_step.context_ops import ContextUpdateOps, build_context_update_ops from agent_env.task_step.task_step import RetryConfig, TaskStep @@ -349,6 +349,8 @@ class Task: encoded in each step's depends_on. Independent steps run concurrently. ``requires`` names the plugin packages its steps and what they reference use, as pip requirements, which a registration records (``agent-env bundle register``); None when nothing recorded it. + ``entity_ids`` maps each id a bundle rewrote in the task's steps, as the task names it, to the id the bundle gave + it; ``run`` puts it in ``context.metadata`` for ``run_entity_id``. None for a task stored as written. """ type: ClassVar[str] = "task" @@ -359,11 +361,13 @@ def __init__( version: Optional[int], steps: list[TaskStep] | None = None, requires: Sequence[str] | None = None, + entity_ids: Mapping[str, str] | None = None, ): self.id = id self.version = version self.steps: list[TaskStep] = steps or [] self.requires: list[str] | None = list(requires) if requires is not None else None + self.entity_ids: dict[str, str] | None = dict(entity_ids) if entity_ids else None self._validate_dag(self.steps) @classmethod @@ -424,6 +428,8 @@ def to_dict(self) -> dict: } if self.requires is not None: data["requires"] = list(self.requires) + if self.entity_ids: + data["entity_ids"] = dict(self.entity_ids) return data @classmethod @@ -450,6 +456,7 @@ def from_dict(cls, data: dict) -> "Task": version=data.get("version"), steps=steps, requires=data["requires"] if isinstance(data.get("requires"), list) else None, + entity_ids=data["entity_ids"] if isinstance(data.get("entity_ids"), dict) else None, ) async def run( @@ -495,6 +502,8 @@ async def _run( if context is None: context = TaskStepContext() context.metadata.setdefault("run_group_id", uuid.uuid4().hex) + if self.entity_ids: + context.metadata.setdefault(ENTITY_IDS_KEY, dict(self.entity_ids)) # Seed substitution for EVERY step field, not just prompts. The hub's # "Seed values" box writes context.metadata["seed"], and until now only diff --git a/src/agent_env/task_step/context.py b/src/agent_env/task_step/context.py index 082a3d736..059183bf9 100644 --- a/src/agent_env/task_step/context.py +++ b/src/agent_env/task_step/context.py @@ -13,6 +13,13 @@ from agent_env.providers.sandbox_providers.sandbox_provider import reachable_url HEARTBEAT_KEY = "_heartbeat_fn" +# context.metadata[ENTITY_IDS_KEY]: the run's task's Task.entity_ids, set by Task.run. +ENTITY_IDS_KEY = "entity_ids" + + +def run_entity_id(metadata: dict, entity_id: str) -> str: + """The id the run's task gave the entity it names as ``entity_id`` (``Task.entity_ids``), else ``entity_id``.""" + return (metadata.get(ENTITY_IDS_KEY) or {}).get(entity_id, entity_id) class StepContextError(RuntimeError): diff --git a/src/agent_env/task_step/task_steps/verifiers/rubrics_verifier.py b/src/agent_env/task_step/task_steps/verifiers/rubrics_verifier.py index 915fca57e..7629ead25 100644 --- a/src/agent_env/task_step/task_steps/verifiers/rubrics_verifier.py +++ b/src/agent_env/task_step/task_steps/verifiers/rubrics_verifier.py @@ -20,7 +20,7 @@ from agent_env.config import get_config from agent_env.config.model import ModelParam from agent_env.providers.sandbox_providers.sandbox import into_container -from agent_env.task_step.context import PromptResponse, StepContextError, TaskStepContext, dual_keyed +from agent_env.task_step.context import PromptResponse, StepContextError, TaskStepContext, dual_keyed, run_entity_id from agent_env.task_step.task_steps.verifiers.judge_utils.judge_output_format import ( SCREENSHOT_GROUNDING, JudgeOutputFormat, @@ -188,10 +188,12 @@ def _bulk_verdicts_correction(eval_prompt: str, bulk: dict) -> str: _MISSING = object() -def _lookup_dotted_path(root: object, path: str) -> object: - node = root +def _lookup_dotted_path(metadata: dict, path: str) -> object: + node = metadata for segment in path.split("."): if isinstance(node, dict): + if segment not in node: + segment = run_entity_id(metadata, segment) if segment not in node: return _MISSING node = node[segment] @@ -217,7 +219,9 @@ def _render_fact(value: object) -> str: def resolve_context_facts(facts: dict[str, str], metadata: dict) -> list[tuple[str, str]]: """Resolve ``{label: dotted path}`` against ``metadata`` into ``[(label, rendered value)]``, in order. A path walks dicts by key and lists by index; one that does not resolve renders as - ``CONTEXT_FACT_NOT_AVAILABLE`` so the judge sees the fact was requested. Never raises.""" + ``CONTEXT_FACT_NOT_AVAILABLE`` so the judge sees the fact was requested. Never raises. + A key the run's task gave an entity another id under (``run_entity_id``, as a bundle does) is also found by the + id the task names it by, so a path naming an env as the task wrote it resolves wherever the task runs.""" rendered: list[tuple[str, str]] = [] for label, path in facts.items(): value = _lookup_dotted_path(metadata, path) diff --git a/tst/unit/bundle/ledger_test.py b/tst/unit/bundle/ledger_test.py index 6a5a981fc..99a132006 100644 --- a/tst/unit/bundle/ledger_test.py +++ b/tst/unit/bundle/ledger_test.py @@ -115,11 +115,13 @@ def _latest_version(collection, id): def _written(write): - """Stands in for the materializer's write: the next version of the entity, in the @local namespace's store.""" + """Stands in for the materializer's write: the next version of the entity, in the @local namespace's store, a + task's with the ids its bundle gave what it names, as Task.put stores them.""" collection = _COLLECTIONS[write.kind.store] latest = local_store().find_one(collection, Filter.of(id=write.id), sort=Sort.by("version")) version = latest["version"] + 1 if latest else 1 - local_store().insert(collection, {"id": write.id, "version": version}) + renamed = write.source.entity_ids if write.kind.store == "task" else {} + local_store().insert(collection, {"id": write.id, "version": version, **({"entity_ids": renamed} if renamed else {})}) return version @@ -293,6 +295,18 @@ def edited_meanwhile(): assert ledger.check(write, {}).reasons == ("the ledger doesn't know what its last version was made from",) +def test_a_task_written_before_it_recorded_its_renamed_ids_is_written_again_with_them(bundle_dir, monkeypatch): + with monkeypatch.context() as earlier: + earlier.setattr(resolve_module.ResolvedEntry, "entity_ids", property(lambda self: {})) + _run(bundle_dir) + + checks = _run(bundle_dir) + + assert checks[f"{ROOT}/t"].reasons == ("its stored version doesn't record the ids the bundle gave what it names",) + assert [id for id, check in checks.items() if not check.unchanged] == [f"{ROOT}/t"] + assert all(check.unchanged for check in _run(bundle_dir).values()) + + def test_a_new_digest_scheme_rewrites_everything_and_says_only_that(bundle_dir, monkeypatch): _run(bundle_dir) monkeypatch.setattr("agent_env.bundle.ledger.SCHEME", ledger_module.SCHEME + 1) @@ -524,6 +538,19 @@ def test_a_version_an_interrupted_run_wrote_is_kept_and_recorded_not_written_aga assert [row["status"] for row in local_store().query(LEDGER_COLLECTION, Filter.of(id=GREETING))] == ["done"] +def test_a_task_an_interrupted_earlier_run_wrote_without_its_renamed_ids_is_written_again(bundle_dir, monkeypatch): + plan = plan_of(bundle_dir) + task = next(w for w in plan.writes if w.id == f"{ROOT}/t") + with monkeypatch.context() as earlier: + earlier.setattr(resolve_module.ResolvedEntry, "entity_ids", property(lambda self: {})) + _interrupted_after_its_write(Ledger.for_plan(plan), task, earlier) + + check = Ledger.for_plan(plan).check(task, {}) + + assert not check.unchanged and not check.adopted + assert check.reasons == ("the store's latest, v1, wasn't recorded by this bundle",) + + @pytest.mark.parametrize("since", ["another-write", "an-edit"]) def test_an_interrupted_runs_version_is_written_over_once_the_store_or_the_files_have_moved_on( bundle_dir, monkeypatch, since, diff --git a/tst/unit/bundle/materialize_test.py b/tst/unit/bundle/materialize_test.py index 994841a1a..3e61edd24 100644 --- a/tst/unit/bundle/materialize_test.py +++ b/tst/unit/bundle/materialize_test.py @@ -367,6 +367,16 @@ def test_an_agent_is_written_from_its_agent_toml_over_a_store_image(bundle_dir): assert _summary(_run(bundle_dir))[f"{ROOT}/solver"] == (1, True, ()) +def test_the_written_task_records_the_ids_its_bundle_gave_what_it_names(bundle_dir): + _put_image("claude-image") + layout(bundle_dir, {"agents/solver/agent.toml": AGENT_TOML}) + _steps(bundle_dir, [{"id": "agent", "type": "deploy_agent", "env_ids": [], "a2a_agent_id": "solver"}]) + + _run(bundle_dir) + + assert Task.get(f"{ROOT}/t").entity_ids == {"solver": f"{ROOT}/solver"} + + @pytest.mark.parametrize(("toml", "problem"), [ ('image = "claude-image"\nowner = "me"\n', "agent.toml: unknown key 'owner'; an a2a_agent agent takes image, default_env_vars, metadata, type and id"), diff --git a/tst/unit/bundle/resolve_test.py b/tst/unit/bundle/resolve_test.py index 23cec79f7..824adb98a 100644 --- a/tst/unit/bundle/resolve_test.py +++ b/tst/unit/bundle/resolve_test.py @@ -2,6 +2,7 @@ import json import random +from dataclasses import replace import pytest @@ -149,6 +150,24 @@ def test_an_output_is_named_under_its_task_and_later_steps_find_it(make): assert refs(entry) == [(EntityKind.ENV, f"{ROOT}/tickets", None, "tickets")] +def test_a_task_records_each_id_it_rewrote_as_written(make): + entry = resolved(make(tasks={"t": [ + {"id": "deploy", "type": "deploy_env", "env_id": "tickets"}, + {"id": "seed", "type": "load_artifact", "env_id": "tickets", "artifact_id": "shared-artifact"}, + {"id": "cli", "type": "build_mcp_cli", "env_id": "tickets", "cli_artifact_id": "tickets-cli"}, + ]}), "t") + assert entry.entity_ids == {"tickets": f"{ROOT}/tickets", "tickets-cli": f"{ROOT}/t/tickets-cli"} + + +def test_a_reference_added_without_rewriting_a_field_is_not_a_renamed_id(make): + bundle = make(tasks={"t": [{"id": "deploy", "type": "deploy_env", "env_id": "tickets"}]}) + entry = resolved(bundle, "t") + solver = next(e for e in bundle.entries if e.name == "solver") + swapped = replace(entry, references=(*entry.references, + Reference(EntityKind.AGENT, solver.id, None, solver, "--agent 'solver'", None))) + assert swapped.entity_ids == entry.entity_ids == {"tickets": f"{ROOT}/tickets"} + + def test_an_output_id_a_step_derives_is_named_under_its_task_too(make): entry = resolved(make(tasks={"t": [ {"id": "cli", "type": "build_mcp_cli", "env_id": "tickets", "command_name": "tickets"}, diff --git a/tst/unit/task/entity_ids_test.py b/tst/unit/task/entity_ids_test.py new file mode 100644 index 000000000..af816a248 --- /dev/null +++ b/tst/unit/task/entity_ids_test.py @@ -0,0 +1,61 @@ +"""Task.entity_ids: the ids a bundle gave what a task names, stored with the task and put in each run's metadata.""" + +import pytest + +from agent_env.config.runtime import Config +from agent_env.task import Task +from agent_env.task_step.context import ENTITY_IDS_KEY, TaskStepContext, run_entity_id +from agent_env.task_step.task_step import TaskStep + +IDS = {"world": "@local/~/runs/pn/world"} + + +class _Seen(TaskStep): + """Records the metadata its run reached it with.""" + + type = "entity_ids_test" + entity_refs = () + seen: list[dict] = [] + + @classmethod + def from_dict(cls, data): + return cls(**cls._base_from_dict(data)) + + async def execute(self, context): + _Seen.seen.append(dict(context.metadata)) + return context + + +@pytest.fixture(autouse=True) +def registries(monkeypatch): + steps = {**Config().task_step_registry(), _Seen.type: _Seen} + monkeypatch.setattr(Config, "task_step_registry", lambda self: steps) + monkeypatch.setattr(_Seen, "seen", []) + + +def _task(**kw) -> Task: + return Task(id="t", version=None, steps=[_Seen(id="s", version=None)], **kw) + + +def test_they_are_stored_with_the_task_and_left_out_when_there_are_none(): + assert Task.from_dict(_task(entity_ids=IDS).to_dict()).entity_ids == IDS + assert "entity_ids" not in _task().to_dict() + assert "entity_ids" not in _task(entity_ids={}).to_dict() + + +@pytest.mark.asyncio +async def test_a_run_puts_them_in_its_metadata_for_run_entity_id(local_stores): + await _task(entity_ids=IDS).run(context=TaskStepContext()) + + metadata = _Seen.seen[0] + assert metadata[ENTITY_IDS_KEY] == IDS + assert run_entity_id(metadata, "world") == "@local/~/runs/pn/world" + assert run_entity_id(metadata, "elsewhere") == "elsewhere" + + +@pytest.mark.asyncio +async def test_a_task_stored_as_written_adds_nothing(local_stores): + await _task().run(context=TaskStepContext()) + + assert ENTITY_IDS_KEY not in _Seen.seen[0] + assert run_entity_id(_Seen.seen[0], "world") == "world" diff --git a/tst/unit/task_step/verifiers/test_rubrics_verifier_context_facts_bundle_ids.py b/tst/unit/task_step/verifiers/test_rubrics_verifier_context_facts_bundle_ids.py new file mode 100644 index 000000000..989555065 --- /dev/null +++ b/tst/unit/task_step/verifiers/test_rubrics_verifier_context_facts_bundle_ids.py @@ -0,0 +1,69 @@ +"""context_facts_for_judge in a bundle run: a path naming an entity as the task wrote it also finds the key under the +id the run's task gave it (``entity_ids``), as a bundle-written task's run records.""" +from __future__ import annotations + +import pytest + +from agent_env.task_step.context import PromptResponse, TaskStepContext +from agent_env.task_step.task_steps.verifiers.rubrics_verifier import ( + CONTEXT_FACT_NOT_AVAILABLE, + RubricsVerifierTaskStep, + resolve_context_facts, +) + +ENV = "@local/~/runs/pn/world" +FACTS = {"wf-1 status": "env_trigger_state.world.statuses.wf-1"} + + +def _resolved(metadata: dict, facts: dict = FACTS) -> str: + return next(iter(dict(resolve_context_facts(facts, metadata)).values())) + + +def _state(key: str, status: str) -> dict: + return {key: {"statuses": {"wf-1": status}}} + + +def test_a_renamed_env_is_found_by_the_id_the_task_names_it_by(): + assert _resolved({"entity_ids": {"world": ENV}, "env_trigger_state": _state(ENV, "armed")}) == '"armed"' + + +def test_a_renamed_step_output_is_found_too(): + snapshot = "@local/~/runs/pn/task-a/snap" + metadata = {"entity_ids": {"snap": snapshot}, "env_changes": {snapshot: {"rows": 3}}} + assert _resolved(metadata, {"rows": "env_changes.snap.rows"}) == "3" + + +def test_a_key_as_written_still_wins(): + state = {**_state("world", "fired"), **_state(ENV, "armed")} + assert _resolved({"entity_ids": {"world": ENV}, "env_trigger_state": state}) == '"fired"' + + +@pytest.mark.parametrize("entity_ids", [None, {}, {"other": ENV}], ids=["no map", "empty map", "not in the map"]) +def test_without_a_rename_a_key_resolves_only_as_written(entity_ids): + metadata = {"env_trigger_state": _state(ENV, "fired"), **({"entity_ids": entity_ids} if entity_ids is not None else {})} + assert _resolved(metadata) == CONTEXT_FACT_NOT_AVAILABLE + + +@pytest.mark.asyncio +async def test_the_judge_sees_the_fact_in_a_bundle_run(monkeypatch): + v = RubricsVerifierTaskStep( + id="validity", version=1, criteria=[{"id": "world", "description": "the events fired"}], prompt_id="p1", + use_agent_judge=False, use_trajectory=True, verifier_id="validity", context_facts_for_judge=FACTS, + ) + captured = {} + + async def fake_run(*, eval_prompt, **kwargs): + captured["eval_prompt"] = eval_prompt + return [{**v.criteria[0], "score": 1.0, "result": True}], 0, [], None + + monkeypatch.setattr(v, "_read_trajectory_text", lambda uri: "EVENTS") + monkeypatch.setattr(v, "_run_judge_with_output_retries", fake_run) + ctx = TaskStepContext( + prompt_responses=[PromptResponse(prompt_id="p1", response="done", prompt_text="do it", + agent_trajectory_object_url="s3://t/1.json")], + metadata={"entity_ids": {"world": ENV}, "env_trigger_state": _state(ENV, "armed")}, + ) + + await v.execute(ctx) + + assert '- wf-1 status: "armed"' in captured["eval_prompt"]