From bc74d18c4df68847e2eeacdb0d97169fcc67c7b8 Mon Sep 17 00:00:00 2001 From: Edgar Arakelyan Date: Thu, 12 Nov 2026 10:36:42 -0800 Subject: [PATCH 1/4] fix(rubrics_verifier): a context fact naming an env as the task wrote it resolves in a bundle run A bundle run namespaces the ids it writes as /, so metadata keyed by an env id, such as env_trigger_state, holds the namespaced key, while a context_facts_for_judge path still names the id as the task wrote it and rendered "(not available)". A dict segment that misses is now also tried as /, the root being the run's task id up to its last segment, so the judge sees the fact as it would in the store the task came from. A key as written still wins; a task the store holds under its own id gets no fallback. Co-Authored-By: Claude Opus 5.5 --- .../task_steps/verifiers/rubrics_verifier.py | 19 ++++- ...brics_verifier_context_facts_bundle_ids.py | 70 +++++++++++++++++++ 2 files changed, 86 insertions(+), 3 deletions(-) create mode 100644 tst/unit/task_step/verifiers/test_rubrics_verifier_context_facts_bundle_ids.py diff --git a/src/agent_env/task_step/task_steps/verifiers/rubrics_verifier.py b/src/agent_env/task_step/task_steps/verifiers/rubrics_verifier.py index 915fca57e..8f626c31c 100644 --- a/src/agent_env/task_step/task_steps/verifiers/rubrics_verifier.py +++ b/src/agent_env/task_step/task_steps/verifiers/rubrics_verifier.py @@ -188,10 +188,12 @@ def _bulk_verdicts_correction(eval_prompt: str, bulk: dict) -> str: _MISSING = object() -def _lookup_dotted_path(root: object, path: str) -> object: +def _lookup_dotted_path(root: object, path: str, bundle_root: Optional[str] = None) -> object: node = root for segment in path.split("."): if isinstance(node, dict): + if segment not in node and bundle_root is not None: + segment = f"{bundle_root}/{segment}" if segment not in node: return _MISSING node = node[segment] @@ -217,10 +219,13 @@ def _render_fact(value: object) -> str: def resolve_context_facts(facts: dict[str, str], metadata: dict) -> list[tuple[str, str]]: """Resolve ``{label: dotted path}`` against ``metadata`` into ``[(label, rendered value)]``, in order. A path walks dicts by key and lists by index; one that does not resolve renders as - ``CONTEXT_FACT_NOT_AVAILABLE`` so the judge sees the fact was requested. Never raises.""" + ``CONTEXT_FACT_NOT_AVAILABLE`` so the judge sees the fact was requested. Never raises. + In a bundle run a key is also found under the id the bundle gave it, ``/``, so a path + naming an env as the task wrote it resolves as it does in the store the task came from.""" rendered: list[tuple[str, str]] = [] + bundle_root = _bundle_root(metadata.get("task_id")) for label, path in facts.items(): - value = _lookup_dotted_path(metadata, path) + value = _lookup_dotted_path(metadata, path, bundle_root) if value is _MISSING: text = CONTEXT_FACT_NOT_AVAILABLE else: @@ -1588,3 +1593,11 @@ def _build_loaded_artifacts_block(entries: list[dict]) -> str: "Do not rely solely on the agent's response text — verify against the actual files when possible.", ] return "\n".join(lines) + + +def _bundle_root(task_id: object) -> Optional[str]: + """The id root of the bundle a run's task came from, its id up to the last segment, or None for a task the + store holds under its own id.""" + if not isinstance(task_id, str) or not is_local_id(task_id): + return None + return task_id.rsplit("/", 1)[0] diff --git a/tst/unit/task_step/verifiers/test_rubrics_verifier_context_facts_bundle_ids.py b/tst/unit/task_step/verifiers/test_rubrics_verifier_context_facts_bundle_ids.py new file mode 100644 index 000000000..0eb98d64e --- /dev/null +++ b/tst/unit/task_step/verifiers/test_rubrics_verifier_context_facts_bundle_ids.py @@ -0,0 +1,70 @@ +"""context_facts_for_judge in a bundle run: a path naming an env as the task wrote it also finds the key the bundle +namespaced, ``/``, within the bundle the run's task came from.""" +from __future__ import annotations + +import pytest + +from agent_env.task_step.context import PromptResponse, TaskStepContext +from agent_env.task_step.task_steps.verifiers.rubrics_verifier import ( + CONTEXT_FACT_NOT_AVAILABLE, + RubricsVerifierTaskStep, + resolve_context_facts, +) + +FACTS = {"wf-1 status": "env_trigger_state.world.statuses.wf-1"} + + +def _resolved(metadata: dict) -> str: + return dict(resolve_context_facts(FACTS, metadata))["wf-1 status"] + + +def _state(key: str, status: str) -> dict: + return {key: {"statuses": {"wf-1": status}}} + + +@pytest.mark.parametrize("task_id, key", [ + ("@local/~/runs/pn/task-a", "@local/~/runs/pn/world"), + ("@hf/org/repo/pn/task-a", "@hf/org/repo/pn/world"), +]) +def test_a_bundle_run_finds_the_key_its_bundle_namespaced(task_id, key): + assert _resolved({"task_id": task_id, "env_trigger_state": _state(key, "armed")}) == '"armed"' + + +def test_a_key_as_written_still_wins(): + state = {**_state("world", "fired"), **_state("@local/b/world", "armed")} + assert _resolved({"task_id": "@local/b/task-a", "env_trigger_state": state}) == '"fired"' + + +def test_another_bundles_key_is_not_found(): + assert _resolved({"task_id": "@local/b/task-a", "env_trigger_state": _state("@local/other/world", "fired")}) \ + == CONTEXT_FACT_NOT_AVAILABLE + + +@pytest.mark.parametrize("metadata", [{"task_id": "task-a"}, {}], ids=["store task", "no task id"]) +def test_a_task_from_the_store_resolves_only_keys_as_written(metadata): + assert _resolved({**metadata, "env_trigger_state": _state("@local/b/world", "fired")}) == CONTEXT_FACT_NOT_AVAILABLE + + +@pytest.mark.asyncio +async def test_the_judge_sees_the_fact_in_a_bundle_run(monkeypatch): + v = RubricsVerifierTaskStep( + id="validity", version=1, criteria=[{"id": "world", "description": "the events fired"}], prompt_id="p1", + use_agent_judge=False, use_trajectory=True, verifier_id="validity", context_facts_for_judge=FACTS, + ) + captured = {} + + async def fake_run(*, eval_prompt, **kwargs): + captured["eval_prompt"] = eval_prompt + return [{**v.criteria[0], "score": 1.0, "result": True}], 0, [], None + + monkeypatch.setattr(v, "_read_trajectory_text", lambda uri: "EVENTS") + monkeypatch.setattr(v, "_run_judge_with_output_retries", fake_run) + ctx = TaskStepContext( + prompt_responses=[PromptResponse(prompt_id="p1", response="done", prompt_text="do it", + agent_trajectory_object_url="s3://t/1.json")], + metadata={"task_id": "@local/~/runs/pn/task-a", "env_trigger_state": _state("@local/~/runs/pn/world", "armed")}, + ) + + await v.execute(ctx) + + assert '- wf-1 status: "armed"' in captured["eval_prompt"] From 95488f15dc84b5ddeb6b30b7780f5ce718ca58e8 Mon Sep 17 00:00:00 2001 From: Edgar Arakelyan Date: Thu, 12 Nov 2026 11:41:14 -0800 Subject: [PATCH 2/4] fix(task): a task carries the ids its bundle gave what it names, and context facts resolve through them Replaces the previous commit's inference from the task id with the renames the bundle actually made. Resolving a bundle task now records each id it rewrote, as the task names it, against the id it names now: one of the bundle's entities by its name, and an output a step writes by the name the step gave it, which sits under the task id rather than the bundle root. The bundle stores that map on the task (Task.entity_ids, like requires; absent when nothing was renamed), and Task.run puts it in context.metadata["entity_ids"], so every runner of a bundle-written task has it, not only agent-env run. run_entity_id(metadata, id) resolves through it, and context facts use it for a key that misses. Translating a plan to registry ids carries over, since the map is derived from the references and outputs it rewrites. Co-Authored-By: Claude Opus 5.5 --- src/agent_env/bundle/materialize.py | 6 +- src/agent_env/bundle/resolve.py | 11 +++- src/agent_env/task/task.py | 13 +++- src/agent_env/task_step/context.py | 7 +++ .../task_steps/verifiers/rubrics_verifier.py | 25 +++----- tst/unit/bundle/materialize_test.py | 10 +++ tst/unit/bundle/resolve_test.py | 11 +++- tst/unit/task/entity_ids_test.py | 61 +++++++++++++++++++ ...brics_verifier_context_facts_bundle_ids.py | 39 ++++++------ 9 files changed, 140 insertions(+), 43 deletions(-) create mode 100644 tst/unit/task/entity_ids_test.py diff --git a/src/agent_env/bundle/materialize.py b/src/agent_env/bundle/materialize.py index 60cd26daf..ef3990380 100644 --- a/src/agent_env/bundle/materialize.py +++ b/src/agent_env/bundle/materialize.py @@ -164,7 +164,8 @@ def through_ledger(write: Write, write_fn: Callable[[], int]) -> None: raise BundleError(problems) for write in tasks: through_ledger(write, lambda: Task.put(id=write.id, steps=built[write.id].steps, - requires=write.requires).version) + requires=write.requires, + entity_ids=built[write.id].entity_ids).version) for write in evals: through_ledger(write, lambda: _write_eval(plan, write, written)) return Materialization(plan, tuple(done[_key(write)] for write in plan.writes), tuple(unchecked), tuple(left)) @@ -418,7 +419,8 @@ def _build(write: Write, plan: Plan, written: Mapping[tuple[str, str], Materiali """The task ``write`` puts: as resolved, or given what was ``written``, with each reference that takes a version naming the one it reads.""" steps = write.source.config if written is None else pinned_steps(plan, write, written) - return Task(id=write.id, version=None, steps=[build_step(step) for step in steps]) + return Task(id=write.id, version=None, steps=[build_step(step) for step in steps], + entity_ids=write.source.entity_ids) def pinned_steps(plan: Plan, write: Write, written: Mapping[tuple[str, str], Materialized]) -> list[dict]: diff --git a/src/agent_env/bundle/resolve.py b/src/agent_env/bundle/resolve.py index 71ff5c527..d933dbd46 100644 --- a/src/agent_env/bundle/resolve.py +++ b/src/agent_env/bundle/resolve.py @@ -90,6 +90,7 @@ class Output: step: int # the writing step's index in the task step_id: Any artifact_type: str | None + name: str # as the step names it @dataclass(frozen=True) @@ -102,6 +103,14 @@ class ResolvedEntry: references: tuple[Reference, ...] outputs: tuple[Output, ...] + @property + def entity_ids(self) -> dict[str, str]: + """Each id the entry's config names that resolving rewrote, as written, to the id it names now + (``Task.entity_ids``): one of the bundle's entities by its name, an output a step writes by the name it gave.""" + renamed = {output.name: output.id for output in self.outputs} + renamed.update((ref.local.name, ref.id) for ref in self.references if isinstance(ref.local, BundleEntry)) + return {name: id for name, id in renamed.items() if name != id} + @dataclass(frozen=True) class ResolvedBundle: @@ -392,7 +401,7 @@ def _outputs(self, entry: BundleEntry, declared: list[tuple[int, dict, type]]) - "root, which only a bundle run from the Hugging Face Hub writes under its own") continue site.rewrite(output_id) - outputs[kind, name] = Output(kind, output_id, index, step.get("id"), site.ref.artifact_type) + outputs[kind, name] = Output(kind, output_id, index, step.get("id"), site.ref.artifact_type, name) return outputs def _depends_on(self, entry: BundleEntry, steps: list[dict]) -> list[list[str] | None]: diff --git a/src/agent_env/task/task.py b/src/agent_env/task/task.py index 69d002d74..823a8ae1b 100644 --- a/src/agent_env/task/task.py +++ b/src/agent_env/task/task.py @@ -13,7 +13,7 @@ import uuid from dataclasses import dataclass from datetime import datetime, timezone -from collections.abc import Sequence +from collections.abc import Mapping, Sequence from typing import TYPE_CHECKING, Callable, ClassVar, Optional from agentenv_protocol import client as protocol_v1 @@ -25,7 +25,7 @@ from agent_env.store.routing import run_scope from agent_env.task_step.preflight import pinned from agent_env.task.teardown import env_record_ids -from agent_env.task_step.context import TaskStepContext, regraft_redacted_keys +from agent_env.task_step.context import ENTITY_IDS_KEY, TaskStepContext, regraft_redacted_keys from agent_env.task_step.context_ops import ContextUpdateOps, build_context_update_ops from agent_env.task_step.task_step import RetryConfig, TaskStep @@ -349,6 +349,8 @@ class Task: encoded in each step's depends_on. Independent steps run concurrently. ``requires`` names the plugin packages its steps and what they reference use, as pip requirements, which a registration records (``agent-env bundle register``); None when nothing recorded it. + ``entity_ids`` maps each id a bundle rewrote in the task's steps, as the task names it, to the id the bundle gave + it; ``run`` puts it in ``context.metadata`` for ``run_entity_id``. None for a task stored as written. """ type: ClassVar[str] = "task" @@ -359,11 +361,13 @@ def __init__( version: Optional[int], steps: list[TaskStep] | None = None, requires: Sequence[str] | None = None, + entity_ids: Mapping[str, str] | None = None, ): self.id = id self.version = version self.steps: list[TaskStep] = steps or [] self.requires: list[str] | None = list(requires) if requires is not None else None + self.entity_ids: dict[str, str] | None = dict(entity_ids) if entity_ids else None self._validate_dag(self.steps) @classmethod @@ -424,6 +428,8 @@ def to_dict(self) -> dict: } if self.requires is not None: data["requires"] = list(self.requires) + if self.entity_ids: + data["entity_ids"] = dict(self.entity_ids) return data @classmethod @@ -450,6 +456,7 @@ def from_dict(cls, data: dict) -> "Task": version=data.get("version"), steps=steps, requires=data["requires"] if isinstance(data.get("requires"), list) else None, + entity_ids=data["entity_ids"] if isinstance(data.get("entity_ids"), dict) else None, ) async def run( @@ -495,6 +502,8 @@ async def _run( if context is None: context = TaskStepContext() context.metadata.setdefault("run_group_id", uuid.uuid4().hex) + if self.entity_ids: + context.metadata.setdefault(ENTITY_IDS_KEY, dict(self.entity_ids)) # Seed substitution for EVERY step field, not just prompts. The hub's # "Seed values" box writes context.metadata["seed"], and until now only diff --git a/src/agent_env/task_step/context.py b/src/agent_env/task_step/context.py index 082a3d736..059183bf9 100644 --- a/src/agent_env/task_step/context.py +++ b/src/agent_env/task_step/context.py @@ -13,6 +13,13 @@ from agent_env.providers.sandbox_providers.sandbox_provider import reachable_url HEARTBEAT_KEY = "_heartbeat_fn" +# context.metadata[ENTITY_IDS_KEY]: the run's task's Task.entity_ids, set by Task.run. +ENTITY_IDS_KEY = "entity_ids" + + +def run_entity_id(metadata: dict, entity_id: str) -> str: + """The id the run's task gave the entity it names as ``entity_id`` (``Task.entity_ids``), else ``entity_id``.""" + return (metadata.get(ENTITY_IDS_KEY) or {}).get(entity_id, entity_id) class StepContextError(RuntimeError): diff --git a/src/agent_env/task_step/task_steps/verifiers/rubrics_verifier.py b/src/agent_env/task_step/task_steps/verifiers/rubrics_verifier.py index 8f626c31c..7629ead25 100644 --- a/src/agent_env/task_step/task_steps/verifiers/rubrics_verifier.py +++ b/src/agent_env/task_step/task_steps/verifiers/rubrics_verifier.py @@ -20,7 +20,7 @@ from agent_env.config import get_config from agent_env.config.model import ModelParam from agent_env.providers.sandbox_providers.sandbox import into_container -from agent_env.task_step.context import PromptResponse, StepContextError, TaskStepContext, dual_keyed +from agent_env.task_step.context import PromptResponse, StepContextError, TaskStepContext, dual_keyed, run_entity_id from agent_env.task_step.task_steps.verifiers.judge_utils.judge_output_format import ( SCREENSHOT_GROUNDING, JudgeOutputFormat, @@ -188,12 +188,12 @@ def _bulk_verdicts_correction(eval_prompt: str, bulk: dict) -> str: _MISSING = object() -def _lookup_dotted_path(root: object, path: str, bundle_root: Optional[str] = None) -> object: - node = root +def _lookup_dotted_path(metadata: dict, path: str) -> object: + node = metadata for segment in path.split("."): if isinstance(node, dict): - if segment not in node and bundle_root is not None: - segment = f"{bundle_root}/{segment}" + if segment not in node: + segment = run_entity_id(metadata, segment) if segment not in node: return _MISSING node = node[segment] @@ -220,12 +220,11 @@ def resolve_context_facts(facts: dict[str, str], metadata: dict) -> list[tuple[s """Resolve ``{label: dotted path}`` against ``metadata`` into ``[(label, rendered value)]``, in order. A path walks dicts by key and lists by index; one that does not resolve renders as ``CONTEXT_FACT_NOT_AVAILABLE`` so the judge sees the fact was requested. Never raises. - In a bundle run a key is also found under the id the bundle gave it, ``/``, so a path - naming an env as the task wrote it resolves as it does in the store the task came from.""" + A key the run's task gave an entity another id under (``run_entity_id``, as a bundle does) is also found by the + id the task names it by, so a path naming an env as the task wrote it resolves wherever the task runs.""" rendered: list[tuple[str, str]] = [] - bundle_root = _bundle_root(metadata.get("task_id")) for label, path in facts.items(): - value = _lookup_dotted_path(metadata, path, bundle_root) + value = _lookup_dotted_path(metadata, path) if value is _MISSING: text = CONTEXT_FACT_NOT_AVAILABLE else: @@ -1593,11 +1592,3 @@ def _build_loaded_artifacts_block(entries: list[dict]) -> str: "Do not rely solely on the agent's response text — verify against the actual files when possible.", ] return "\n".join(lines) - - -def _bundle_root(task_id: object) -> Optional[str]: - """The id root of the bundle a run's task came from, its id up to the last segment, or None for a task the - store holds under its own id.""" - if not isinstance(task_id, str) or not is_local_id(task_id): - return None - return task_id.rsplit("/", 1)[0] diff --git a/tst/unit/bundle/materialize_test.py b/tst/unit/bundle/materialize_test.py index 994841a1a..3e61edd24 100644 --- a/tst/unit/bundle/materialize_test.py +++ b/tst/unit/bundle/materialize_test.py @@ -367,6 +367,16 @@ def test_an_agent_is_written_from_its_agent_toml_over_a_store_image(bundle_dir): assert _summary(_run(bundle_dir))[f"{ROOT}/solver"] == (1, True, ()) +def test_the_written_task_records_the_ids_its_bundle_gave_what_it_names(bundle_dir): + _put_image("claude-image") + layout(bundle_dir, {"agents/solver/agent.toml": AGENT_TOML}) + _steps(bundle_dir, [{"id": "agent", "type": "deploy_agent", "env_ids": [], "a2a_agent_id": "solver"}]) + + _run(bundle_dir) + + assert Task.get(f"{ROOT}/t").entity_ids == {"solver": f"{ROOT}/solver"} + + @pytest.mark.parametrize(("toml", "problem"), [ ('image = "claude-image"\nowner = "me"\n', "agent.toml: unknown key 'owner'; an a2a_agent agent takes image, default_env_vars, metadata, type and id"), diff --git a/tst/unit/bundle/resolve_test.py b/tst/unit/bundle/resolve_test.py index 23cec79f7..1b783cebb 100644 --- a/tst/unit/bundle/resolve_test.py +++ b/tst/unit/bundle/resolve_test.py @@ -145,10 +145,19 @@ def test_an_output_is_named_under_its_task_and_later_steps_find_it(make): output = f"{ROOT}/t/tickets-cli" assert [entry.config[0]["cli_artifact_id"], entry.config[1]["artifact_id"], entry.config[2]["cli_artifact_ids"][0]] == [output] * 3 - assert entry.outputs == (Output(EntityKind.ARTIFACT, output, 0, "cli", "cli"),) + assert entry.outputs == (Output(EntityKind.ARTIFACT, output, 0, "cli", "cli", "tickets-cli"),) assert refs(entry) == [(EntityKind.ENV, f"{ROOT}/tickets", None, "tickets")] +def test_a_task_records_each_id_it_rewrote_as_written(make): + entry = resolved(make(tasks={"t": [ + {"id": "deploy", "type": "deploy_env", "env_id": "tickets"}, + {"id": "seed", "type": "load_artifact", "env_id": "tickets", "artifact_id": "shared-artifact"}, + {"id": "cli", "type": "build_mcp_cli", "env_id": "tickets", "cli_artifact_id": "tickets-cli"}, + ]}), "t") + assert entry.entity_ids == {"tickets": f"{ROOT}/tickets", "tickets-cli": f"{ROOT}/t/tickets-cli"} + + def test_an_output_id_a_step_derives_is_named_under_its_task_too(make): entry = resolved(make(tasks={"t": [ {"id": "cli", "type": "build_mcp_cli", "env_id": "tickets", "command_name": "tickets"}, diff --git a/tst/unit/task/entity_ids_test.py b/tst/unit/task/entity_ids_test.py new file mode 100644 index 000000000..af816a248 --- /dev/null +++ b/tst/unit/task/entity_ids_test.py @@ -0,0 +1,61 @@ +"""Task.entity_ids: the ids a bundle gave what a task names, stored with the task and put in each run's metadata.""" + +import pytest + +from agent_env.config.runtime import Config +from agent_env.task import Task +from agent_env.task_step.context import ENTITY_IDS_KEY, TaskStepContext, run_entity_id +from agent_env.task_step.task_step import TaskStep + +IDS = {"world": "@local/~/runs/pn/world"} + + +class _Seen(TaskStep): + """Records the metadata its run reached it with.""" + + type = "entity_ids_test" + entity_refs = () + seen: list[dict] = [] + + @classmethod + def from_dict(cls, data): + return cls(**cls._base_from_dict(data)) + + async def execute(self, context): + _Seen.seen.append(dict(context.metadata)) + return context + + +@pytest.fixture(autouse=True) +def registries(monkeypatch): + steps = {**Config().task_step_registry(), _Seen.type: _Seen} + monkeypatch.setattr(Config, "task_step_registry", lambda self: steps) + monkeypatch.setattr(_Seen, "seen", []) + + +def _task(**kw) -> Task: + return Task(id="t", version=None, steps=[_Seen(id="s", version=None)], **kw) + + +def test_they_are_stored_with_the_task_and_left_out_when_there_are_none(): + assert Task.from_dict(_task(entity_ids=IDS).to_dict()).entity_ids == IDS + assert "entity_ids" not in _task().to_dict() + assert "entity_ids" not in _task(entity_ids={}).to_dict() + + +@pytest.mark.asyncio +async def test_a_run_puts_them_in_its_metadata_for_run_entity_id(local_stores): + await _task(entity_ids=IDS).run(context=TaskStepContext()) + + metadata = _Seen.seen[0] + assert metadata[ENTITY_IDS_KEY] == IDS + assert run_entity_id(metadata, "world") == "@local/~/runs/pn/world" + assert run_entity_id(metadata, "elsewhere") == "elsewhere" + + +@pytest.mark.asyncio +async def test_a_task_stored_as_written_adds_nothing(local_stores): + await _task().run(context=TaskStepContext()) + + assert ENTITY_IDS_KEY not in _Seen.seen[0] + assert run_entity_id(_Seen.seen[0], "world") == "world" diff --git a/tst/unit/task_step/verifiers/test_rubrics_verifier_context_facts_bundle_ids.py b/tst/unit/task_step/verifiers/test_rubrics_verifier_context_facts_bundle_ids.py index 0eb98d64e..989555065 100644 --- a/tst/unit/task_step/verifiers/test_rubrics_verifier_context_facts_bundle_ids.py +++ b/tst/unit/task_step/verifiers/test_rubrics_verifier_context_facts_bundle_ids.py @@ -1,5 +1,5 @@ -"""context_facts_for_judge in a bundle run: a path naming an env as the task wrote it also finds the key the bundle -namespaced, ``/``, within the bundle the run's task came from.""" +"""context_facts_for_judge in a bundle run: a path naming an entity as the task wrote it also finds the key under the +id the run's task gave it (``entity_ids``), as a bundle-written task's run records.""" from __future__ import annotations import pytest @@ -11,38 +11,37 @@ resolve_context_facts, ) +ENV = "@local/~/runs/pn/world" FACTS = {"wf-1 status": "env_trigger_state.world.statuses.wf-1"} -def _resolved(metadata: dict) -> str: - return dict(resolve_context_facts(FACTS, metadata))["wf-1 status"] +def _resolved(metadata: dict, facts: dict = FACTS) -> str: + return next(iter(dict(resolve_context_facts(facts, metadata)).values())) def _state(key: str, status: str) -> dict: return {key: {"statuses": {"wf-1": status}}} -@pytest.mark.parametrize("task_id, key", [ - ("@local/~/runs/pn/task-a", "@local/~/runs/pn/world"), - ("@hf/org/repo/pn/task-a", "@hf/org/repo/pn/world"), -]) -def test_a_bundle_run_finds_the_key_its_bundle_namespaced(task_id, key): - assert _resolved({"task_id": task_id, "env_trigger_state": _state(key, "armed")}) == '"armed"' +def test_a_renamed_env_is_found_by_the_id_the_task_names_it_by(): + assert _resolved({"entity_ids": {"world": ENV}, "env_trigger_state": _state(ENV, "armed")}) == '"armed"' -def test_a_key_as_written_still_wins(): - state = {**_state("world", "fired"), **_state("@local/b/world", "armed")} - assert _resolved({"task_id": "@local/b/task-a", "env_trigger_state": state}) == '"fired"' +def test_a_renamed_step_output_is_found_too(): + snapshot = "@local/~/runs/pn/task-a/snap" + metadata = {"entity_ids": {"snap": snapshot}, "env_changes": {snapshot: {"rows": 3}}} + assert _resolved(metadata, {"rows": "env_changes.snap.rows"}) == "3" -def test_another_bundles_key_is_not_found(): - assert _resolved({"task_id": "@local/b/task-a", "env_trigger_state": _state("@local/other/world", "fired")}) \ - == CONTEXT_FACT_NOT_AVAILABLE +def test_a_key_as_written_still_wins(): + state = {**_state("world", "fired"), **_state(ENV, "armed")} + assert _resolved({"entity_ids": {"world": ENV}, "env_trigger_state": state}) == '"fired"' -@pytest.mark.parametrize("metadata", [{"task_id": "task-a"}, {}], ids=["store task", "no task id"]) -def test_a_task_from_the_store_resolves_only_keys_as_written(metadata): - assert _resolved({**metadata, "env_trigger_state": _state("@local/b/world", "fired")}) == CONTEXT_FACT_NOT_AVAILABLE +@pytest.mark.parametrize("entity_ids", [None, {}, {"other": ENV}], ids=["no map", "empty map", "not in the map"]) +def test_without_a_rename_a_key_resolves_only_as_written(entity_ids): + metadata = {"env_trigger_state": _state(ENV, "fired"), **({"entity_ids": entity_ids} if entity_ids is not None else {})} + assert _resolved(metadata) == CONTEXT_FACT_NOT_AVAILABLE @pytest.mark.asyncio @@ -62,7 +61,7 @@ async def fake_run(*, eval_prompt, **kwargs): ctx = TaskStepContext( prompt_responses=[PromptResponse(prompt_id="p1", response="done", prompt_text="do it", agent_trajectory_object_url="s3://t/1.json")], - metadata={"task_id": "@local/~/runs/pn/task-a", "env_trigger_state": _state("@local/~/runs/pn/world", "armed")}, + metadata={"entity_ids": {"world": ENV}, "env_trigger_state": _state(ENV, "armed")}, ) await v.execute(ctx) From d4cbaf16c6e99cc0d927868d468876050b5d3684 Mon Sep 17 00:00:00 2001 From: Edgar Arakelyan Date: Thu, 12 Nov 2026 12:36:09 -0800 Subject: [PATCH 3/4] fix(bundle): rewrite a task whose stored version predates its entity_ids A bundle run reuses the task version its ledger recorded when the task's inputs are unchanged, so a bundle written before tasks carried entity_ids kept a stored task without them, and its runs still resolved renamed ids as written. The ledger now rewrites a task whose store's latest version doesn't record the entity_ids its bundle gives it, saying so. Digests are unchanged, so no other entity is rewritten and no image rebuilt; a task with no renamed ids is reused as before. Co-Authored-By: Claude Opus 5.5 --- src/agent_env/bundle/ledger.py | 11 +++++++---- tst/unit/bundle/ledger_test.py | 18 ++++++++++++++++-- 2 files changed, 23 insertions(+), 6 deletions(-) diff --git a/src/agent_env/bundle/ledger.py b/src/agent_env/bundle/ledger.py index be1407ceb..f473339a0 100644 --- a/src/agent_env/bundle/ledger.py +++ b/src/agent_env/bundle/ledger.py @@ -104,7 +104,8 @@ def check(self, write: Write, needs: Mapping[tuple[str, str], int]) -> Check: """Compare ``write`` with the latest version the ledger records for it. ``needs`` holds, by (store, id), the version each earlier write it needs left or, in a dry run, would leave; an env or agent is hashed with them, so one written anew rewrites it too.""" - stored = self._stored_version(write) + stored_doc = self._stored(write) + stored = stored_doc["version"] if stored_doc else None digest = self.digest(write, needs) if digest is None: untracked = ("its inputs aren't tracked yet, so it is written every run",) @@ -122,6 +123,8 @@ def check(self, write: Write, needs: Mapping[tuple[str, str], int]) -> Check: reasons.append("new" if row is None else f"its v{recorded} is not in the store") elif stored != recorded: reasons.append(f"the store's latest, v{stored}, wasn't recorded by this bundle") + elif write.kind is BundleKind.TASK and (stored_doc.get("entity_ids") or {}) != write.source.entity_ids: + reasons.append("its stored version doesn't record the ids the bundle gave what it names") if row is not None: reasons.extend(_changes(row, digest)) return Check(write, digest, None if reasons else recorded, tuple(reasons), stored, needs) @@ -214,9 +217,9 @@ def _bundle(self, write: Write) -> str: def _latest(self, store: str, id: str) -> dict | None: return self._find(id, self.collection, Filter.of(store=store, id=id, status="done")) - def _stored_version(self, write: Write) -> int | None: - doc = self._find(write.id, _COLLECTIONS[write.kind.store], Filter.of(id=write.id)) - return doc["version"] if doc else None + def _stored(self, write: Write) -> dict | None: + """The store's latest version of ``write``'s entity.""" + return self._find(write.id, _COLLECTIONS[write.kind.store], Filter.of(id=write.id)) def _find(self, entity_id: str, collection: str, filter: Filter) -> dict | None: """The highest version matching ``filter`` in ``entity_id``'s store, read without creating a local store.""" diff --git a/tst/unit/bundle/ledger_test.py b/tst/unit/bundle/ledger_test.py index 6a5a981fc..e78406eec 100644 --- a/tst/unit/bundle/ledger_test.py +++ b/tst/unit/bundle/ledger_test.py @@ -115,11 +115,13 @@ def _latest_version(collection, id): def _written(write): - """Stands in for the materializer's write: the next version of the entity, in the @local namespace's store.""" + """Stands in for the materializer's write: the next version of the entity, in the @local namespace's store, a + task's with the ids its bundle gave what it names, as Task.put stores them.""" collection = _COLLECTIONS[write.kind.store] latest = local_store().find_one(collection, Filter.of(id=write.id), sort=Sort.by("version")) version = latest["version"] + 1 if latest else 1 - local_store().insert(collection, {"id": write.id, "version": version}) + renamed = write.source.entity_ids if write.kind.store == "task" else {} + local_store().insert(collection, {"id": write.id, "version": version, **({"entity_ids": renamed} if renamed else {})}) return version @@ -293,6 +295,18 @@ def edited_meanwhile(): assert ledger.check(write, {}).reasons == ("the ledger doesn't know what its last version was made from",) +def test_a_task_written_before_it_recorded_its_renamed_ids_is_written_again_with_them(bundle_dir, monkeypatch): + with monkeypatch.context() as earlier: + earlier.setattr(resolve_module.ResolvedEntry, "entity_ids", property(lambda self: {})) + _run(bundle_dir) + + checks = _run(bundle_dir) + + assert checks[f"{ROOT}/t"].reasons == ("its stored version doesn't record the ids the bundle gave what it names",) + assert [id for id, check in checks.items() if not check.unchanged] == [f"{ROOT}/t"] + assert all(check.unchanged for check in _run(bundle_dir).values()) + + def test_a_new_digest_scheme_rewrites_everything_and_says_only_that(bundle_dir, monkeypatch): _run(bundle_dir) monkeypatch.setattr("agent_env.bundle.ledger.SCHEME", ledger_module.SCHEME + 1) From 9a5b7de75de00d311542d40f62eb1a09576c008f Mon Sep 17 00:00:00 2001 From: Edgar Arakelyan Date: Thu, 12 Nov 2026 12:59:30 -0800 Subject: [PATCH 4/4] fix(bundle): entity_ids are the ids a task's own fields were rewritten to, and an earlier orphan is rewritten ResolvedEntry.entity_ids now pairs each reference field of a task's steps as written with the same field as resolved, so it holds exactly the ids the bundle rewrote: its own entities, the outputs its steps write, alias rewrites, and a registration's translation, which all rewrite the config. A reference the plan adds without rewriting a field, as --agent's for a run's solver, no longer enters it, so a run with --agent stores the task as written and the next run reuses it. Outputs no longer need the name their step gave them. A version an interrupted run left unrecorded is adopted only when it records the entity_ids the task gives now, so a task an earlier release wrote and didn't record is written again too. Co-Authored-By: Claude Opus 5.5 --- src/agent_env/bundle/ledger.py | 11 +++++++++-- src/agent_env/bundle/resolve.py | 28 +++++++++++++++++++++------- tst/unit/bundle/ledger_test.py | 13 +++++++++++++ tst/unit/bundle/resolve_test.py | 12 +++++++++++- 4 files changed, 54 insertions(+), 10 deletions(-) diff --git a/src/agent_env/bundle/ledger.py b/src/agent_env/bundle/ledger.py index f473339a0..3a144e990 100644 --- a/src/agent_env/bundle/ledger.py +++ b/src/agent_env/bundle/ledger.py @@ -110,7 +110,8 @@ def check(self, write: Write, needs: Mapping[tuple[str, str], int]) -> Check: if digest is None: untracked = ("its inputs aren't tracked yet, so it is written every run",) return Check(write, None, None, untracked, stored, needs) - orphan = self._orphan(write, stored, digest) + stale = _predates_entity_ids(write, stored_doc) + orphan = None if stale else self._orphan(write, stored, digest) if orphan is not None: return Check(write, digest, orphan, ("written by an interrupted run that didn't record it",), stored, needs, adopted=True) @@ -123,7 +124,7 @@ def check(self, write: Write, needs: Mapping[tuple[str, str], int]) -> Check: reasons.append("new" if row is None else f"its v{recorded} is not in the store") elif stored != recorded: reasons.append(f"the store's latest, v{stored}, wasn't recorded by this bundle") - elif write.kind is BundleKind.TASK and (stored_doc.get("entity_ids") or {}) != write.source.entity_ids: + elif stale: reasons.append("its stored version doesn't record the ids the bundle gave what it names") if row is not None: reasons.extend(_changes(row, digest)) @@ -353,3 +354,9 @@ def _file_sha256(path: Path) -> str: def _now() -> str: return datetime.now(timezone.utc).isoformat() + + +def _predates_entity_ids(write: Write, stored: dict | None) -> bool: + """Whether the store's latest version of task ``write``, written before tasks recorded them, lacks the entity_ids + its bundle gives it now, so reusing it would leave its runs naming those entities as written.""" + return write.kind is BundleKind.TASK and stored is not None and (stored.get("entity_ids") or {}) != write.source.entity_ids diff --git a/src/agent_env/bundle/resolve.py b/src/agent_env/bundle/resolve.py index d933dbd46..686330e62 100644 --- a/src/agent_env/bundle/resolve.py +++ b/src/agent_env/bundle/resolve.py @@ -90,7 +90,6 @@ class Output: step: int # the writing step's index in the task step_id: Any artifact_type: str | None - name: str # as the step names it @dataclass(frozen=True) @@ -105,11 +104,11 @@ class ResolvedEntry: @property def entity_ids(self) -> dict[str, str]: - """Each id the entry's config names that resolving rewrote, as written, to the id it names now - (``Task.entity_ids``): one of the bundle's entities by its name, an output a step writes by the name it gave.""" - renamed = {output.name: output.id for output in self.outputs} - renamed.update((ref.local.name, ref.id) for ref in self.references if isinstance(ref.local, BundleEntry)) - return {name: id for name, id in renamed.items() if name != id} + """For a task, each id a field of its steps names as written that ``config`` names by another id, to that id + (``Task.entity_ids``); a reference the plan adds without rewriting a field, as ``--agent``'s, adds nothing.""" + if self.entry.kind is not BundleKind.TASK: + return {} + return _rewritten_ids(self.entry.config, self.config) @dataclass(frozen=True) @@ -401,7 +400,7 @@ def _outputs(self, entry: BundleEntry, declared: list[tuple[int, dict, type]]) - "root, which only a bundle run from the Hugging Face Hub writes under its own") continue site.rewrite(output_id) - outputs[kind, name] = Output(kind, output_id, index, step.get("id"), site.ref.artifact_type, name) + outputs[kind, name] = Output(kind, output_id, index, step.get("id"), site.ref.artifact_type) return outputs def _depends_on(self, entry: BundleEntry, steps: list[dict]) -> list[list[str] | None]: @@ -570,3 +569,18 @@ def _strings(value: Any, path: str = "") -> Iterator[tuple[str, str]]: elif isinstance(value, list): for i, item in enumerate(value): yield from _strings(item, f"{path}[{i}]") + + +def _rewritten_ids(written: list[dict], resolved: list[dict]) -> dict[str, str]: + """Each id a reference field of ``written``'s steps holds that the same field of ``resolved``'s holds another id + in place of, to that id.""" + registry = get_task_step_registry() + renamed = {} + for before, after in zip(written, resolved): + cls = registry.get(before.get("type")) + if cls is None or not cls.entity_refs: + continue + now = {site.path: site.value for site in ref_sites(cls.entity_refs, after)} + renamed.update((site.value, now[site.path]) for site in ref_sites(cls.entity_refs, before) + if site.path in now and now[site.path] != site.value) + return renamed diff --git a/tst/unit/bundle/ledger_test.py b/tst/unit/bundle/ledger_test.py index e78406eec..99a132006 100644 --- a/tst/unit/bundle/ledger_test.py +++ b/tst/unit/bundle/ledger_test.py @@ -538,6 +538,19 @@ def test_a_version_an_interrupted_run_wrote_is_kept_and_recorded_not_written_aga assert [row["status"] for row in local_store().query(LEDGER_COLLECTION, Filter.of(id=GREETING))] == ["done"] +def test_a_task_an_interrupted_earlier_run_wrote_without_its_renamed_ids_is_written_again(bundle_dir, monkeypatch): + plan = plan_of(bundle_dir) + task = next(w for w in plan.writes if w.id == f"{ROOT}/t") + with monkeypatch.context() as earlier: + earlier.setattr(resolve_module.ResolvedEntry, "entity_ids", property(lambda self: {})) + _interrupted_after_its_write(Ledger.for_plan(plan), task, earlier) + + check = Ledger.for_plan(plan).check(task, {}) + + assert not check.unchanged and not check.adopted + assert check.reasons == ("the store's latest, v1, wasn't recorded by this bundle",) + + @pytest.mark.parametrize("since", ["another-write", "an-edit"]) def test_an_interrupted_runs_version_is_written_over_once_the_store_or_the_files_have_moved_on( bundle_dir, monkeypatch, since, diff --git a/tst/unit/bundle/resolve_test.py b/tst/unit/bundle/resolve_test.py index 1b783cebb..824adb98a 100644 --- a/tst/unit/bundle/resolve_test.py +++ b/tst/unit/bundle/resolve_test.py @@ -2,6 +2,7 @@ import json import random +from dataclasses import replace import pytest @@ -145,7 +146,7 @@ def test_an_output_is_named_under_its_task_and_later_steps_find_it(make): output = f"{ROOT}/t/tickets-cli" assert [entry.config[0]["cli_artifact_id"], entry.config[1]["artifact_id"], entry.config[2]["cli_artifact_ids"][0]] == [output] * 3 - assert entry.outputs == (Output(EntityKind.ARTIFACT, output, 0, "cli", "cli", "tickets-cli"),) + assert entry.outputs == (Output(EntityKind.ARTIFACT, output, 0, "cli", "cli"),) assert refs(entry) == [(EntityKind.ENV, f"{ROOT}/tickets", None, "tickets")] @@ -158,6 +159,15 @@ def test_a_task_records_each_id_it_rewrote_as_written(make): assert entry.entity_ids == {"tickets": f"{ROOT}/tickets", "tickets-cli": f"{ROOT}/t/tickets-cli"} +def test_a_reference_added_without_rewriting_a_field_is_not_a_renamed_id(make): + bundle = make(tasks={"t": [{"id": "deploy", "type": "deploy_env", "env_id": "tickets"}]}) + entry = resolved(bundle, "t") + solver = next(e for e in bundle.entries if e.name == "solver") + swapped = replace(entry, references=(*entry.references, + Reference(EntityKind.AGENT, solver.id, None, solver, "--agent 'solver'", None))) + assert swapped.entity_ids == entry.entity_ids == {"tickets": f"{ROOT}/tickets"} + + def test_an_output_id_a_step_derives_is_named_under_its_task_too(make): entry = resolved(make(tasks={"t": [ {"id": "cli", "type": "build_mcp_cli", "env_id": "tickets", "command_name": "tickets"},