From be1781f2fcaeecb0a414f5f9670929d81db5ec1e Mon Sep 17 00:00:00 2001 From: Austin Jian Date: Mon, 19 Oct 2026 21:27:26 -0400 Subject: [PATCH 1/2] fix(orchestrator): stop hands-off paid work once the package is unreachable A blocked or failed paid prerequisite of the package is final in hands-off mode, so further paid stages (such as the world generation) can no longer lead to a preview. Refuse them and finish the run as failed. Observed on a real upload whose pi3x was blocked by the shared ledger while clean and avatars continued. --- docs/automatic-uploads.md | 11 ++++++++ orchestrator/run_policy.py | 39 +++++++++++++++++++++++++++- scripts/tests/test_hands_off_runs.py | 28 +++++++++++++++++++- 3 files changed, 76 insertions(+), 2 deletions(-) diff --git a/docs/automatic-uploads.md b/docs/automatic-uploads.md index 39cbd4d..949da10 100644 --- a/docs/automatic-uploads.md +++ b/docs/automatic-uploads.md @@ -97,6 +97,17 @@ change the submitted run through extra stage flags or automatically retry a paid Existing provider ledgers and unresolved-operation guards continue to apply. A known provider operation must be recovered separately rather than submitted again. +Once a paid stage that the package depends on fails or is blocked, the run cannot produce a +preview, because hands-off mode never repeats a paid stage. The next paid launch is then +refused and the run finishes `failed`, so it does not spend on a world generation or avatars +that can no longer be packaged. Stages already running finish and keep their outputs. + +The shared stage ledger counts executions per source SHA-256 across all runs, including +historical imports. A second execution needs an explicit new hypothesis, which a hands-off run +cannot supply. Uploading a video that was already processed therefore blocks at its first +previously executed paid stage. Use a source that has not been processed before. An upload +does not reuse earlier outputs or worlds. + When the agent needs a decision, exhausts its useful work, or stalls, the run records a final failure with its reason instead of waiting for a question in the UI. Existing outputs and evidence remain available. A readable packaged scene may be delivered as an **unaccepted diff --git a/orchestrator/run_policy.py b/orchestrator/run_policy.py index 1090a41..7c867f5 100644 --- a/orchestrator/run_policy.py +++ b/orchestrator/run_policy.py @@ -11,7 +11,7 @@ from typing import Any from orchestrator.journal import Journal -from orchestrator.steps import STEPS, base_step +from orchestrator.steps import STEPS, base_step, stage_graph def hands_off(options: dict[str, Any]) -> bool: @@ -53,6 +53,43 @@ def admission_problem(context, step: str, extra: list[str], journal: Journal, at f"Hands-off paid step {step} already has an attempt or retained result. " "Automatic paid retries are disabled; retain its outputs and provider receipt." ) + if lost := lost_package_prerequisite(context, journal): + return ( + f"Hands-off paid step {step} was not started: required paid step {lost} cannot " + "run again, so no packaged preview can follow and further paid work is wasted." + ) + return None + + +def lost_package_prerequisite(context, journal: Journal) -> str | None: + """A paid prerequisite of the package that failed or was blocked, if there is one. + + Hands-off runs never repeat a paid step, so such a failure is final: the package, and with + it any preview, can no longer be produced. Unpaid steps can still be rerun and do not count. + """ + stages, deps = stage_graph(context.options) + package = "package_people" if "package_people" in stages else "package" + required, pending = set(), [package] + while pending: + name = pending.pop() + if name not in required: + required.add(name) + pending.extend(deps.get(name, [])) + outcomes = { + entry.data.get("step"): entry.data.get("status") + for entry in journal.entries() + if entry.kind == "step.finished" + } + for name, record in stage_records(context.run_dir).items(): + if isinstance(record, dict) and record.get("status") == "failed": + outcomes.setdefault(name, "failed") + for name in stages: + if ( + name in required + and STEPS[base_step(name)].paid + and outcomes.get(name) in {"failed", "blocked"} + ): + return f"{name} ({outcomes[name]})" return None diff --git a/scripts/tests/test_hands_off_runs.py b/scripts/tests/test_hands_off_runs.py index 1183677..f13a9dd 100644 --- a/scripts/tests/test_hands_off_runs.py +++ b/scripts/tests/test_hands_off_runs.py @@ -14,7 +14,7 @@ from orchestrator.agent.contracts import AgentResult from orchestrator.agent.harness import HarnessAgent, HarnessOutcome, HarnessPolicy from orchestrator.journal import Journal -from orchestrator.run_policy import delivery_metadata +from orchestrator.run_policy import admission_problem, delivery_metadata from orchestrator.session import RunSession from orchestrator.steps import RunOptions from orchestrator.supervisor import runs_needing_work, work_on @@ -158,6 +158,32 @@ def test_paid_retry_is_refused_after_restart_and_resume(run, monkeypatch, record popen.assert_not_called() +@pytest.mark.parametrize("status", ["blocked", "failed"]) +def test_lost_package_prerequisite_stops_further_paid_work(run, monkeypatch, status): + journal = run.journal() + journal.append("step.started", step="pi3x", attempt="one", number=1) + journal.append("step.finished", step="pi3x", attempt="one", status=status) + popen = Mock( + side_effect=AssertionError("paid work launched after the package became impossible") + ) + monkeypatch.setattr(cli.subprocess, "Popen", popen) + assert cli.main(["step", "--wait", "lhm_frozen"]) == 2 + finished = Journal(run.run_dir).entries()[-1] + assert finished.kind == "run.finished" and finished.data["status"] == "failed" + assert "pi3x" in finished.data["summary"] + popen.assert_not_called() + + +def test_lost_prerequisite_check_ignores_unpaid_and_unrelated_failures(run): + journal = run.journal() + # Unpaid steps can be rerun; clean does not feed the package of a run without a world. + journal.append("step.finished", step="frame_align", attempt="one", status="failed") + journal.append("step.finished", step="clean", attempt="one", status="failed") + assert admission_problem(run, "lhm_frozen", [], journal) is None + (run.run_dir / "state.json").write_text('{"stages":{"lhm_frozen":{"status":"failed"}}}') + assert "lhm_frozen" in admission_problem(run, "pi3x", [], journal) + + def test_existing_detached_attempt_can_claim_execution_once(run): journal = run.journal() journal.append( From ebaed0f68cd6eb880cbdbfa7830f096be6c45395 Mon Sep 17 00:00:00 2001 From: Austin Jian Date: Tue, 20 Oct 2026 14:11:09 -0400 Subject: [PATCH 2/2] fix(orchestrator): hold hands-off paid work while an unpaid prerequisite is failed A real upload with no fully visible person failed person_prep, an unpaid step the package needs, while the world generation would still have launched. Paid launches are now held (not terminal) until that step succeeds on a rerun. --- docs/automatic-uploads.md | 6 +++++- orchestrator/cli.py | 5 +++++ orchestrator/run_policy.py | 28 ++++++++++++++++++++++------ scripts/tests/test_hands_off_runs.py | 20 +++++++++++++++++++- 4 files changed, 51 insertions(+), 8 deletions(-) diff --git a/docs/automatic-uploads.md b/docs/automatic-uploads.md index 949da10..7c3ae91 100644 --- a/docs/automatic-uploads.md +++ b/docs/automatic-uploads.md @@ -100,7 +100,11 @@ operation must be recovered separately rather than submitted again. Once a paid stage that the package depends on fails or is blocked, the run cannot produce a preview, because hands-off mode never repeats a paid stage. The next paid launch is then refused and the run finishes `failed`, so it does not spend on a world generation or avatars -that can no longer be packaged. Stages already running finish and keep their outputs. +that can no longer be packaged. Stages already running finish and keep their outputs. While an +unpaid package prerequisite (such as `person_prep`) is failed, paid launches are held instead: +rerunning that step is still admitted, and paid work resumes only once it succeeds. Unpaid steps +use the same fixed defaults on a rerun, so a failure caused by the footage itself, such as no +fully visible person, repeats and the run ends without spending on generation. The shared stage ledger counts executions per source SHA-256 across all runs, including historical imports. A second execution needs an explicit new hypothesis, which a hands-off run diff --git a/orchestrator/cli.py b/orchestrator/cli.py index efe68c2..c7fca50 100644 --- a/orchestrator/cli.py +++ b/orchestrator/cli.py @@ -31,6 +31,7 @@ delivery_metadata, finish_hands_off, hands_off, + unpaid_prerequisite_problem, ) from orchestrator.steps import ( SAME_AS, @@ -295,6 +296,10 @@ def admit_step( finish_hands_off(context, journal, problem) print(problem, file=sys.stderr) return 2 + if not args.attempt and (held := unpaid_prerequisite_problem(context, args.step, journal)): + # Not terminal: the unpaid prerequisite may still succeed when rerun. + print(held, file=sys.stderr) + return 2 if context.retained_delivery is not None: problem = retained_delivery_problem(context, args, journal) if problem: diff --git a/orchestrator/run_policy.py b/orchestrator/run_policy.py index 7c867f5..03d9756 100644 --- a/orchestrator/run_policy.py +++ b/orchestrator/run_policy.py @@ -53,7 +53,7 @@ def admission_problem(context, step: str, extra: list[str], journal: Journal, at f"Hands-off paid step {step} already has an attempt or retained result. " "Automatic paid retries are disabled; retain its outputs and provider receipt." ) - if lost := lost_package_prerequisite(context, journal): + if lost := failed_package_prerequisite(context, journal, paid=True): return ( f"Hands-off paid step {step} was not started: required paid step {lost} cannot " "run again, so no packaged preview can follow and further paid work is wasted." @@ -61,11 +61,27 @@ def admission_problem(context, step: str, extra: list[str], journal: Journal, at return None -def lost_package_prerequisite(context, journal: Journal) -> str | None: - """A paid prerequisite of the package that failed or was blocked, if there is one. +def unpaid_prerequisite_problem(context, step: str, journal: Journal) -> str | None: + """Hold paid work while an unpaid prerequisite of the package is failed. - Hands-off runs never repeat a paid step, so such a failure is final: the package, and with - it any preview, can no longer be produced. Unpaid steps can still be rerun and do not count. + Unlike a paid failure this is not final: rerunning the unpaid step is admitted, and paid + work resumes once it succeeds. A failure that repeats leaves the run without runnable work. + """ + if not hands_off(context.options) or not STEPS[base_step(step)].paid: + return None + if failed := failed_package_prerequisite(context, journal, paid=False): + return ( + f"Hands-off paid step {step} was not started: required step {failed} has not " + "succeeded, so no packaged preview can follow yet. Rerun it first." + ) + return None + + +def failed_package_prerequisite(context, journal: Journal, *, paid: bool) -> str | None: + """A paid (or unpaid) prerequisite of the package whose latest outcome is a failure. + + Hands-off runs never repeat a paid step, so a paid failure is final: the package, and with + it any preview, can no longer be produced. An unpaid step can still be rerun. """ stages, deps = stage_graph(context.options) package = "package_people" if "package_people" in stages else "package" @@ -86,7 +102,7 @@ def lost_package_prerequisite(context, journal: Journal) -> str | None: for name in stages: if ( name in required - and STEPS[base_step(name)].paid + and STEPS[base_step(name)].paid == paid and outcomes.get(name) in {"failed", "blocked"} ): return f"{name} ({outcomes[name]})" diff --git a/scripts/tests/test_hands_off_runs.py b/scripts/tests/test_hands_off_runs.py index f13a9dd..ef54587 100644 --- a/scripts/tests/test_hands_off_runs.py +++ b/scripts/tests/test_hands_off_runs.py @@ -14,7 +14,11 @@ from orchestrator.agent.contracts import AgentResult from orchestrator.agent.harness import HarnessAgent, HarnessOutcome, HarnessPolicy from orchestrator.journal import Journal -from orchestrator.run_policy import admission_problem, delivery_metadata +from orchestrator.run_policy import ( + admission_problem, + delivery_metadata, + unpaid_prerequisite_problem, +) from orchestrator.session import RunSession from orchestrator.steps import RunOptions from orchestrator.supervisor import runs_needing_work, work_on @@ -184,6 +188,20 @@ def test_lost_prerequisite_check_ignores_unpaid_and_unrelated_failures(run): assert "lhm_frozen" in admission_problem(run, "pi3x", [], journal) +def test_failed_unpaid_prerequisite_holds_paid_work_until_it_succeeds(run, monkeypatch): + journal = run.journal() + journal.append("step.finished", step="person_prep", attempt="one", status="failed") + popen = Mock(side_effect=AssertionError("paid work launched without its prerequisite")) + monkeypatch.setattr(cli.subprocess, "Popen", popen) + assert cli.main(["step", "--wait", "lhm_frozen"]) == 2 + # Held, not finished: the unpaid step may still be rerun. + assert Journal(run.run_dir).finished() is None + popen.assert_not_called() + assert unpaid_prerequisite_problem(run, "person_prep", journal) is None + journal.append("step.finished", step="person_prep", attempt="two", status="succeeded") + assert unpaid_prerequisite_problem(run, "lhm_frozen", journal) is None + + def test_existing_detached_attempt_can_claim_execution_once(run): journal = run.journal() journal.append(