diff --git a/Makefile b/Makefile index 5c398134e9..7f5278cb46 100644 --- a/Makefile +++ b/Makefile @@ -191,13 +191,14 @@ build-check: test-quick: build-check vet fmt-check test-packed-manual changelog-check manual-gates test-laws test-carried-help test-belt-pages -# The shell runners are executable infrastructure that Go's package walk cannot +# The script runners are executable infrastructure that Go's package walk cannot # discover, so their acceptance scripts are named here. The benchmark clone # regression is kept beside the helper it drives. They are touched-only, like # the Go packages: `pr-ready` runs them when the change touches scripts/, the # covered benchmark paths, or this Makefile. Together they take about forty # seconds and most pull requests never go near them. test-tooling: + python3 scripts/hosted-evidence_test.py bash scripts/touched-packages_test.sh bash scripts/touched-verdict_test.sh bash scripts/one-suite_test.sh diff --git a/docs/changes/unreleased/1816-hosted-outcome-evidence.md b/docs/changes/unreleased/1816-hosted-outcome-evidence.md new file mode 100644 index 0000000000..ce6f3a593a --- /dev/null +++ b/docs/changes/unreleased/1816-hosted-outcome-evidence.md @@ -0,0 +1,9 @@ +--- +kind: fixed +title: hosted acceptance checks actual task endings and retains diagnostics +pr: 1816 +surface: [build] +invalidates: + - "Hosted acceptance treated landed files as successful task completion and could print ALL PASS while a task's durable ending was failed. Completion now requires all three handed-off tasks to end done, without changing the existing file/UI assertions or five-minute guards." + - "A successful hosted drive deleted the full outcome and action records before they could explain a later failed-task report. Private sanitized phase and final records now survive cleanup, model-call metadata is kept inside the owned home without request/response bodies, and externally bound profiles are handled by the application without the drive reading or copying their credential file." +--- diff --git a/scripts/hosted-drive.sh b/scripts/hosted-drive.sh index 96076e976c..5e25d3f94e 100755 --- a/scripts/hosted-drive.sh +++ b/scripts/hosted-drive.sh @@ -4,7 +4,7 @@ # screen and the folder back. A surface change is accepted on this, never on a # test with a stand-in agent: what a stand-in proves is what the stand-in does. # -# scripts/hosted-drive.sh [folder to keep failed screens in] +# scripts/hosted-drive.sh [folder to keep evidence in] # # IT MAKES REAL MODEL CALLS, so it is never part of `make check` or any default # target. Run it by hand, after `make build`, when a change touches the run, the @@ -12,15 +12,20 @@ # # WHAT IT NEEDS: tmux, git, python3, and a key the product can find, either in # the environment or in your profile's config file. Everything it makes is -# thrown away: a home of its own, a repository of its own, and a tmux server on +# isolated: a home of its own, a repository of its own, and a tmux server on # a socket of its own. It never writes to your own home folder's state. +# Outcome, action and provider records are kept in the evidence folder even +# after a pass, so a later diagnosis does not depend on a deleted home. # -# YOUR KEY: the drive copies your profile's key into a private temporary home, +# YOUR KEY: when CODEAF_PROFILE_DIR is set, the application uses that bound +# profile and the drive never reads or copies its credential file. Otherwise +# the drive copies your profile's key into a private temporary home, # readable by you alone, and removes that copy on every exit: a pass, a failed # check, an interrupt. Nothing else the product writes there carries the key. # A screen kept from a failed check has any key the drive knows struck out. It # knows a key by a field whose name says so, or by an environment variable that -# ends in _API_KEY. A credential kept under any other name it cannot know, so +# ends in _API_KEY. Bound-profile records mask common bearer/OpenRouter key +# spellings without reading that profile. Other spellings cannot be known, so # kept screens are private files: read them before you share them. # # WHAT IT ENDS, and how it chooses: its own tmux server, by its own socket's @@ -39,6 +44,7 @@ # - The run works in one copy of its own, kept under the conversation. # - The second hand-off joins the first and shows on the side list. # 2. The run ends and its work comes home. +# - Both handed-off tasks ended done, rather than failed or still running. # - One new commit holds the run's files. # - Your unsaved file is untouched and still unsaved. # - The run's copy is given back. @@ -47,6 +53,7 @@ # - No row on the side list is still running. # 3. A task handed off after the run has ended is a run of its own, and the # first run stays readable. +# - The later handed-off task ended done. # - The first run's record is kept beside the live one. # - A press on the first run's row opens that task's page over the # conversation, with the task as it was given. @@ -59,12 +66,14 @@ # shellcheck disable=SC2329 set -u -BIN=${1:?usage: scripts/hosted-drive.sh [folder for failed screens]} +BIN=${1:?usage: scripts/hosted-drive.sh [folder for evidence]} [ -x "$BIN" ] || { echo "no binary at $BIN: run make build first"; exit 2; } for tool in tmux git python3; do command -v "$tool" >/dev/null || { echo "this drive needs $tool"; exit 2; } done BIN=$(cd "$(dirname "$BIN")" && pwd -P)/$(basename "$BIN") +EVIDENCE_HELPER=$(cd "$(dirname "$0")" && pwd -P)/hosted-evidence.py +[ -f "$EVIDENCE_HELPER" ] || { echo "missing hosted-evidence.py beside the drive"; exit 2; } umask 077 OUT=${2:-$(mktemp -d /tmp/codeaf-drive-screens.XXXXXX)} mkdir -p "$OUT" @@ -73,9 +82,14 @@ HOME_DIR=$(mktemp -d /tmp/codeaf-drive-home.XXXXXX) WORK=$(mktemp -d /tmp/codeaf-drive-work.XXXXXX) # mktemp -d makes a folder only its owner can enter on every system this runs # on; said again here because a key is about to be copied into it. -chmod 700 "$HOME_DIR" +chmod 700 "$HOME_DIR" "$OUT" +EVIDENCE_ARGS=(--home "$HOME_DIR") +if [ -z "${CODEAF_PROFILE_DIR:-}" ]; then + EVIDENCE_ARGS+=(--copied-config "$HOME_DIR/config.json") +fi fails=0 passed=no +final_kept=no say() { printf '%s\n' "$*"; } t() { tmux -L "$SOCK" "$@"; } @@ -87,34 +101,24 @@ import sys sys.stdout.write(''.join('*' if 0xe000 <= ord(c) <= 0xf8ff else c for c in sys.stdin.read()))" } -# strike_keys removes every key the drive knows from what passes through it: -# the ones in the copied profile and the ones in the environment. The drive -# opens no page that draws a key; this is for the day some page does. +# A bound credential file never reaches this redactor. The legacy mode names +# only its own temporary copy; common bearer/OpenRouter spellings are masked +# without opening the externally bound profile. strike_keys() { - python3 -c " -import json, os, sys -known = set() -def walk(node): - if isinstance(node, dict): - for name, value in node.items(): - if isinstance(value, str) and 'key' in name.lower() and len(value) > 8: - known.add(value) - else: - walk(value) - elif isinstance(node, list): - for value in node: - walk(value) -try: - walk(json.load(open(sys.argv[1]))) -except Exception: - pass -for name, value in os.environ.items(): - if name.endswith('_API_KEY') and len(value) > 8: - known.add(value) -text = sys.stdin.read() -for value in known: - text = text.replace(value, '[key struck out]') -sys.stdout.write(text)" "$HOME_DIR/config.json" + python3 "$EVIDENCE_HELPER" redact "${EVIDENCE_ARGS[@]}" +} + +# Capture before another hand-off rotates the store, and before cleanup gives +# back the home. A failed run retains the same durable fields as a clean one. +capture_evidence() { + python3 "$EVIDENCE_HELPER" snapshot --output "$OUT/$1" "${EVIDENCE_ARGS[@]}" || return 1 + screen | strike_keys >"$OUT/$1/screen.txt" 2>/dev/null + [ "$1" != final ] || final_kept=yes + return 0 +} + +terminal_tasks() { + python3 "$EVIDENCE_HELPER" terminal --home "$HOME_DIR" --ids "$@" } pass() { say " PASS $*"; } @@ -214,13 +218,20 @@ holders_of() { return 0 } -# cleanup runs on EVERY exit. The key's copy goes first and goes always; the +# cleanup runs on EVERY exit. Sanitized evidence is kept before the home +# goes away. The key's own temporary copy is removed on every exit; the # rest of the home may stay for reading when a check failed. # # THE ENGINE IS ENDED BEFORE ANYTHING IS REMOVED, because it is found through a # socket that lives in the home. A clean pass then removes what the drive made, # by the names it made them under and nothing else. cleanup() { + local status=$? + if [ "$final_kept" != yes ] && ! capture_evidence final; then + say " FAIL diagnostic evidence could not be kept" + passed=no + [ "$status" -ne 0 ] || status=1 + fi rm -f "$HOME_DIR/config.json" t kill-server 2>/dev/null end_engine @@ -228,6 +239,8 @@ cleanup() { case "$HOME_DIR" in /tmp/codeaf-drive-home.*) rm -rf "$HOME_DIR" ;; esac case "$WORK" in /tmp/codeaf-drive-work.*) rm -rf "$WORK" ;; esac fi + trap - EXIT + exit "$status" } trap cleanup EXIT trap 'exit 130' INT TERM @@ -243,14 +256,14 @@ first_on_the_list() { screen | cut -c119- | grep -q 'add Mul and Div'; } # only one spelling reads an open column as closed on the other system, and the # key pressed to open it closes it instead. column_open() { screen | grep -Eq '│ Tasks [0-9]+.*(alt|opt)\+l'; } -first_run_home() { [ -f "$WORK/README.md" ] && [ -f "$WORK/muldiv.go" ]; } +first_run_home() { [ -f "$WORK/README.md" ] && [ -f "$WORK/muldiv.go" ] && terminal_tasks 1 2; } one_new_commit() { [ "$(git -C "$WORK" log --oneline | wc -l | tr -d ' ')" = "2" ]; } my_file_untouched() { [ "$(cat "$WORK/notes.txt")" = "mine, unsaved" ] && only_my_file; } copy_given_back() { [ -z "$(copies)" ]; } no_task_branch() { [ -z "$(git -C "$WORK" branch --list 'task/*')" ]; } card_says_merged() { screen | grep -q '· merged' && ! screen | grep -q 'branch kept'; } nothing_running() { ! screen | cut -c119- | grep -Eq 'Running [1-9][0-9]*'; } -second_run_home() { grep -q 'func Neg' "$WORK"/*.go 2>/dev/null; } +second_run_home() { grep -q 'func Neg' "$WORK"/*.go 2>/dev/null && terminal_tasks 3; } first_record_kept() { ls "$HOME_DIR"/v3/projects/*/*/plandb.db.1 >/dev/null 2>&1; } page_drawn() { screen | grep -q 'transcript · work esc/← main'; } page_is_the_first_runs() { page_drawn && screen | grep -q '› add Mul and Div with table tests'; } @@ -259,7 +272,9 @@ ended_run_answers() { screen | grep -q '· this task has finished — say it to draft_is_back() { screen | grep -q '› my draft stays'; } profile=${CODEAF_HOME:-$HOME/.codeaf}/config.json -[ -f "$profile" ] && cp "$profile" "$HOME_DIR/config.json" +if [ -z "${CODEAF_PROFILE_DIR:-}" ] && [ -f "$profile" ]; then + cp "$profile" "$HOME_DIR/config.json" +fi ( cd "$WORK" || exit 1 git init -q -b work @@ -270,8 +285,11 @@ profile=${CODEAF_HOME:-$HOME/.codeaf}/config.json echo 'mine, unsaved' >notes.txt ) say "home $HOME_DIR folder $WORK" +# The application normally logs beside its profile. Pin this drive's metadata +# to its own home so bound-profile calls survive in the same evidence walk; +# request and response bodies are deliberately left off. t new-session -d -s x -x 170 -y 50 \ - "cd '$WORK' && CODEAF_HOME='$HOME_DIR' CODEAF_TASK_BELT=bash CODEAF_TELEMETRY=off '$BIN' chat; sleep 3600" + "cd '$WORK' && CODEAF_HOME='$HOME_DIR' CODEAF_CALL_LOG='$HOME_DIR/logs/calls.jsonl' CODEAF_CALL_LOG_BODIES=off CODEAF_TASK_BELT=bash CODEAF_TELEMETRY=off '$BIN' chat; sleep 3600" sleep 8 # A home with no profile opens first-run setup; esc leaves it for the conversation. screen | grep -q 'Daily limit' && { @@ -302,6 +320,8 @@ check "the second hand-off shows on the side list" second_on_the_list say "2. the run ends and its work comes home" wait_for 60 5 first_run_home || fail "the run did not end inside five minutes" sleep 12 +capture_evidence first-run || fail "first-run evidence could not be kept" +check "both first hand-offs ended successfully" terminal_tasks 1 2 check "one new commit holds the run's files" one_new_commit check "your unsaved file is untouched" my_file_untouched check "the copy is given back" copy_given_back @@ -313,6 +333,8 @@ say "3. a hand-off after the run has ended is a run of its own, and the first st typed '/task add a Neg function with a table test' wait_for 60 5 second_run_home || fail "the second run did not end inside five minutes" sleep 12 +capture_evidence second-run || fail "second-run evidence could not be kept" +check "the later hand-off ended successfully" terminal_tasks 3 check "the first run's record was kept beside the live one" first_record_kept t send-keys -t x -l 'my draft stays' sleep 1 @@ -333,6 +355,7 @@ t send-keys -t x Escape sleep 2 check "esc returns to the conversation with your draft as you left it" draft_is_back +capture_evidence final || fail "final evidence could not be kept" say "" if [ "$fails" -eq 0 ]; then say "ALL PASS" diff --git a/scripts/hosted-evidence.py b/scripts/hosted-evidence.py new file mode 100644 index 0000000000..0877b8d15a --- /dev/null +++ b/scripts/hosted-evidence.py @@ -0,0 +1,154 @@ +#!/usr/bin/env python3 +"""Check durable hosted outcomes and keep private diagnostics before cleanup.""" + +import argparse +import json +import os +from pathlib import Path +import re +import sqlite3 +import sys + + +def secrets(copied_config=None): + known = {value for name, value in os.environ.items() + if name.endswith("_API_KEY") and len(value) > 8} + + def walk(value): + if isinstance(value, dict): + for name, child in value.items(): + if isinstance(child, str) and "key" in name.lower() and len(child) > 8: + known.add(child) + else: + walk(child) + elif isinstance(value, list): + for child in value: + walk(child) + + # A bound profile is deliberately never opened by the drive. Only the + # legacy mode's own temporary credential copy can be supplied here. + if copied_config and Path(copied_config).is_file(): + walk(json.loads(Path(copied_config).read_text())) + return known + + +def redact(text, known): + for value in known: + text = text.replace(value, "[key struck out]") + text = re.sub(r"(?i)(Bearer\s+)[^\s\"\\]+", r"\1[key struck out]", text) + return re.sub(r"sk-or-[A-Za-z0-9_-]+", "[key struck out]", text) + + +def owned_files(home): + # A worker-created link must never turn an evidence walk into a read of + # another profile. Config files themselves are never evidence. + for directory, dirs, files in os.walk(home, followlinks=False): + dirs[:] = [name for name in dirs if not (Path(directory) / name).is_symlink()] + for name in files: + path = Path(directory) / name + if name != "config.json" and not path.is_symlink(): + yield path + + +def outcomes(home): + latest = {} + for path in owned_files(home): + if path.name != "tasks.jsonl": + continue + body = path.read_text() + lines = body.splitlines() + for index, line in enumerate(lines): + try: + row = json.loads(line) + except json.JSONDecodeError: + # A live append may be between writes. Do not accept an + # earlier done row while a newer ending is still being written. + if index == len(lines) - 1 and not body.endswith("\n"): + raise ValueError("incomplete task journal append") + raise + if not isinstance(row, dict) or not isinstance(row.get("id"), str): + raise ValueError("task journal row has no string id") + latest[(row.get("sessionId", ""), str(row["id"]))] = row + return list(latest.values()) + + +def successful(home, ids): + rows = outcomes(home) + for task_id in ids: + matches = [row for row in rows if row["id"] == task_id] + if len(matches) != 1 or matches[0].get("status") != "done": + return False + return True + + +def snapshot(home, output, known): + output.mkdir(parents=True, exist_ok=True, mode=0o700) + errors = [] + retained = [] + for path in owned_files(home): + relative = path.relative_to(home) + if path.suffix in {".json", ".jsonl", ".log", ".md", ".txt"}: + try: + # A diagnostic must not accidentally copy an unbounded tool + # artifact. Report its omission alongside the durable outcome. + if path.stat().st_size > 16 * 1024 * 1024: + errors.append({"file": str(relative), "error": "exceeds sixteen-MiB evidence bound"}) + continue + text = path.read_text(errors="replace") + target = output / "records" / relative + target.parent.mkdir(parents=True, exist_ok=True) + target.write_text(redact(text, known)) + retained.append(str(relative)) + except OSError as error: + errors.append({"file": str(relative), "error": str(error)}) + elif re.fullmatch(r"plandb\.db(?:\.\d+)?", path.name): + connection = None + try: + connection = sqlite3.connect(path.resolve().as_uri() + "?mode=ro", uri=True, timeout=1) + connection.row_factory = sqlite3.Row + rows = [dict(row) for row in connection.execute("SELECT * FROM tasks")] + target = output / "sqlite" / Path(str(relative) + ".json") + target.parent.mkdir(parents=True, exist_ok=True) + target.write_text(redact(json.dumps(rows, indent=2), known)) + retained.append(str(relative)) + except (OSError, sqlite3.Error) as error: + errors.append({"file": str(relative), "error": str(error)}) + finally: + if connection: + connection.close() + try: + rows = outcomes(home) + except (OSError, ValueError, KeyError) as error: + rows = [] + errors.append({"file": "task outcomes", "error": str(error)}) + summary = {"outcomes": rows, "retained": retained, "errors": errors} + (output / "summary.json").write_text(redact(json.dumps(summary, indent=2), known)) + return summary + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("mode", choices=["terminal", "snapshot", "redact"]) + parser.add_argument("--home", type=Path) + parser.add_argument("--ids", nargs="+", default=[]) + parser.add_argument("--output", type=Path) + parser.add_argument("--copied-config") + args = parser.parse_args() + if args.mode == "redact": + sys.stdout.write(redact(sys.stdin.read(), secrets(args.copied_config))) + elif args.mode == "terminal": + if not args.home or not args.ids: + parser.error("terminal needs --home and --ids") + try: + return 0 if successful(args.home, args.ids) else 1 + except (OSError, ValueError, KeyError): + return 1 + else: + if not args.home or not args.output: + parser.error("snapshot needs --home and --output") + snapshot(args.home, args.output, secrets(args.copied_config)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/hosted-evidence_test.py b/scripts/hosted-evidence_test.py new file mode 100644 index 0000000000..f12ce2d76d --- /dev/null +++ b/scripts/hosted-evidence_test.py @@ -0,0 +1,207 @@ +#!/usr/bin/env python3 +"""Exercise the drive's real outcome predicates and evidence after home removal.""" + +import importlib.util +import json +import os +from pathlib import Path +import re +import shutil +import sqlite3 +import subprocess +import sys +import tempfile +import unittest +from unittest import mock + + +HERE = Path(__file__).resolve().parent +HELPER = HERE / "hosted-evidence.py" +DRIVE = HERE / "hosted-drive.sh" +spec = importlib.util.spec_from_file_location("hosted_evidence", HELPER) +evidence = importlib.util.module_from_spec(spec) +spec.loader.exec_module(evidence) + + +class HostedEvidenceTest(unittest.TestCase): + def setUp(self): + self.temp = tempfile.TemporaryDirectory() + self.addCleanup(self.temp.cleanup) + self.root = Path(self.temp.name) + self.home = self.root / "home" + self.home.mkdir() + self.journal = self.home / "v3/projects/project/tasks.jsonl" + self.journal.parent.mkdir(parents=True) + + def rows(self, *rows): + self.journal.write_text("".join(json.dumps(row) + "\n" for row in rows)) + + def run_helper(self, *args): + return subprocess.run([sys.executable, str(HELPER), *args], capture_output=True, text=True) + + def test_real_file_presence_predicate_refuses_failed_or_pending_handoffs(self): + work = self.root / "work" + work.mkdir() + for name in ["README.md", "muldiv.go"]: + (work / name).touch() + body = re.search(r"^first_run_home\(\) \{.*$", DRIVE.read_text(), re.M)[0] + shell = 'terminal_tasks() { python3 "$EVIDENCE_HELPER" terminal --home "$HOME_DIR" --ids "$@"; }\n' + body + "\nfirst_run_home" + env = dict(os.environ, WORK=str(work), HOME_DIR=str(self.home), EVIDENCE_HELPER=str(HELPER)) + for status in ["failed", "running", "queued", "your call"]: + self.rows({"id": "1", "status": status}, {"id": "2", "status": "done"}) + self.assertNotEqual(subprocess.run(["bash", "-c", shell], env=env).returncode, 0, status) + self.rows({"id": "1", "status": "done"}, {"id": "2", "status": "done"}) + self.assertEqual(subprocess.run(["bash", "-c", shell], env=env).returncode, 0) + + def test_latest_outcome_wins_and_missing_or_ambiguous_tasks_fail_closed(self): + self.rows({"id": "1", "sessionId": "a", "status": "done"}, + {"id": "1", "sessionId": "a", "status": "failed", "ending": "refused"}) + self.assertNotEqual(self.run_helper("terminal", "--home", str(self.home), "--ids", "1").returncode, 0) + self.assertFalse(evidence.successful(self.home, ["2"])) + self.rows({"id": "1", "sessionId": "a", "status": "done"}, + {"id": "1", "sessionId": "b", "status": "done"}) + self.assertFalse(evidence.successful(self.home, ["1"])) + self.journal.write_text('{"id": "1", "status": "done"}\n{bad json}\n') + self.assertNotEqual(self.run_helper("terminal", "--home", str(self.home), "--ids", "1").returncode, 0) + self.journal.write_text('{"id": "1", "status": "done"}\n{"id":"1","status":"fail') + self.assertNotEqual(self.run_helper("terminal", "--home", str(self.home), "--ids", "1").returncode, 0) + + def test_failed_reason_actions_and_run_basis_survive_home_removal(self): + row = {"id": "1", "status": "failed", "ending": "refused", "outcome": "task reached its spending limit"} + self.rows(row) + task = self.journal.parent / "tasks/1" + task.mkdir(parents=True) + (task / "trajectory.jsonl").write_text('{"tool":"bash","exit_code":1,"observation":"real failed command"}\n') + database = self.journal.parent / "plandb.db" + connection = sqlite3.connect(database) + connection.execute("CREATE TABLE tasks (id TEXT, status TEXT, err TEXT, verdict_basis TEXT)") + connection.execute("INSERT INTO tasks VALUES (?, ?, ?, ?)", ("1", "failed", "real failure", '{"kind":"run","runs":[]}')) + connection.commit() + connection.close() + output = self.root / "kept" + result = self.run_helper("snapshot", "--home", str(self.home), "--output", str(output)) + self.assertEqual(result.returncode, 0, result.stderr) + shutil.rmtree(self.home) + self.assertEqual(json.loads((output / "summary.json").read_text())["outcomes"], [row]) + self.assertIn("real failed command", (output / "records" / task.relative_to(self.home) / "trajectory.jsonl").read_text()) + stored = json.loads((output / "sqlite" / Path(str(database.relative_to(self.home)) + ".json")).read_text()) + self.assertEqual(stored[0]["err"], "real failure") + self.assertEqual(json.loads(stored[0]["verdict_basis"])["kind"], "run") + + def test_bound_mode_does_not_open_configs_or_follow_worker_links(self): + config = self.home / "config.json" + config.write_text("MUST NEVER BE READ") + external = self.root / "external-profile" + external.mkdir() + (external / "config.json").write_text("MUST NEVER BE READ") + (self.home / "profile-link").symlink_to(external, target_is_directory=True) + (self.home / "credential.json").symlink_to(external / "config.json") + self.rows({"id": "1", "status": "done"}) + read_text = Path.read_text + + def guarded(path, *args, **kwargs): + if path.name == "config.json" or path.is_symlink(): + self.fail("bound credential file/link was opened") + return read_text(path, *args, **kwargs) + + output = self.root / "kept" + with mock.patch.object(Path, "read_text", guarded): + evidence.snapshot(self.home, output, evidence.secrets()) + self.assertFalse(list(output.rglob("config.json"))) + self.assertFalse(list(output.rglob("credential.json"))) + + def test_redaction_preserves_diagnostic_json_without_known_key(self): + key = "synthetic_known_secret_123" + self.rows({"id": "1", "status": "failed", "outcome": "Bearer " + key}) + output = self.root / "kept" + evidence.snapshot(self.home, output, {key}) + for path in output.rglob("*"): + if path.is_file(): + self.assertNotIn(key, path.read_text()) + self.assertIn("struck out", json.loads((output / "summary.json").read_text())["outcomes"][0]["outcome"]) + + def test_success_cleanup_keeps_full_outcome_before_removing_owned_home(self): + home = Path(tempfile.mkdtemp(prefix="codeaf-drive-home.", dir="/tmp")) + work = Path(tempfile.mkdtemp(prefix="codeaf-drive-work.", dir="/tmp")) + self.addCleanup(shutil.rmtree, home, True) + self.addCleanup(shutil.rmtree, work, True) + journal = home / "tasks.jsonl" + journal.write_text('{"id":"1","status":"failed","ending":"refused","outcome":"actual reason"}\n') + output = self.root / "kept" + output.mkdir() + source = DRIVE.read_text() + functions = "\n".join(re.search(r"^" + name + r"\(\) \{\n.*?^\}", source, re.M | re.S)[0] + for name in ["strike_keys", "capture_evidence", "cleanup"]) + shell = '''set -u +EVIDENCE_ARGS=(--home "$HOME_DIR") +passed=yes +final_kept=no +say() { printf '%s\\n' "$*"; } +t() { :; } +end_engine() { :; } +screen() { printf 'final screen\\n'; } +''' + functions + "\ncleanup\n" + env = dict(os.environ, HOME_DIR=str(home), WORK=str(work), OUT=str(output), EVIDENCE_HELPER=str(HELPER)) + result = subprocess.run(["bash", "-c", shell], env=env, capture_output=True, text=True) + self.assertEqual(result.returncode, 0, result.stderr) + self.assertFalse(home.exists()) + self.assertFalse(work.exists()) + row = json.loads((output / "final/summary.json").read_text())["outcomes"][0] + self.assertEqual((row["status"], row["ending"], row["outcome"]), ("failed", "refused", "actual reason")) + + def test_bound_profile_copy_block_never_calls_copy(self): + source = DRIVE.read_text() + block = re.search(r'^profile=.*?^fi$', source, re.M | re.S)[0] + profile = self.root / "profile" + profile.mkdir() + (profile / "config.json").write_text("synthetic credential file") + env = dict(os.environ, CODEAF_PROFILE_DIR=str(profile), CODEAF_HOME=str(profile), HOME_DIR=str(self.home)) + result = subprocess.run(["bash", "-c", 'cp() { echo "unexpected credential copy" >&2; return 99; }\n' + block], + env=env, capture_output=True, text=True) + self.assertEqual(result.returncode, 0, result.stderr) + self.assertNotIn("unexpected credential copy", result.stderr) + self.assertFalse((self.home / "config.json").exists()) + + def test_launch_pins_call_metadata_to_owned_home_without_bodies(self): + work = self.root / "work" + work.mkdir() + fake = self.root / "fixture-cli" + fake.write_text('''#!/usr/bin/env python3 +import json, os +from pathlib import Path +path = Path(os.environ["CODEAF_CALL_LOG"]) +path.parent.mkdir(parents=True, exist_ok=True) +path.write_text(json.dumps({"model":"fixture-model", "status":400, "error":"actual endpoint refusal", "bodies":os.environ["CODEAF_CALL_LOG_BODIES"]})+"\\n") +''') + fake.chmod(0o700) + launch = re.search(r'^t new-session.*?^\s*".*?sleep 3600"$', DRIVE.read_text(), re.M | re.S)[0] + # Execute the production environment binding with a deterministic + # process fixture, without tmux or the drive's post-exit pane delay. + shell = '''t() { + for command in "$@"; do :; done + command=${command%; sleep 3600} + bash -c "$command" +} +''' + launch + env = dict(os.environ, HOME_DIR=str(self.home), WORK=str(work), BIN=str(fake), CODEAF_PROFILE_DIR=str(self.root / "external")) + result = subprocess.run(["bash", "-c", shell], env=env, capture_output=True, text=True) + self.assertEqual(result.returncode, 0, result.stderr) + output = self.root / "kept" + evidence.snapshot(self.home, output, set()) + shutil.rmtree(self.home) + record = json.loads((output / "records/logs/calls.jsonl").read_text()) + self.assertEqual((record["status"], record["error"], record["bodies"]), (400, "actual endpoint refusal", "off")) + + def test_original_guards_and_all_fourteen_assertions_still_exist(self): + source = DRIVE.read_text() + self.assertIn("wait_for 60 5 first_run_home", source) + self.assertIn("wait_for 60 5 second_run_home", source) + self.assertEqual(len(re.findall(r'^check "', source, re.M)), 16) + # This test covers the newly added outcome checks without accepting a + # weaker UI-only result; the live drive still owns real-model proof. + self.assertIn('check "both first hand-offs ended successfully" terminal_tasks 1 2', source) + self.assertIn('check "the later hand-off ended successfully" terminal_tasks 3', source) + + +if __name__ == "__main__": + unittest.main()