diff --git a/tests/bounce-state-simulation.sh b/tests/bounce-state-simulation.sh index f76ed22..246390a 100755 --- a/tests/bounce-state-simulation.sh +++ b/tests/bounce-state-simulation.sh @@ -121,15 +121,22 @@ DOC # --------------------------------------------------------------------------- # Scenario 1: co-evolve-bouncer --vanilla --bounce-only -> conforming state # --------------------------------------------------------------------------- +# Policy (b): co-evolve-bouncer.sh already supports CO_EVOLVE_RUNS_DIR (v1.4, +# added for the npm MCP embedder) to redirect run artifacts away from its own +# runs/ dir. Pointing it at this sim's own $TEST_DIR makes discovery within +# that private root race-free by construction — no other concurrent suite can +# ever write into it — instead of racing on `ls -dt` against the shared repo +# runs/ dir that every concurrent suite's default invocation also writes to. +CO_EVOLVE_RUNS_DIR="$TEST_DIR/co-evolve-runs" TOTAL=$((TOTAL + 1)) s1_doc="$TEST_DIR/s1-doc.md" cp "$DOC_SRC" "$s1_doc" rc=0 -BOUNCE_STUB_COUNTER="$TEST_DIR/s1-count" PATH="$TEST_DIR/bin:$PATH" \ +BOUNCE_STUB_COUNTER="$TEST_DIR/s1-count" PATH="$TEST_DIR/bin:$PATH" CO_EVOLVE_RUNS_DIR="$CO_EVOLVE_RUNS_DIR" \ bash "$REPO_ROOT/co-evolve-bouncer.sh" --vanilla --bounce-only "$s1_doc" \ > "$TEST_DIR/s1-stdout.log" 2> "$TEST_DIR/s1-stderr.log" || rc=$? -s1_run_dir=$(ls -dt "$REPO_ROOT"/runs/co-evolve-* 2>/dev/null | head -1) +s1_run_dir=$(ls -dt "$CO_EVOLVE_RUNS_DIR"/co-evolve-* 2>/dev/null | head -1) s1_state="$s1_run_dir/state.json" if [[ "$rc" -eq 0 && -f "$s1_state" ]] \ && jq -e '.schema == "bounce-state/1.0" @@ -193,10 +200,10 @@ cp "$TEST_DIR/authbin/claude" "$TEST_DIR/authbin/codex" s4_doc="$TEST_DIR/s4-doc.md" cp "$DOC_SRC" "$s4_doc" rc=0 -PATH="$TEST_DIR/authbin:$PATH" \ +PATH="$TEST_DIR/authbin:$PATH" CO_EVOLVE_RUNS_DIR="$CO_EVOLVE_RUNS_DIR" \ bash "$REPO_ROOT/co-evolve-bouncer.sh" --vanilla --bounce-only "$s4_doc" \ > /dev/null 2>&1 || rc=$? -s4_run_dir=$(ls -dt "$REPO_ROOT"/runs/co-evolve-* 2>/dev/null | head -1) +s4_run_dir=$(ls -dt "$CO_EVOLVE_RUNS_DIR"/co-evolve-* 2>/dev/null | head -1) s4_state="$s4_run_dir/state.json" if [[ "$rc" -ne 0 && -f "$s4_state" ]] \ && jq -e '.status == "aborted" and (.passes | length) == 0' "$s4_state" >/dev/null; then @@ -209,14 +216,34 @@ rm -rf "$s4_run_dir" # --------------------------------------------------------------------------- # Scenario 5: agent-bouncer -> conforming state + normalized artifact names # --------------------------------------------------------------------------- +# Policy (c): agent-bouncer.sh hardcodes RUNS_DIR="${REPO_ROOT}/runs" with no +# env override (unlike co-evolve-bouncer.sh's CO_EVOLVE_RUNS_DIR), so it always +# writes into the shared repo runs/ dir that a concurrent suite's own bouncer +# runs also write into. Attributable snapshot-diff instead of `ls -dt | head -1`: +# the run dir label derives from the input doc's basename (agent-bouncer.sh's +# filename fallback — the stubbed LLM naming output never passes its kebab-case +# regex), so a PID-tagged doc name makes OUR dir identifiable. List before, +# list after, take the one new entry carrying our tag; a concurrent suite's S5 +# dirs carry a different PID and never match, so zero or multiple MATCHING +# entries indicates a real bug — fail loudly. TOTAL=$((TOTAL + 1)) -s5_doc="$TEST_DIR/s5-doc.md" +s5_doc="$TEST_DIR/s5-doc-$$.md" cp "$DOC_SRC" "$s5_doc" +s5_before=$(ls -d "$REPO_ROOT"/runs/bouncer-s5-doc-$$-* 2>/dev/null || true) rc=0 BOUNCE_STUB_COUNTER="$TEST_DIR/s5-count" PATH="$TEST_DIR/bin:$PATH" \ bash "$REPO_ROOT/agent-bouncer/agent-bouncer.sh" "$s5_doc" 2 claude claude \ > "$TEST_DIR/s5-stdout.log" 2>&1 || rc=$? -s5_run_dir=$(ls -dt "$REPO_ROOT"/runs/bouncer-* 2>/dev/null | head -1) +s5_after=$(ls -d "$REPO_ROOT"/runs/bouncer-s5-doc-$$-* 2>/dev/null || true) +s5_new=$(comm -13 <(printf '%s\n' "$s5_before" | sort) <(printf '%s\n' "$s5_after" | sort)) +s5_run_dir="" +# grep -c exits 1 on zero matches, which would trip `set -e` inside a command +# substitution (zero-new-dirs is a real failure mode here, e.g. the invocation +# died before creating one) — `|| true` keeps that a value, not a script kill. +s5_new_count=$(printf '%s\n' "$s5_new" | grep -c . || true) +if [[ "$s5_new_count" -eq 1 ]]; then + s5_run_dir="$s5_new" +fi s5_state="$s5_run_dir/state.json" if [[ "$rc" -eq 0 && -f "$s5_state" ]] \ && jq -e '.schema == "bounce-state/1.0" diff --git a/tests/observability-lifecycle-simulation.sh b/tests/observability-lifecycle-simulation.sh index 7544674..1e1a676 100644 --- a/tests/observability-lifecycle-simulation.sh +++ b/tests/observability-lifecycle-simulation.sh @@ -137,7 +137,10 @@ harmless line to a tracked file so the run produces a diff for verify to score. - None identified. PLAN -latest_run_dir() { ls -dt "$REPO_ROOT"/runs/dev-review-* 2>/dev/null | head -1; } +# Explicit per-scenario run dirs (policy a) — avoids the `ls -dt runs/dev-review-*` +# newest-mtime race that cross-reads another concurrent suite's run dir. Each +# scenario passes --run-dir under this sim's own $TEST_DIR, so cleanup rides the +# existing TEST_DIR EXIT trap instead of a per-scenario rm -rf. # =========================================================================== # Scenario 1: full lifecycle — current_phase=null at EOF, runner_pid present, @@ -145,18 +148,17 @@ latest_run_dir() { ls -dt "$REPO_ROOT"/runs/dev-review-* 2>/dev/null | head -1; # =========================================================================== TOTAL=$((TOTAL + 1)) repo=$(make_scratch_repo s1) -before=$(latest_run_dir || true) +run1="$TEST_DIR/run-s1" ( unset CLAUDE_MODEL CLAUDE_EFFORT CODEX_MODEL CODEX_REASONING_EFFORT unset COMPOSER_MODEL COMPOSER_EFFORT EXECUTOR_MODEL EXECUTOR_EFFORT VERIFIER_MODEL VERIFIER_EFFORT export PATH="$TEST_DIR/bin:$PATH" bash "$RUNNER" --skip-plan --plan "$PLAN_FIXTURE" --bounces 0 --verify --verifier codex \ - --workdir "$repo" --timeout 60 -- "obs lifecycle probe" + --run-dir "$run1" --workdir "$repo" --timeout 60 -- "obs lifecycle probe" ) > "$TEST_DIR/s1.out" 2>&1 || true -run1=$(latest_run_dir || true) s1_state="$run1/state.json" s1_ok=true -if [[ -z "$run1" || "$run1" == "$before" || ! -f "$s1_state" ]]; then +if [[ ! -f "$s1_state" ]]; then s1_ok=false else jq -e '.current_phase == null' "$s1_state" >/dev/null 2>&1 || { s1_ok=false; echo " current_phase not null" >&2; } @@ -172,28 +174,25 @@ else [[ -f "$s1_state" ]] && cat "$s1_state" >&2 cat "$TEST_DIR/s1.out" >&2 fi -[[ -n "$run1" && "$run1" != "$before" ]] && rm -rf "$run1" # =========================================================================== # Scenario 2: --parent-run lands in .orchestration.parent_run_id. # =========================================================================== TOTAL=$((TOTAL + 1)) repo=$(make_scratch_repo s2) -before=$(latest_run_dir || true) +run2="$TEST_DIR/run-s2" ( export PATH="$TEST_DIR/bin:$PATH" bash "$RUNNER" --skip-plan --plan "$PLAN_FIXTURE" --bounces 0 --verify --verifier codex \ - --parent-run "abc-123" --workdir "$repo" --timeout 60 -- "obs parent-run probe" + --parent-run "abc-123" --run-dir "$run2" --workdir "$repo" --timeout 60 -- "obs parent-run probe" ) > "$TEST_DIR/s2.out" 2>&1 || true -run2=$(latest_run_dir || true) -if [[ -n "$run2" && "$run2" != "$before" && -f "$run2/state.json" ]] \ +if [[ -f "$run2/state.json" ]] \ && jq -e '.orchestration.parent_run_id == "abc-123"' "$run2/state.json" >/dev/null 2>&1; then pass "S2: --parent-run abc-123 recorded in .orchestration.parent_run_id" else fail "S2: parent_run_id missing (run=$run2)" [[ -f "$run2/state.json" ]] && cat "$run2/state.json" >&2 fi -[[ -n "$run2" && "$run2" != "$before" ]] && rm -rf "$run2" # =========================================================================== # Scenario 3: --parent-run rejects a garbage token (path traversal / shell meta). @@ -215,25 +214,25 @@ fi # =========================================================================== TOTAL=$((TOTAL + 1)) repo=$(make_scratch_repo s4) -before=$(latest_run_dir || true) +run4="$TEST_DIR/run-s4" ( export PATH="$TEST_DIR/bin:$PATH" export OBS_EXECUTE_SLEEP=4 bash "$RUNNER" --skip-plan --plan "$PLAN_FIXTURE" --bounces 0 --verify --verifier codex \ - --workdir "$repo" --timeout 60 -- "obs mid-run probe" + --run-dir "$run4" --workdir "$repo" --timeout 60 -- "obs mid-run probe" ) > "$TEST_DIR/s4.out" 2>&1 & runner_bg=$! -# Poll for a run dir whose state.json reports current_phase.name == execute. +# Poll the known run dir's state.json for current_phase.name == execute (no +# newest-mtime discovery needed — the path is pinned by --run-dir above). captured=false captured_pid="" for _ in $(seq 1 60); do - rd=$(latest_run_dir || true) - if [[ -n "$rd" && "$rd" != "$before" && -f "$rd/state.json" ]]; then - name=$(jq -r '.current_phase.name // empty' "$rd/state.json" 2>/dev/null || true) + if [[ -f "$run4/state.json" ]]; then + name=$(jq -r '.current_phase.name // empty' "$run4/state.json" 2>/dev/null || true) if [[ "$name" == "execute" ]]; then captured=true - captured_pid=$(jq -r '.runner_pid // empty' "$rd/state.json" 2>/dev/null || true) + captured_pid=$(jq -r '.runner_pid // empty' "$run4/state.json" 2>/dev/null || true) break fi fi @@ -246,8 +245,7 @@ s4_ok=true # runner_pid captured mid-run must be a positive integer. [[ "$captured_pid" =~ ^[0-9]+$ ]] || { s4_ok=false; echo " captured runner_pid not numeric: $captured_pid" >&2; } # And the run must still terminate cleanly with current_phase cleared. -run4=$(latest_run_dir || true) -if [[ -n "$run4" && -f "$run4/state.json" ]]; then +if [[ -f "$run4/state.json" ]]; then jq -e '.current_phase == null and .status == "completed"' "$run4/state.json" >/dev/null 2>&1 \ || { s4_ok=false; echo " post-run state not clean terminal" >&2; } fi @@ -258,7 +256,6 @@ else [[ -f "$run4/state.json" ]] && cat "$run4/state.json" >&2 cat "$TEST_DIR/s4.out" >&2 fi -[[ -n "$run4" && "$run4" != "$before" ]] && rm -rf "$run4" # =========================================================================== # Scenario 5: parity — a default skip-plan run with NO new flags still produces @@ -267,21 +264,19 @@ fi # =========================================================================== TOTAL=$((TOTAL + 1)) repo=$(make_scratch_repo s5) -before=$(latest_run_dir || true) +run5="$TEST_DIR/run-s5" ( export PATH="$TEST_DIR/bin:$PATH" bash "$RUNNER" --skip-plan --plan "$PLAN_FIXTURE" --bounces 0 --verify --verifier codex \ - --workdir "$repo" --timeout 60 -- "obs parity probe" + --run-dir "$run5" --workdir "$repo" --timeout 60 -- "obs parity probe" ) > "$TEST_DIR/s5.out" 2>&1 || true -run5=$(latest_run_dir || true) -if [[ -n "$run5" && "$run5" != "$before" && -f "$run5/state.json" ]] \ +if [[ -f "$run5/state.json" ]] \ && jq -e '.status == "completed" and .current_phase == null' "$run5/state.json" >/dev/null 2>&1; then pass "S5: no-new-flags run reaches completed with current_phase cleared" else fail "S5: parity run not clean (run=$run5)" [[ -f "$run5/state.json" ]] && cat "$run5/state.json" >&2 fi -[[ -n "$run5" && "$run5" != "$before" ]] && rm -rf "$run5" # --------------------------------------------------------------------------- printf '%d/%d scenarios passed' "$PASSED" "$TOTAL" diff --git a/tests/preset-expansion-simulation.sh b/tests/preset-expansion-simulation.sh index 15693a9..2582868 100644 --- a/tests/preset-expansion-simulation.sh +++ b/tests/preset-expansion-simulation.sh @@ -261,15 +261,17 @@ TOTAL=$((TOTAL + 1)) repo=$(make_scratch_repo a) claude_log="$TEST_DIR/a_claude.log"; codex_log="$TEST_DIR/a_codex.log" : > "$claude_log"; : > "$codex_log" -run_dir_before=$(ls -dt "$REPO_ROOT"/runs/dev-review-* 2>/dev/null | head -1 || true) +# Explicit run dir (policy a) — avoids the `ls -dt runs/dev-review-*` newest-mtime +# race that cross-reads another concurrent suite's run dir. Scenario (b) below +# reuses this same path directly instead of re-discovering "newest". +a_run_dir="$TEST_DIR/run-a" ( unset CLAUDE_MODEL CLAUDE_EFFORT CODEX_MODEL CODEX_REASONING_EFFORT unset COMPOSER_MODEL COMPOSER_EFFORT EXECUTOR_MODEL EXECUTOR_EFFORT VERIFIER_MODEL VERIFIER_EFFORT BOUNCER_MODEL BOUNCER_EFFORT export PATH="$TEST_DIR/bin:$PATH" export CLAUDE_ARGV_LOG="$claude_log" CODEX_ARGV_LOG="$codex_log" - bash "$RUNNER" --preset codex-build --workdir "$repo" --timeout 60 -- "preset seat argv probe" + bash "$RUNNER" --preset codex-build --run-dir "$a_run_dir" --workdir "$repo" --timeout 60 -- "preset seat argv probe" ) >"$TEST_DIR/a.out" 2>&1 || true -a_run_dir_k=$(ls -dt "$REPO_ROOT"/runs/dev-review-* 2>/dev/null | head -1) a_ok=true # composer = claude with the `best` model alias resolved (-> claude-opus-4-8) + high effort. @@ -284,9 +286,9 @@ if ! grep -Eq -- '-c model_reasoning_effort=xhigh' "$codex_log"; then a_ok=false if ! grep -Eq -- '--effort max' "$claude_log"; then a_ok=false; fi # v1.5 Phase 1 acceptance: a PRESET run's state.json must carry NO (default) seat # — every seat (incl. the A-8 bounce counterparty, codex here) is pinned concrete. -if [[ -n "$a_run_dir_k" && "$a_run_dir_k" != "$run_dir_before" ]]; then - if ! jq -e '.seat_models.bouncer == "codex:gpt-5.5@xhigh"' "$a_run_dir_k/state.json" >/dev/null 2>&1; then a_ok=false; fi - if ! jq -e '[.seat_models[] | select(contains("(default)"))] | length == 0' "$a_run_dir_k/state.json" >/dev/null 2>&1; then a_ok=false; fi +if [[ -f "$a_run_dir/state.json" ]]; then + if ! jq -e '.seat_models.bouncer == "codex:gpt-5.5@xhigh"' "$a_run_dir/state.json" >/dev/null 2>&1; then a_ok=false; fi + if ! jq -e '[.seat_models[] | select(contains("(default)"))] | length == 0' "$a_run_dir/state.json" >/dev/null 2>&1; then a_ok=false; fi else a_ok=false fi @@ -303,10 +305,8 @@ fi # preset run already produced a claude verdict through the hardening path). # =========================================================================== TOTAL=$((TOTAL + 1)) -# Find the run dir the scenario-(a) run created (newest under runs/). -a_run_dir=$(ls -dt "$REPO_ROOT"/runs/dev-review-* 2>/dev/null | head -1) b_ok=false -if [[ -n "$a_run_dir" && -f "$a_run_dir/verdict.json" ]]; then +if [[ -f "$a_run_dir/verdict.json" ]]; then if jq -e '.verdict == "APPROVED" and (.confidence | type) == "number" and has("summary") and has("issues")' \ "$a_run_dir/verdict.json" >/dev/null 2>&1; then b_ok=true @@ -316,10 +316,8 @@ if [[ "$b_ok" == true ]]; then pass "verdict hardening: prose/fence-wrapped verdict.json is jq-parseable with expected fields" else fail "verdict.json not jq-parseable after hardening (run dir: ${a_run_dir:-})" - [[ -n "${a_run_dir:-}" && -f "$a_run_dir/verdict.json" ]] && { echo "--- verdict.json ---"; cat "$a_run_dir/verdict.json"; } >&2 + [[ -f "$a_run_dir/verdict.json" ]] && { echo "--- verdict.json ---"; cat "$a_run_dir/verdict.json"; } >&2 fi -# Clean up the runs/ artifacts this gate created so it leaves no side effects. -[[ -n "${a_run_dir:-}" ]] && rm -rf "$a_run_dir" # =========================================================================== # Scenario (c): --preset codex-build --verifier codex → verifier is codex @@ -329,18 +327,17 @@ TOTAL=$((TOTAL + 1)) repo=$(make_scratch_repo c) codex_log="$TEST_DIR/c_codex.log"; claude_log="$TEST_DIR/c_claude.log" : > "$codex_log"; : > "$claude_log" -run_dir_before=$(ls -dt "$REPO_ROOT"/runs/dev-review-* 2>/dev/null | head -1 || true) +c_run_dir="$TEST_DIR/run-c" ( unset CLAUDE_MODEL CLAUDE_EFFORT CODEX_MODEL CODEX_REASONING_EFFORT unset COMPOSER_MODEL COMPOSER_EFFORT EXECUTOR_MODEL EXECUTOR_EFFORT VERIFIER_MODEL VERIFIER_EFFORT export PATH="$TEST_DIR/bin:$PATH" export CLAUDE_ARGV_LOG="$claude_log" CODEX_ARGV_LOG="$codex_log" bash "$RUNNER" --preset codex-build --verifier codex --skip-plan --plan "$PLAN_FIXTURE" \ - --workdir "$repo" --timeout 60 -- "preset verifier override probe" + --run-dir "$c_run_dir" --workdir "$repo" --timeout 60 -- "preset verifier override probe" ) >"$TEST_DIR/c.out" 2>&1 || true -c_run_dir=$(ls -dt "$REPO_ROOT"/runs/dev-review-* 2>/dev/null | head -1) c_ok=false -if [[ -n "$c_run_dir" && "$c_run_dir" != "$run_dir_before" ]]; then +if [[ -f "$c_run_dir/state.json" ]]; then # state.json seat_models.verifier must report codex; banner Verifier line too. if jq -e '.seat_models.verifier | startswith("codex:")' "$c_run_dir/state.json" >/dev/null 2>&1 \ && grep -Eq '^ Verifier: codex' "$c_run_dir/run.log"; then @@ -351,9 +348,8 @@ if [[ "$c_ok" == true ]]; then pass "preset codex-build --verifier codex: verifier seat is codex (last-wins)" else fail "verifier override did not win over preset (run dir: ${c_run_dir:-})" - [[ -n "${c_run_dir:-}" ]] && grep -E '^ Verifier:' "$c_run_dir/run.log" >&2 || true + [[ -f "$c_run_dir/run.log" ]] && grep -E '^ Verifier:' "$c_run_dir/run.log" >&2 || true fi -[[ -n "${c_run_dir:-}" && "$c_run_dir" != "$run_dir_before" ]] && rm -rf "$c_run_dir" # =========================================================================== # Scenario (d): COMPOSER_EFFORT=low pre-set + preset → composer gets @@ -363,23 +359,21 @@ TOTAL=$((TOTAL + 1)) repo=$(make_scratch_repo d) claude_log="$TEST_DIR/d_claude.log"; codex_log="$TEST_DIR/d_codex.log" : > "$claude_log"; : > "$codex_log" -run_dir_before=$(ls -dt "$REPO_ROOT"/runs/dev-review-* 2>/dev/null | head -1 || true) +d_run_dir="$TEST_DIR/run-d" ( unset CLAUDE_MODEL CLAUDE_EFFORT CODEX_MODEL CODEX_REASONING_EFFORT unset EXECUTOR_MODEL EXECUTOR_EFFORT VERIFIER_MODEL VERIFIER_EFFORT export COMPOSER_EFFORT="low" export PATH="$TEST_DIR/bin:$PATH" export CLAUDE_ARGV_LOG="$claude_log" CODEX_ARGV_LOG="$codex_log" - bash "$RUNNER" --preset codex-build --workdir "$repo" --timeout 60 -- "preset composer effort env probe" + bash "$RUNNER" --preset codex-build --run-dir "$d_run_dir" --workdir "$repo" --timeout 60 -- "preset composer effort env probe" ) >"$TEST_DIR/d.out" 2>&1 || true -d_run_dir=$(ls -dt "$REPO_ROOT"/runs/dev-review-* 2>/dev/null | head -1) if grep -Eq -- '--effort low' "$claude_log" && ! grep -Eq -- '--effort high' "$claude_log"; then pass "preset + COMPOSER_EFFORT=low: composer gets --effort low (fill-if-empty env wins)" else fail "COMPOSER_EFFORT env did not beat preset high (claude argv below)" cat "$claude_log" >&2 fi -[[ -n "${d_run_dir:-}" && "$d_run_dir" != "$run_dir_before" ]] && rm -rf "$d_run_dir" # =========================================================================== # Scenario (e): --preset bogus dies with the unknown-preset message. @@ -413,7 +407,7 @@ TOTAL=$((TOTAL + 1)) repo=$(make_scratch_repo g) claude_log="$TEST_DIR/g_claude.log"; codex_log="$TEST_DIR/g_codex.log" : > "$claude_log"; : > "$codex_log" -run_dir_before=$(ls -dt "$REPO_ROOT"/runs/dev-review-* 2>/dev/null | head -1 || true) +g_run_dir="$TEST_DIR/run-g" ( unset CLAUDE_MODEL CLAUDE_EFFORT CODEX_MODEL CODEX_REASONING_EFFORT unset COMPOSER_MODEL COMPOSER_EFFORT EXECUTOR_MODEL EXECUTOR_EFFORT VERIFIER_MODEL VERIFIER_EFFORT @@ -422,9 +416,8 @@ run_dir_before=$(ls -dt "$REPO_ROOT"/runs/dev-review-* 2>/dev/null | head -1 || # Default flavor: codex composes + executes, opus verifies. --verify so a # claude verdict path runs too. No preset, no effort knobs. bash "$RUNNER" --verify --bounces 0 --skip-plan --plan "$PLAN_FIXTURE" \ - --workdir "$repo" --timeout 60 -- "parity guard probe" + --run-dir "$g_run_dir" --workdir "$repo" --timeout 60 -- "parity guard probe" ) >"$TEST_DIR/g.out" 2>&1 || true -g_run_dir=$(ls -dt "$REPO_ROOT"/runs/dev-review-* 2>/dev/null | head -1) g_ok=true if grep -Eq -- '--effort ' "$claude_log"; then g_ok=false; fi if grep -Eq -- 'model_reasoning_effort=' "$codex_log"; then g_ok=false; fi @@ -434,7 +427,6 @@ else fail "parity guard: an effort flag leaked into a default run (logs below)" { echo "--- claude argv ---"; cat "$claude_log"; echo "--- codex argv ---"; cat "$codex_log"; } >&2 fi -[[ -n "${g_run_dir:-}" && "$g_run_dir" != "$run_dir_before" ]] && rm -rf "$g_run_dir" # =========================================================================== # Scenario (h): cross-agent leak guard — --preset codex-build --verifier codex. @@ -450,16 +442,15 @@ TOTAL=$((TOTAL + 1)) repo=$(make_scratch_repo h) codex_log="$TEST_DIR/h_codex.log"; claude_log="$TEST_DIR/h_claude.log" : > "$codex_log"; : > "$claude_log" -run_dir_before=$(ls -dt "$REPO_ROOT"/runs/dev-review-* 2>/dev/null | head -1 || true) +h_run_dir="$TEST_DIR/run-h" ( unset CLAUDE_MODEL CLAUDE_EFFORT CODEX_MODEL CODEX_REASONING_EFFORT unset COMPOSER_MODEL COMPOSER_EFFORT EXECUTOR_MODEL EXECUTOR_EFFORT VERIFIER_MODEL VERIFIER_EFFORT export PATH="$TEST_DIR/bin:$PATH" export CLAUDE_ARGV_LOG="$claude_log" CODEX_ARGV_LOG="$codex_log" bash "$RUNNER" --preset codex-build --verifier codex --skip-plan --plan "$PLAN_FIXTURE" \ - --workdir "$repo" --timeout 60 -- "preset codex-verifier leak guard probe" + --run-dir "$h_run_dir" --workdir "$repo" --timeout 60 -- "preset codex-verifier leak guard probe" ) >"$TEST_DIR/h.out" 2>&1 || true -h_run_dir=$(ls -dt "$REPO_ROOT"/runs/dev-review-* 2>/dev/null | head -1) # The verify invocation is the codex argv line carrying --output-schema. h_verify_argv=$(grep -- '--output-schema' "$codex_log" || true) h_ok=true @@ -471,7 +462,7 @@ if printf '%s' "$h_verify_argv" | grep -Eq -- 'model_reasoning_effort=max'; then # Sanity: the codex verify invocation actually happened (so the asserts are real). if [[ -z "$h_verify_argv" ]]; then h_ok=false; fi # state.json must report the fallback seat, not the leaked fable@max pair. -if [[ -n "$h_run_dir" && "$h_run_dir" != "$run_dir_before" ]]; then +if [[ -f "$h_run_dir/state.json" ]]; then if ! jq -e '.seat_models.verifier == "codex:(default)@(default)"' "$h_run_dir/state.json" >/dev/null 2>&1; then h_ok=false fi @@ -483,10 +474,9 @@ if [[ "$h_ok" == true ]]; then else fail "cross-agent leak guard failed (run dir: ${h_run_dir:-})" { echo "--- codex verify argv ---"; printf '%s\n' "$h_verify_argv" - [[ -n "${h_run_dir:-}" && -f "$h_run_dir/state.json" ]] && { echo "--- seat_models ---"; jq -c '.seat_models' "$h_run_dir/state.json"; } + [[ -f "$h_run_dir/state.json" ]] && { echo "--- seat_models ---"; jq -c '.seat_models' "$h_run_dir/state.json"; } } >&2 fi -[[ -n "${h_run_dir:-}" && "$h_run_dir" != "$run_dir_before" ]] && rm -rf "$h_run_dir" # =========================================================================== # Scenario (i): alias resolution. v1.5 Phase 1 (A-4a) moved @@ -525,15 +515,14 @@ TOTAL=$((TOTAL + 1)) repo=$(make_scratch_repo j) claude_log="$TEST_DIR/j_claude.log"; codex_log="$TEST_DIR/j_codex.log" : > "$claude_log"; : > "$codex_log" -run_dir_before=$(ls -dt "$REPO_ROOT"/runs/dev-review-* 2>/dev/null | head -1 || true) +j_run_dir="$TEST_DIR/run-j" ( unset CLAUDE_MODEL CLAUDE_EFFORT CODEX_MODEL CODEX_REASONING_EFFORT unset COMPOSER_MODEL COMPOSER_EFFORT EXECUTOR_MODEL EXECUTOR_EFFORT VERIFIER_MODEL VERIFIER_EFFORT export PATH="$TEST_DIR/bin:$PATH" export CLAUDE_ARGV_LOG="$claude_log" CODEX_ARGV_LOG="$codex_log" - bash "$RUNNER" --preset claude-build --workdir "$repo" --timeout 60 -- "preset claude-build seat argv probe" + bash "$RUNNER" --preset claude-build --run-dir "$j_run_dir" --workdir "$repo" --timeout 60 -- "preset claude-build seat argv probe" ) >"$TEST_DIR/j.out" 2>&1 || true -j_run_dir=$(ls -dt "$REPO_ROOT"/runs/dev-review-* 2>/dev/null | head -1) j_claude_execute_argv=$(grep -- '--permission-mode bypassPermissions' "$claude_log" || true) j_codex_verify_argv=$(grep -- '--output-schema' "$codex_log" || true) j_ok=true @@ -551,7 +540,7 @@ if ! printf '%s' "$j_claude_execute_argv" | grep -Eq -- '--model claude-opus-4-8 if ! printf '%s' "$j_claude_execute_argv" | grep -Eq -- '--effort high'; then j_ok=false; fi if printf '%s' "$j_claude_execute_argv" | grep -Eq -- '--model (gpt-|codex)'; then j_ok=false; fi -if [[ -n "$j_run_dir" && "$j_run_dir" != "$run_dir_before" ]]; then +if [[ -f "$j_run_dir/state.json" ]]; then if ! jq -e '.seat_models.composer == "codex:gpt-5.5@xhigh" and .seat_models.executor == "opus:claude-opus-4-8@high" and .seat_models.verifier == "codex:gpt-5.5@xhigh"' \ @@ -568,10 +557,9 @@ else fail "preset claude-build seat argv mismatch (run dir: ${j_run_dir:-})" { echo "--- claude execute argv ---"; printf '%s\n' "$j_claude_execute_argv" echo "--- codex argv ---"; cat "$codex_log" - [[ -n "${j_run_dir:-}" && -f "$j_run_dir/state.json" ]] && { echo "--- seat_models ---"; jq -c '.seat_models' "$j_run_dir/state.json"; } + [[ -f "$j_run_dir/state.json" ]] && { echo "--- seat_models ---"; jq -c '.seat_models' "$j_run_dir/state.json"; } } >&2 fi -[[ -n "${j_run_dir:-}" && "$j_run_dir" != "$run_dir_before" ]] && rm -rf "$j_run_dir" # =========================================================================== # Scenario (k): v1.5 Phase 1 (A-8) bounce counterparty seat. Default flavor @@ -585,16 +573,15 @@ TOTAL=$((TOTAL + 1)) repo=$(make_scratch_repo k) claude_log="$TEST_DIR/k_claude.log"; codex_log="$TEST_DIR/k_codex.log" : > "$claude_log"; : > "$codex_log" -run_dir_before=$(ls -dt "$REPO_ROOT"/runs/dev-review-* 2>/dev/null | head -1 || true) +k_run_dir="$TEST_DIR/run-k" ( unset CLAUDE_MODEL CLAUDE_EFFORT CODEX_MODEL CODEX_REASONING_EFFORT unset COMPOSER_MODEL COMPOSER_EFFORT EXECUTOR_MODEL EXECUTOR_EFFORT VERIFIER_MODEL VERIFIER_EFFORT export BOUNCER_MODEL="claude-bouncer-xyz" BOUNCER_EFFORT="high" export PATH="$TEST_DIR/bin:$PATH" export CLAUDE_ARGV_LOG="$claude_log" CODEX_ARGV_LOG="$codex_log" - bash "$RUNNER" --bounces 2 --plan-only --workdir "$repo" --timeout 60 -- "bouncer seat probe" + bash "$RUNNER" --bounces 2 --plan-only --run-dir "$k_run_dir" --workdir "$repo" --timeout 60 -- "bouncer seat probe" ) >"$TEST_DIR/k.out" 2>&1 || true -k_run_dir=$(ls -dt "$REPO_ROOT"/runs/dev-review-* 2>/dev/null | head -1) k_ok=true # The bounce reviewer pass (claude) argv must carry BOUNCER_MODEL + effort. if ! grep -Eq -- '--model claude-bouncer-xyz' "$claude_log"; then k_ok=false; fi @@ -602,7 +589,7 @@ if ! grep -Eq -- '--effort high' "$claude_log"; then k_ok=false; fi # No leak: the bouncer claude id must not reach any codex argv. if grep -Eq -- 'claude-bouncer-xyz' "$codex_log"; then k_ok=false; fi # state.json seat_models.bouncer records the resolved reviewer seat. -if [[ -n "$k_run_dir" && "$k_run_dir" != "$run_dir_before" ]]; then +if [[ -f "$k_run_dir/state.json" ]]; then if ! jq -e '.seat_models.bouncer == "opus:claude-bouncer-xyz@high"' "$k_run_dir/state.json" >/dev/null 2>&1; then k_ok=false fi @@ -614,10 +601,9 @@ if [[ "$k_ok" == true ]]; then else fail "bouncer seat propagation failed (run dir: ${k_run_dir:-})" { echo "--- claude argv ---"; cat "$claude_log" - [[ -n "${k_run_dir:-}" && -f "$k_run_dir/state.json" ]] && { echo "--- seat_models ---"; jq -c '.seat_models' "$k_run_dir/state.json"; } + [[ -f "$k_run_dir/state.json" ]] && { echo "--- seat_models ---"; jq -c '.seat_models' "$k_run_dir/state.json"; } } >&2 fi -[[ -n "${k_run_dir:-}" && "$k_run_dir" != "$run_dir_before" ]] && rm -rf "$k_run_dir" # =========================================================================== # Scenario (l): v1.5 Phase 1 (H1) — an explicit --model beats the preset's @@ -631,22 +617,21 @@ TOTAL=$((TOTAL + 1)) repo=$(make_scratch_repo l) claude_log="$TEST_DIR/l_claude.log"; codex_log="$TEST_DIR/l_codex.log" : > "$claude_log"; : > "$codex_log" -run_dir_before=$(ls -dt "$REPO_ROOT"/runs/dev-review-* 2>/dev/null | head -1 || true) +l_run_dir="$TEST_DIR/run-l" ( unset CLAUDE_MODEL CLAUDE_EFFORT CODEX_MODEL CODEX_REASONING_EFFORT unset COMPOSER_MODEL COMPOSER_EFFORT EXECUTOR_MODEL EXECUTOR_EFFORT VERIFIER_MODEL VERIFIER_EFFORT BOUNCER_MODEL BOUNCER_EFFORT export PATH="$TEST_DIR/bin:$PATH" export CLAUDE_ARGV_LOG="$claude_log" CODEX_ARGV_LOG="$codex_log" bash "$RUNNER" --model o4-mini --preset codex-build --skip-plan --plan "$PLAN_FIXTURE" \ - --workdir "$repo" --timeout 60 -- "flag-over-preset precedence probe" + --run-dir "$l_run_dir" --workdir "$repo" --timeout 60 -- "flag-over-preset precedence probe" ) >"$TEST_DIR/l.out" 2>&1 || true -l_run_dir=$(ls -dt "$REPO_ROOT"/runs/dev-review-* 2>/dev/null | head -1) l_ok=true # Executor codex argv carries the FLAG model, the preset effort — and never gpt-5.5. if ! grep -Eq -- '-c model=o4-mini' "$codex_log"; then l_ok=false; fi if ! grep -Eq -- '-c model_reasoning_effort=xhigh' "$codex_log"; then l_ok=false; fi if grep -Eq -- '-c model=gpt-5.5' "$codex_log"; then l_ok=false; fi -if [[ -n "$l_run_dir" && "$l_run_dir" != "$run_dir_before" ]]; then +if [[ -f "$l_run_dir/state.json" ]]; then if ! jq -e '.seat_models.executor == "codex:o4-mini@xhigh"' "$l_run_dir/state.json" >/dev/null 2>&1; then l_ok=false; fi # The override is surfaced, not silent. if ! grep -Fq 'NOTE: preset executor codex model pin (gpt-5.5) overridden by explicit --model o4-mini' "$l_run_dir/run.log"; then l_ok=false; fi @@ -658,11 +643,10 @@ if [[ "$l_ok" == true ]]; then else fail "H1 flag-over-preset precedence failed (run dir: ${l_run_dir:-})" { echo "--- codex argv ---"; cat "$codex_log" - [[ -n "${l_run_dir:-}" && -f "$l_run_dir/state.json" ]] && { echo "--- seat_models ---"; jq -c '.seat_models' "$l_run_dir/state.json"; } - [[ -n "${l_run_dir:-}" && -f "$l_run_dir/run.log" ]] && { echo "--- NOTE lines ---"; grep 'NOTE:' "$l_run_dir/run.log" || true; } + [[ -f "$l_run_dir/state.json" ]] && { echo "--- seat_models ---"; jq -c '.seat_models' "$l_run_dir/state.json"; } + [[ -f "$l_run_dir/run.log" ]] && { echo "--- NOTE lines ---"; grep 'NOTE:' "$l_run_dir/run.log" || true; } } >&2 fi -[[ -n "${l_run_dir:-}" && "$l_run_dir" != "$run_dir_before" ]] && rm -rf "$l_run_dir" # =========================================================================== # Scenario (m): v1.5 Phase 1 (H1, warn-only side) — --claude-model with a preset @@ -674,22 +658,21 @@ TOTAL=$((TOTAL + 1)) repo=$(make_scratch_repo m) claude_log="$TEST_DIR/m_claude.log"; codex_log="$TEST_DIR/m_codex.log" : > "$claude_log"; : > "$codex_log" -run_dir_before=$(ls -dt "$REPO_ROOT"/runs/dev-review-* 2>/dev/null | head -1 || true) +m_run_dir="$TEST_DIR/run-m" ( unset CLAUDE_MODEL CLAUDE_EFFORT CODEX_MODEL CODEX_REASONING_EFFORT unset COMPOSER_MODEL COMPOSER_EFFORT EXECUTOR_MODEL EXECUTOR_EFFORT VERIFIER_MODEL VERIFIER_EFFORT BOUNCER_MODEL BOUNCER_EFFORT export PATH="$TEST_DIR/bin:$PATH" export CLAUDE_ARGV_LOG="$claude_log" CODEX_ARGV_LOG="$codex_log" bash "$RUNNER" --claude-model claude-shadow-test --preset codex-build --skip-plan --plan "$PLAN_FIXTURE" \ - --workdir "$repo" --timeout 60 -- "claude-model shadow warn probe" + --run-dir "$m_run_dir" --workdir "$repo" --timeout 60 -- "claude-model shadow warn probe" ) >"$TEST_DIR/m.out" 2>&1 || true -m_run_dir=$(ls -dt "$REPO_ROOT"/runs/dev-review-* 2>/dev/null | head -1) m_ok=true # Pre-existing precedence unchanged: the preset's claude pin still wins the argv. if ! grep -Eq -- '--model claude-opus-4-8' "$claude_log"; then m_ok=false; fi if grep -Eq -- '--model claude-shadow-test' "$claude_log"; then m_ok=false; fi # ...but the shadow is warned about (one line per shadowed preset claude seat). -if [[ -n "$m_run_dir" && "$m_run_dir" != "$run_dir_before" ]]; then +if [[ -f "$m_run_dir/run.log" ]]; then if ! grep -Eq 'WARNING: explicit --claude-model \(claude-shadow-test\) is shadowed by the preset (composer|verifier) model pin' "$m_run_dir/run.log"; then m_ok=false; fi else m_ok=false @@ -699,10 +682,9 @@ if [[ "$m_ok" == true ]]; then else fail "H1 --claude-model warn-only behavior mismatch (run dir: ${m_run_dir:-})" { echo "--- claude argv ---"; cat "$claude_log" - [[ -n "${m_run_dir:-}" && -f "$m_run_dir/run.log" ]] && { echo "--- WARNING lines ---"; grep 'WARNING:' "$m_run_dir/run.log" || true; } + [[ -f "$m_run_dir/run.log" ]] && { echo "--- WARNING lines ---"; grep 'WARNING:' "$m_run_dir/run.log" || true; } } >&2 fi -[[ -n "${m_run_dir:-}" && "$m_run_dir" != "$run_dir_before" ]] && rm -rf "$m_run_dir" # --- summary ---------------------------------------------------------------- passed=$((TOTAL - FAILURES)) diff --git a/tests/token-capture-simulation.sh b/tests/token-capture-simulation.sh index ecfb1a1..0e7134b 100755 --- a/tests/token-capture-simulation.sh +++ b/tests/token-capture-simulation.sh @@ -228,7 +228,10 @@ make_scratch_repo() { printf '%s' "$repo" } -latest_run_dir() { ls -dt "$REPO_ROOT"/runs/dev-review-* 2>/dev/null | head -1; } +# Explicit per-scenario run dirs (policy a) — avoids the `ls -dt runs/dev-review-*` +# newest-mtime race that cross-reads another concurrent suite's run dir. Each +# scenario passes --run-dir under this sim's own $TEST_DIR, so cleanup rides the +# existing TEST_DIR EXIT trap instead of a per-scenario rm -rf. # =========================================================================== # Scenario (a): flag ON, mixed run — claude composes + verifies, codex executes. @@ -239,7 +242,7 @@ TOTAL=$((TOTAL + 1)) repo=$(make_scratch_repo a) claude_log="$TEST_DIR/a_claude.log"; codex_log="$TEST_DIR/a_codex.log" : > "$claude_log"; : > "$codex_log" -run_dir_before=$(latest_run_dir || true) +a_run_dir="$TEST_DIR/run-a" ( unset CLAUDE_MODEL CLAUDE_EFFORT CODEX_MODEL CODEX_REASONING_EFFORT MALFORMED_ENVELOPE unset COMPOSER_MODEL COMPOSER_EFFORT EXECUTOR_MODEL EXECUTOR_EFFORT VERIFIER_MODEL VERIFIER_EFFORT @@ -249,13 +252,12 @@ run_dir_before=$(latest_run_dir || true) # composer=claude, executor=codex, verifier=claude (opus alias). bounces 0 so # exactly two claude calls (compose + verify) and one codex call (execute). bash "$RUNNER" --composer opus --executor codex --verifier opus \ - --verify --bounces 0 --workdir "$repo" --timeout 60 -- "token capture mixed run" + --verify --bounces 0 --run-dir "$a_run_dir" --workdir "$repo" --timeout 60 -- "token capture mixed run" ) >"$TEST_DIR/a.out" 2>&1 || true -a_run_dir=$(latest_run_dir) a_ok=true a_msg="" -if [[ -z "$a_run_dir" || "$a_run_dir" == "$run_dir_before" ]]; then - a_ok=false; a_msg="no new run dir" +if [[ ! -f "$a_run_dir/state.json" ]]; then + a_ok=false; a_msg="no run dir/state.json produced" else state="$a_run_dir/state.json" # tokens block present, both phase kinds. @@ -289,7 +291,6 @@ else fail "flag ON mixed run failed ($a_msg) — run dir: ${a_run_dir:-}" [[ -n "${a_run_dir:-}" ]] && jq -c '.tokens // ""' "$a_run_dir/state.json" >&2 2>/dev/null || true fi -[[ -n "${a_run_dir:-}" && "$a_run_dir" != "$run_dir_before" ]] && rm -rf "$a_run_dir" # =========================================================================== # Scenario (b): flag ON, malformed claude envelope. The compose output file @@ -298,7 +299,7 @@ fi # =========================================================================== TOTAL=$((TOTAL + 1)) repo=$(make_scratch_repo b) -run_dir_before=$(latest_run_dir || true) +b_run_dir="$TEST_DIR/run-b" ( unset CLAUDE_MODEL CLAUDE_EFFORT CODEX_MODEL CODEX_REASONING_EFFORT unset COMPOSER_MODEL COMPOSER_EFFORT EXECUTOR_MODEL EXECUTOR_EFFORT VERIFIER_MODEL VERIFIER_EFFORT @@ -309,13 +310,12 @@ run_dir_before=$(latest_run_dir || true) # malformed compose envelope is the one under test. plan-only keeps it short: # the run reaches the plan-only terminal (still calls maybe_collect_token_usage). bash "$RUNNER" --composer opus --plan-only --bounces 0 \ - --workdir "$repo" --timeout 60 -- "token capture malformed envelope" + --run-dir "$b_run_dir" --workdir "$repo" --timeout 60 -- "token capture malformed envelope" ) >"$TEST_DIR/b.out" 2>&1 || true -b_run_dir=$(latest_run_dir) b_ok=true b_msg="" -if [[ -z "$b_run_dir" || "$b_run_dir" == "$run_dir_before" ]]; then - b_ok=false; b_msg="no new run dir" +if [[ ! -f "$b_run_dir/state.json" ]]; then + b_ok=false; b_msg="no run dir/state.json produced" else state="$b_run_dir/state.json" # The run produced a plan.md from the envelope-copy fallback (non-empty). @@ -338,7 +338,6 @@ if [[ "$b_ok" == true ]]; then else fail "flag ON malformed envelope failed ($b_msg) — run dir: ${b_run_dir:-}" fi -[[ -n "${b_run_dir:-}" && "$b_run_dir" != "$run_dir_before" ]] && rm -rf "$b_run_dir" # =========================================================================== # Scenario (c): flag OFF (default) — byte-parity guard. claude argv carries @@ -349,7 +348,7 @@ TOTAL=$((TOTAL + 1)) repo=$(make_scratch_repo c) claude_log="$TEST_DIR/c_claude.log"; codex_log="$TEST_DIR/c_codex.log" : > "$claude_log"; : > "$codex_log" -run_dir_before=$(latest_run_dir || true) +c_run_dir="$TEST_DIR/run-c" ( unset CLAUDE_MODEL CLAUDE_EFFORT CODEX_MODEL CODEX_REASONING_EFFORT MALFORMED_ENVELOPE unset COMPOSER_MODEL COMPOSER_EFFORT EXECUTOR_MODEL EXECUTOR_EFFORT VERIFIER_MODEL VERIFIER_EFFORT @@ -357,16 +356,15 @@ run_dir_before=$(latest_run_dir || true) export PATH="$TEST_DIR/bin:$PATH" export CLAUDE_ARGV_LOG="$claude_log" CODEX_ARGV_LOG="$codex_log" bash "$RUNNER" --composer opus --executor codex --verifier opus \ - --verify --bounces 0 --workdir "$repo" --timeout 60 -- "token capture parity off" + --verify --bounces 0 --run-dir "$c_run_dir" --workdir "$repo" --timeout 60 -- "token capture parity off" ) >"$TEST_DIR/c.out" 2>&1 || true -c_run_dir=$(latest_run_dir) c_ok=true c_msg="" # claude argv: text mode, never json. if ! grep -Eq -- '--output-format text' "$claude_log"; then c_ok=false; c_msg="claude not in text mode"; fi if grep -Eq -- '--output-format json' "$claude_log"; then c_ok=false; c_msg="claude leaked json mode with flag off"; fi -if [[ -z "$c_run_dir" || "$c_run_dir" == "$run_dir_before" ]]; then - c_ok=false; c_msg="no new run dir" +if [[ ! -f "$c_run_dir/state.json" ]]; then + c_ok=false; c_msg="no run dir/state.json produced" else # No sidecars anywhere (they were never written). if ls "$c_run_dir"/*.usage.json "$c_run_dir"/.*.usage.json >/dev/null 2>&1; then c_ok=false; c_msg="usage sidecar exists with flag off"; fi @@ -380,7 +378,6 @@ else fail "flag OFF parity guard failed ($c_msg) — run dir: ${c_run_dir:-}" [[ -s "$claude_log" ]] && { echo "--- claude argv ---"; cat "$claude_log"; } >&2 fi -[[ -n "${c_run_dir:-}" && "$c_run_dir" != "$run_dir_before" ]] && rm -rf "$c_run_dir" # =========================================================================== # Scenario (d): --help is unaffected by the capture machinery (exits 0, emits