From 4dba41a9e4b6748b7b51f02bf99fca3045ec52ea Mon Sep 17 00:00:00 2001 From: Daniel Young Date: Mon, 26 Oct 2026 18:24:56 -0400 Subject: [PATCH 01/11] feat: let a host name an adapter's session roots An adapter other than claude only read its manifest's default roots. A harness that keeps one session store per profile, as omp does under each profile's agent/sessions, was invisible: the default root can sit empty for weeks while every live session is somewhere else. AUTODREAM_SESSION_ROOTS_ (colon-separated) replaces the manifest defaults for that one adapter, the way SESSION_ROOTS works for claude. --- bin/run.sh | 11 ++++++++++- tests/run-all.sh | 14 ++++++++++++++ 2 files changed, 24 insertions(+), 1 deletion(-) diff --git a/bin/run.sh b/bin/run.sh index d2880cb..05a5dd5 100755 --- a/bin/run.sh +++ b/bin/run.sh @@ -78,6 +78,8 @@ # else (bin/triage-dream.sh; never fatal). Off, nothing changes. # AUTODREAM_L2_MODEL pin the L2 aggregator model (every engine) default: the adapter's own (claude: the CLI default) # AUTODREAM_L2_MODEL_ / AUTODREAM_L1_MODEL_ the same for one adapter only +# AUTODREAM_SESSION_ROOTS_ colon-separated session roots for one adapter other than +# claude, in place of its manifest's defaults default: the manifest's # AUTODREAM_MARKER_EPOCH first date whose report is REQUIRED to carry the # open-questions marker; earlier unmarked reports are treated # as complete (legacy) rather than abandoned @@ -674,7 +676,14 @@ adapter_roots() { # $1=adapter name -> one root per line printf '%s\n' "$SESSION_ROOTS" | tr ':' '\n' return 0 fi - local roots + local roots up var + # A host whose harness keeps sessions somewhere else (one store per profile, say) names them. + up=$(printf '%s' "$1" | tr 'a-z-' 'A-Z_') + var="AUTODREAM_SESSION_ROOTS_$up" + if [ -n "${!var:-}" ]; then + printf '%s\n' "${!var}" | tr ':' '\n' + return 0 + fi roots=$(adapter_manifest_get "$1" '.session_roots_default[]' 2>/dev/null) || return 0 [ -n "$roots" ] || return 0 printf '%s\n' "$roots" diff --git a/tests/run-all.sh b/tests/run-all.sh index 42ce0dd..c5f8b67 100755 --- a/tests/run-all.sh +++ b/tests/run-all.sh @@ -4252,6 +4252,19 @@ test_omp_session_is_linearized_for_the_worker(){ rm -rf "$root" } +test_omp_session_roots_come_from_the_host_setting(){ + echo "# AUTODREAM_SESSION_ROOTS_OMP replaces the omp manifest's default roots" + local root o alt fd; root=$(setup_env); mk_session "$root" sess1 + mk_omp_session "$root" jjjj >/dev/null + o=$(mk_omp_session "$root" kkkk); alt="$root/store/sessions/proj-alt"; mkdir -p "$alt" + mv "$o" "$alt/" + AUTODREAM_SESSION_ROOTS_OMP="$root/elsewhere:$root/store/sessions" run_dream_omp "$root" + fd=$(fdir "$root") + assert_grep "$fd/sessions.txt" 'store/sessions/proj-alt/.*_kkkk.jsonl' "a session under the configured root is enumerated" + assert_nogrep "$fd/sessions.txt" '_jjjj.jsonl' "the manifest's default root is not" + rm -rf "$root" +} + test_omp_stats_describe_the_live_branch_only(){ echo "# omp stats are computed on the live branch, so abandoned turns cannot lift a session past the noise gate" local root; root=$(setup_env); mk_session "$root" sess1 @@ -5432,6 +5445,7 @@ _run test_l1_precheck_probes_the_l1_provider _run test_worker_failure_probe_names_the_l1_provider _run test_one_dead_provider_does_not_hold_back_the_others _run test_omp_session_is_linearized_for_the_worker +_run test_omp_session_roots_come_from_the_host_setting _run test_omp_nested_child_is_a_sidechain_of_its_bucket _run test_omp_session_that_cannot_be_linearized_is_an_error_record _run test_omp_stats_describe_the_live_branch_only From b9906c6174cbb0340edc60717b2b34503f7c52fd Mon Sep 17 00:00:00 2001 From: Daniel Young Date: Tue, 27 Oct 2026 11:11:22 -0400 Subject: [PATCH 02/11] feat: separate who reviews a session from the harness that holds it Which harness holds a transcript and which engine reviews it were the same choice: each adapter's sessions went to its own engine in L1. With omp enabled, omp sessions could only be reviewed through omp and whichever provider its model routes to. They are now separate. A run has one reviewer, the engine that writes the summary (AUTODREAM_L2_ENGINE, else the first enabled adapter), and by default it reviews every source's sessions in L1 as well. AUTODREAM_L1_ENGINE picks another engine for L1, and AUTODREAM_L1_ENGINE_ one for a single source. The source's own adapter still enumerates, linearizes, describes and adds its triage notes; only the model call, its warmup and the network probe move. The default L1 model comes from the engine's manifest. A host with one harness behaves exactly as before. A host with several now has them all reviewed by one engine unless it asks otherwise, and AUTODREAM_L1_ENGINE_= keeps a harness reviewing its own. The engine is resolved in the parent and exported beside the source and model maps, so a worker cannot change which binary the next one runs. A choice that names no accepted adapter fails closed: those sessions get error records and no model call. run-stats records l1_engine_. Tests: tests/adapters.sh covers the resolution order. In run-all, the reviewer reads omp sessions while a tripwire omp binary records no calls, AUTODREAM_L2_ENGINE moves both passes whatever the adapter order, AUTODREAM_L1_ENGINE moves every source, and an unusable choice starts no worker. The existing omp tests pin AUTODREAM_L1_ENGINE_OMP=omp so they keep testing the omp engine. --- bin/adapters.sh | 34 +++++++++++++++--- bin/run.sh | 86 +++++++++++++++++++++++++++++++++----------- tests/adapters.sh | 17 +++++++++ tests/run-all.sh | 92 +++++++++++++++++++++++++++++++++++++++++++++-- 4 files changed, 203 insertions(+), 26 deletions(-) diff --git a/bin/adapters.sh b/bin/adapters.sh index 21e82ae..2bbbd32 100755 --- a/bin/adapters.sh +++ b/bin/adapters.sh @@ -196,21 +196,47 @@ adapter_run() { # $1=name $2=subcommand [args...] "$root/$name/adapter.sh" "$@" } +# The adapter whose engine reviews one source's sessions in L1. Which harness holds a transcript +# and which engine reads it are separate choices: the run has one reviewer, the engine that writes +# the summary, unless the host picks another, resolved in a fixed order: +# AUTODREAM_L1_ENGINE_ that source only (the name upper-cased, dashes to underscores) +# AUTODREAM_L1_ENGINE every source +# $2 the run's reviewer: the L2 engine (AUTODREAM_L2_ENGINE, else the +# first enabled adapter), passed by the runner +# the source itself only when no reviewer is passed +# Only the model call moves. The source's own adapter still enumerates, normalizes, describes and +# adds its triage notes. A choice that names no accepted adapter resolves nothing, so the caller +# leaves those sessions untriaged instead of falling back to some other engine. +adapter_l1_engine() { # $1=source $2=the run's reviewer -> engine adapter on stdout, exit 1 when none resolves + local name="$1" up var v + _adapter_name_safe "$name" || return 1 + up=$(printf '%s' "$name" | tr 'a-z-' 'A-Z_') + var="AUTODREAM_L1_ENGINE_$up" + v="${!var:-${AUTODREAM_L1_ENGINE:-${2:-$name}}}" + if [ "$v" != "$name" ]; then + _adapter_name_safe "$v" && _adapter_ok "$v" || return 1 + fi + printf '%s' "$v" +} + # The model one adapter's L1 workers run, resolved in a fixed order so a host can pin one # harness without touching another: # AUTODREAM_L1_MODEL_ that adapter only (the name upper-cased, dashes to underscores) # AUTODREAM_L1_MODEL every adapter -# the manifest's l1_model the adapter's own default +# the manifest's l1_model the default of the adapter whose engine runs it (adapter_l1_engine) # A model id belongs to one engine (neuralwatt/glm-5.3-flash means nothing to claude), which # is why the per-adapter form exists: the generic one is an override for hosts with one engine. -adapter_l1_model() { # $1=name -> model on stdout, exit 1 when none resolves - local name="$1" up var v +adapter_l1_model() { # $1=source $2=the run's reviewer -> model on stdout, exit 1 when none resolves + local name="$1" up var v engine _adapter_name_safe "$name" || return 1 up=$(printf '%s' "$name" | tr 'a-z-' 'A-Z_') var="AUTODREAM_L1_MODEL_$up" v="${!var:-}" [ -n "$v" ] || v="${AUTODREAM_L1_MODEL:-}" - [ -n "$v" ] || v=$(adapter_manifest_get "$name" '.l1_model' 2>/dev/null) || v="" + if [ -z "$v" ]; then + engine=$(adapter_l1_engine "$name" "${2:-}") || return 1 + v=$(adapter_manifest_get "$engine" '.l1_model' 2>/dev/null) || v="" + fi [ -n "$v" ] || return 1 printf '%s' "$v" } diff --git a/bin/run.sh b/bin/run.sh index 05a5dd5..03b0351 100755 --- a/bin/run.sh +++ b/bin/run.sh @@ -72,12 +72,21 @@ # AUTODREAM_WINDOW set 0 to place a session in a report day by its file mtime # alone, as before. On (default) it is placed by the timestamps # INSIDE the transcript, and stats and L1 see only that day default: 1 -# AUTODREAM_L2_ENGINE adapter whose engine runs L2 default: the first enabled adapter +# AUTODREAM_L2_ENGINE adapter whose engine runs L2, and by default reviews every source's +# sessions in L1 too: the run's reviewer default: the first enabled adapter # AUTODREAM_TRIAGE set 1 to triage each delivered report into dreams/DATE.triage.md default: 0 # One extra read-only call on the L2 engine and model, after everything # else (bin/triage-dream.sh; never fatal). Off, nothing changes. # AUTODREAM_L2_MODEL pin the L2 aggregator model (every engine) default: the adapter's own (claude: the CLI default) # AUTODREAM_L2_MODEL_ / AUTODREAM_L1_MODEL_ the same for one adapter only +# AUTODREAM_L1_ENGINE accepted adapter whose engine reviews every source's sessions in L1 +# default: the L2 engine +# AUTODREAM_L1_ENGINE_ the same for one source only, e.g. AUTODREAM_L1_ENGINE_OMP=omp +# Which harness holds a transcript and which engine reads it are separate: +# the source's own adapter still enumerates, linearizes, describes and adds +# its triage notes, and only the model call goes to the engine. A name that +# is not an accepted adapter leaves those sessions untriaged, never on +# another engine. The L1 model defaults to that engine's manifest l1_model. # AUTODREAM_SESSION_ROOTS_ colon-separated session roots for one adapter other than # claude, in place of its manifest's defaults default: the manifest's # AUTODREAM_MARKER_EPOCH first date whose report is REQUIRED to carry the @@ -725,6 +734,14 @@ resolve_enabled_adapters() { ENABLED_ADAPTERS_RESOLVED=1 return 0 } +# The run's reviewer: the engine that writes the summary (AUTODREAM_L2_ENGINE, else the first +# enabled adapter), and by default the engine that reviews every source in L1 as well. One +# function for both, so the per-session pass and the summary cannot drift onto different engines +# without the host asking (AUTODREAM_L1_ENGINE, AUTODREAM_L1_ENGINE_). +review_engine() { + resolve_enabled_adapters + printf '%s' "${AUTODREAM_L2_ENGINE:-${ENABLED_ADAPTERS%% *}}" +} _enabled_adapters_uncached() { # "Genuinely absent" means the loader is not sourced OR the adapters tree does # not exist — a tarball or partial install. That is a legacy install and it @@ -1916,14 +1933,14 @@ NET_DOWN_SECONDS=0 # The URLs the L1 workers need, one per distinct provider (any one answering opens the gate): each enabled adapter's own L1 model, # not a fixed host. Empty when no adapter has a resolvable provider, which wait_for_network reads -# as "nothing to ask". +# as "nothing to ask". Read from the per-source map, which was resolved against each source's L1 +# engine (AUTODREAM_L1_ENGINE_): an omp source on the claude engine needs anthropic. l1_probe_urls() { - local src model - while IFS=$'\t' read -r src model; do - [ -n "$src" ] || continue - provider_probe_url "$src" "$model" 2>/dev/null - echo - done <<< "${AUTODREAM_L1_MODELS:-}" | awk 'NF && !seen[$0]++' + local src url + while IFS=$'\t' read -r src url; do + [ -n "$src" ] && [ -n "$url" ] || continue + printf '%s\n' "$url" + done <<< "${AUTODREAM_L1_PROBE_URLS:-}" | awk 'NF && !seen[$0]++' } l2_probe_url() { provider_probe_url "$L2_ENGINE" "$L2_MODEL" 2>/dev/null; } @@ -2423,14 +2440,20 @@ dispatch_l1() { # one parallel pass; idempotent worker → only the still-missin # environment (tests/adapter-contract.sh), so nothing here names claude or omp. src=$(printf "%s\n" "$AUTODREAM_SOURCE_MAP" | awk -F"\t" -v h="$hash" "\$1 == h { print \$2; exit }") model=$(printf "%s\n" "$AUTODREAM_L1_MODELS" | awk -F"\t" -v s="$src" "\$1 == s { print \$2; exit }") + # The engine comes from the same parent-built environment (AUTODREAM_L1_ENGINE_), and is + # usually the source itself. An empty one means the override named no accepted adapter. + engine=$(printf "%s\n" "$AUTODREAM_L1_ENGINES" | awk -F"\t" -v s="$src" "\$1 == s { print \$2; exit }") argv=() envs=() case "$src" in ""|*[!abcdefghijklmnopqrstuvwxyz0123456789_-]*) src="" ;; esac - if [ -n "$src" ] && [ -n "$model" ] && [ -x "$ADAPTERS_DIR/$src/adapter.sh" ]; then - while IFS= read -r -d "" arg; do argv+=("$arg"); done < <("$ADAPTERS_DIR/$src/adapter.sh" l1-argv "$model" 2>/dev/null) - while IFS= read -r line; do [ -n "$line" ] && envs+=("$line"); done < <("$ADAPTERS_DIR/$src/adapter.sh" l1-env 2>/dev/null) + case "$engine" in + ""|*[!abcdefghijklmnopqrstuvwxyz0123456789_-]*) engine="" ;; + esac + if [ -n "$src" ] && [ -n "$engine" ] && [ -n "$model" ] && [ -x "$ADAPTERS_DIR/$engine/adapter.sh" ]; then + while IFS= read -r -d "" arg; do argv+=("$arg"); done < <("$ADAPTERS_DIR/$engine/adapter.sh" l1-argv "$model" 2>/dev/null) + while IFS= read -r line; do [ -n "$line" ] && envs+=("$line"); done < <("$ADAPTERS_DIR/$engine/adapter.sh" l1-env 2>/dev/null) fi if [ "${#argv[@]}" -eq 0 ]; then # Deterministic, so a structured error record that is left in place and skipped on re-run @@ -3319,16 +3342,31 @@ EOF ADAPTERS_DIR=$(adapters_root) AUTODREAM_SOURCE_MAP=$(cat "$FINDINGS_DIR/sessions-source.txt" 2>/dev/null) AUTODREAM_L1_MODELS="" + AUTODREAM_L1_ENGINES="" AUTODREAM_L1_PROBE_URLS="" - local _src _model _purl + local _src _model _purl _engine _reviewer + _reviewer=$(review_engine) while IFS= read -r _src; do [ -n "$_src" ] || continue - _model=$(adapter_l1_model "$_src" 2>/dev/null) || _model="" + # The engine first: a choice that names no accepted adapter leaves this source with no model, + # so its sessions get error records instead of a call on some other engine. + _engine=$(adapter_l1_engine "$_src" "$_reviewer" 2>/dev/null) || _engine="" + _model="" + if [ -n "$_engine" ]; then + _model=$(adapter_l1_model "$_src" "$_reviewer" 2>/dev/null) || _model="" + else + log "WARNING: the L1 engine chosen for adapter $_src names no accepted adapter; its sessions will not be triaged" + fi AUTODREAM_L1_MODELS="${AUTODREAM_L1_MODELS}${_src}"$'\t'"${_model}"$'\n' - _purl=$(provider_probe_url "$_src" "$_model" 2>/dev/null) || _purl="" + AUTODREAM_L1_ENGINES="${AUTODREAM_L1_ENGINES}${_src}"$'\t'"${_engine}"$'\n' + _purl=$(provider_probe_url "$_engine" "$_model" 2>/dev/null) || _purl="" AUTODREAM_L1_PROBE_URLS="${AUTODREAM_L1_PROBE_URLS}${_src}"$'\t'"${_purl}"$'\n' [ -n "$_model" ] || log "WARNING: no L1 model resolves for adapter $_src; its sessions will not be triaged" - log "L1 model for $_src: ${_model:-}" + if [ -n "$_engine" ] && [ "$_engine" != "$_src" ]; then + log "L1 model for $_src: ${_model:-} (on the $_engine engine)" + else + log "L1 model for $_src: ${_model:-}" + fi done < <(printf '%s\n' "$AUTODREAM_SOURCE_MAP" | awk -F'\t' 'NF >= 2 && !seen[$2]++ { print $2 }') AUTODREAM_NORMALIZE_SOURCES="" while IFS= read -r _src; do @@ -3336,7 +3374,7 @@ EOF [ "$(adapter_manifest_get "$_src" '.normalize' 2>/dev/null)" = "true" ] \ && AUTODREAM_NORMALIZE_SOURCES="${AUTODREAM_NORMALIZE_SOURCES:+$AUTODREAM_NORMALIZE_SOURCES }$_src" done < <(printf '%s\n' "$AUTODREAM_SOURCE_MAP" | awk -F'\t' 'NF >= 2 && !seen[$2]++ { print $2 }') - export ADAPTERS_DIR AUTODREAM_SOURCE_MAP AUTODREAM_L1_MODELS AUTODREAM_L1_PROBE_URLS AUTODREAM_NORMALIZE_SOURCES + export ADAPTERS_DIR AUTODREAM_SOURCE_MAP AUTODREAM_L1_MODELS AUTODREAM_L1_ENGINES AUTODREAM_L1_PROBE_URLS AUTODREAM_NORMALIZE_SOURCES # AUTODREAM_L1_ROUNDS is referenced by the dispatcher subshell to decide # whether this is the last retry round (gates the metadata-stub fallback). export AUTODREAM_L1_ROUNDS @@ -3390,9 +3428,12 @@ EOF L1_WARMUP=ok while IFS=$'\t' read -r _wsrc _wmodel; do [ -n "$_wsrc" ] && [ -n "$_wmodel" ] || continue + # Warm the engine the workers will call, which an AUTODREAM_L1_ENGINE_ override moves. + _weng=$(printf '%s\n' "$AUTODREAM_L1_ENGINES" | awk -F'\t' -v s="$_wsrc" '$1 == s { print $2; exit }') + [ -n "$_weng" ] || continue _wargv=(); _wenv=() - while IFS= read -r -d "" _a; do _wargv+=("$_a"); done < <(adapter_run "$_wsrc" warmup-argv "$_wmodel" 2>/dev/null) - while IFS= read -r _l; do [ -n "$_l" ] && _wenv+=("$_l"); done < <(adapter_run "$_wsrc" l1-env 2>/dev/null) + while IFS= read -r -d "" _a; do _wargv+=("$_a"); done < <(adapter_run "$_weng" warmup-argv "$_wmodel" 2>/dev/null) + while IFS= read -r _l; do [ -n "$_l" ] && _wenv+=("$_l"); done < <(adapter_run "$_weng" l1-env 2>/dev/null) if [ "${#_wargv[@]}" -eq 0 ]; then L1_WARMUP=failed log "L1 auth warmup FAILED for $_wsrc: the adapter printed no warmup command" @@ -3417,7 +3458,7 @@ EOF log "L1 auth warmup FAILED for $_wsrc (exit $_wrc): stdout=[${_wout:-}] stderr=[$(head -c 300 "$_werr" 2>/dev/null | tr '\n' ' ')]" fi done < <(printf '%s' "$AUTODREAM_L1_MODELS") - unset _wsrc _wmodel _wargv _wenv _a _l _werr _wout _wrc _wword + unset _wsrc _wmodel _weng _wargv _wenv _a _l _werr _wout _wrc _wword fi L1_START=$(date +%s) @@ -3837,6 +3878,11 @@ PY [ -n "$_msrc" ] || continue printf 'l1_model_%s: %s\n' "$(printf '%s' "$_msrc" | tr '-' '_')" "${_mval:-none}" done <<< "$AUTODREAM_L1_MODELS" + # And on which adapter's engine, which AUTODREAM_L1_ENGINE_ can move off the source. + while IFS=$'\t' read -r _msrc _mval; do + [ -n "$_msrc" ] || continue + printf 'l1_engine_%s: %s\n' "$(printf '%s' "$_msrc" | tr '-' '_')" "${_mval:-none}" + done <<< "${AUTODREAM_L1_ENGINES:-}" # yes when the circuit breaker cut the retry budget, so a short l1_rounds_used is not # mistaken for a run that finished early and cleanly. printf 'l1_breaker_fired: %s\n' "${L1_BREAKER:-not_reached}" @@ -4056,7 +4102,7 @@ PY # # bash 3.2 with set -u treats "${a[@]}" on an EMPTY array as an unbound variable, so every # expansion of the argv and env arrays below is guarded. - L2_ENGINE="${AUTODREAM_L2_ENGINE:-${ENABLED_ADAPTERS%% *}}" + L2_ENGINE=$(review_engine) L2_MODEL=$(adapter_l2_model "$L2_ENGINE" 2>/dev/null) || L2_MODEL="" log "L2 engine: $L2_ENGINE, model: ${L2_MODEL:-}" # Recorded because neither is fixed by this file. When a report's character changes, the first diff --git a/tests/adapters.sh b/tests/adapters.sh index 9edf6fa..e87a315 100644 --- a/tests/adapters.sh +++ b/tests/adapters.sh @@ -175,6 +175,23 @@ l1counts=$(mktemp "${TMPDIR:-/tmp}/l1model.XXXXXX") adapter_l1_model ../evil >/dev/null 2>&1; assert_eq "$?" "1" "an unsafe name resolves nothing" unset AUTODREAM_L1_MODEL AUTODREAM_L1_MODEL_OMP adapter_l1_model nonesuch >/dev/null 2>&1; assert_eq "$?" "1" "an adapter with no manifest and no override resolves nothing" + assert_eq "$(adapter_l1_engine omp)" "omp" "with no reviewer passed, a source reviews its own sessions" + assert_eq "$(adapter_l1_engine omp claude)" "claude" "the run's reviewer reviews every source" + assert_eq "$(adapter_l1_model omp claude)" "claude-haiku-5-5" "and the default model is the reviewer's, not the source's" + assert_eq "$(AUTODREAM_L1_ENGINE=omp adapter_l1_engine claude claude)" "omp" "AUTODREAM_L1_ENGINE replaces the reviewer for every source" + assert_eq "$(AUTODREAM_L1_ENGINE=omp AUTODREAM_L1_ENGINE_CLAUDE=claude adapter_l1_engine claude omp)" "claude" "and the per-source form beats it" + assert_eq "$(AUTODREAM_L1_ENGINE= adapter_l1_engine omp claude)" "claude" "an empty AUTODREAM_L1_ENGINE is not a choice" + adapter_l1_engine omp nonesuch >/dev/null 2>&1; assert_eq "$?" "1" "a reviewer that is not an accepted adapter resolves nothing" + AUTODREAM_L1_ENGINE_OMP=claude + assert_eq "$(adapter_l1_engine omp)" "claude" "the per-source engine choice moves it" + assert_eq "$(adapter_l1_model omp)" "claude-haiku-5-5" "and its default model is the engine's, not its own" + assert_eq "$(AUTODREAM_L1_MODEL_OMP=claude-x adapter_l1_model omp)" "claude-x" "an explicit model still wins" + AUTODREAM_L1_ENGINE_OMP=nonesuch + adapter_l1_engine omp >/dev/null 2>&1; assert_eq "$?" "1" "an override naming no accepted adapter resolves nothing" + adapter_l1_model omp >/dev/null 2>&1; assert_eq "$?" "1" "and leaves no default model to fall back on" + AUTODREAM_L1_ENGINE_OMP=../evil + adapter_l1_engine omp >/dev/null 2>&1; assert_eq "$?" "1" "an unsafe engine name resolves nothing" + unset AUTODREAM_L1_ENGINE_OMP printf '%s %s\n' "$pass" "$fail" > "$l1counts" ) # A missing counts file means the subshell died before reporting, which is a failure, not a diff --git a/tests/run-all.sh b/tests/run-all.sh index c5f8b67..189c72f 100755 --- a/tests/run-all.sh +++ b/tests/run-all.sh @@ -4170,9 +4170,13 @@ mk_omp_session(){ # $1=root $2=name [$3=cwd] -> path on stdout; abandoned branc printf '%s' "$f" } run_dream_omp(){ # $1=root ; claude + omp enabled, a sandbox HOME, both engines are the mock + # omp sessions are pinned to the omp engine, so the omp tests below exercise that engine. With + # claude enabled first the run's reviewer would be claude; a test about the reviewer exports + # AUTODREAM_L1_ENGINE_OMP= (empty) to clear the pin. mkdir -p "$1/home" - HOME="$1/home" AUTODREAM_ADAPTERS="${AUTODREAM_ADAPTERS:-claude,omp}" AUTODREAM_L1_MODEL_OMP="${TEST_OMP_L1_MODEL:-omp/test-model}" \ - AUTODREAM_CHANGELOG=0 CLAUDE_BIN="$MOCK" OMP_BIN="$MOCK" \ + HOME="$1/home" AUTODREAM_ADAPTERS="${AUTODREAM_ADAPTERS:-claude,omp}" AUTODREAM_L1_MODEL_OMP="${TEST_OMP_L1_MODEL-omp/test-model}" \ + AUTODREAM_L1_ENGINE_OMP="${AUTODREAM_L1_ENGINE_OMP-omp}" \ + AUTODREAM_CHANGELOG=0 CLAUDE_BIN="$MOCK" OMP_BIN="${TEST_OMP_BIN:-$MOCK}" \ AUTODREAM_CONFIG="$1/autodream/config" AUTODREAM_CONSUME_DATE="$DATE" \ AUTODREAM_NETCHECK="${AUTODREAM_NETCHECK:-0}" AUTODREAM_RETRY_WAIT=0 AUTODREAM_L1_ROUNDS=1 \ PROJECTS_DIR="$1/projects" AUTODREAM_DIR="$1/autodream" DREAMS_DIR="$1/dreams" \ @@ -4252,6 +4256,87 @@ test_omp_session_is_linearized_for_the_worker(){ rm -rf "$root" } +# An omp binary that records every call and fails, so a test can prove the omp engine never ran. +mk_omp_tripwire(){ # $1=root -> path on stdout; calls land in $1/omp-calls.log + printf '%s\n' '#!/bin/bash' "printf '%s\n' \"\$*\" >> '$1/omp-calls.log'" 'exit 7' > "$1/omp-tripwire" + chmod +x "$1/omp-tripwire" + printf '%s' "$1/omp-tripwire" +} + +test_omp_sessions_run_on_another_adapters_engine(){ + echo "# AUTODREAM_L1_ENGINE_OMP=claude: the claude engine reads omp sessions, and omp never runs" + local root o h fd trip; root=$(setup_env); mk_session "$root" sess1 + o=$(mk_omp_session "$root" ffff); h=$(hash_of "$o"); trip=$(mk_omp_tripwire "$root") + export FANOUT=1 MOCK_CAPTURE_DIR="$root/cap" AUTODREAM_L1_ENGINE_OMP=claude + TEST_OMP_BIN="$trip" TEST_OMP_L1_MODEL=claude-test-model run_dream_omp "$root" + unset FANOUT MOCK_CAPTURE_DIR AUTODREAM_L1_ENGINE_OMP + fd=$(fdir "$root") + assert_grep "$fd/sessions-source.txt" "^$h omp$" "the session is still the omp adapter's" + assert_grep "$root/cap/l1-read-$h.txt" 'LIVE_BRANCH_MARKER' "the omp adapter still linearized it" + assert_grep "$root/cap/l1-args-$h.txt" '^claude-test-model$' "the worker ran the configured model" + assert_grep "$root/cap/l1-args-$h.txt" '^--no-session-persistence$' "with the claude engine's flags" + assert_nogrep "$root/cap/l1-args-$h.txt" '^--allow-home$' "and not the omp engine's" + assert_nogrep "$fd/$h.json" '"error"' "the session was triaged" + assert_grep "$fd/run-stats.txt" '^l1_engine_omp: claude$' "run-stats names the engine" + assert_grep "$root/run.out" 'L1 model for omp: claude-test-model (on the claude engine)' "and so does the log" + assert_no_file "$root/omp-calls.log" "the omp binary was never called" + rm -rf "$root" +} + +test_the_runs_reviewer_reviews_every_source_by_default(){ + echo "# no L1 engine chosen: the L2 engine reviews every source's sessions, and omp never runs" + local root o h fd trip; root=$(setup_env); mk_session "$root" sess1 + o=$(mk_omp_session "$root" rrrr); h=$(hash_of "$o"); trip=$(mk_omp_tripwire "$root") + export FANOUT=1 MOCK_CAPTURE_DIR="$root/cap" AUTODREAM_L1_ENGINE_OMP= + TEST_OMP_BIN="$trip" TEST_OMP_L1_MODEL= run_dream_omp "$root" + unset FANOUT MOCK_CAPTURE_DIR AUTODREAM_L1_ENGINE_OMP + fd=$(fdir "$root") + assert_grep "$fd/run-stats.txt" '^l2_engine: claude$' "claude, the first enabled adapter, is the run's reviewer" + assert_grep "$fd/run-stats.txt" '^l1_engine_omp: claude$' "and reviews the omp sessions" + assert_grep "$root/cap/l1-args-$h.txt" '^claude-haiku-5-5$' "with the reviewer's default model, not omp's" + assert_grep "$root/cap/l1-read-$h.txt" 'LIVE_BRANCH_MARKER' "while the omp adapter still linearized the session" + assert_no_file "$root/omp-calls.log" "the omp binary was never called" + rm -rf "$root" + echo "# AUTODREAM_L2_ENGINE picks the reviewer for both passes, whatever the adapter order" + root=$(setup_env); mk_session "$root" sess1 + o=$(mk_omp_session "$root" ssss); h=$(hash_of "$o"); trip=$(mk_omp_tripwire "$root") + export FANOUT=1 MOCK_CAPTURE_DIR="$root/cap" AUTODREAM_L1_ENGINE_OMP= AUTODREAM_ADAPTERS=omp,claude AUTODREAM_L2_ENGINE=claude + TEST_OMP_BIN="$trip" TEST_OMP_L1_MODEL= run_dream_omp "$root" + unset FANOUT MOCK_CAPTURE_DIR AUTODREAM_L1_ENGINE_OMP AUTODREAM_ADAPTERS AUTODREAM_L2_ENGINE + fd=$(fdir "$root") + assert_grep "$fd/run-stats.txt" '^l1_engine_omp: claude$' "omp, though enabled first, is reviewed by the L2 engine" + assert_no_file "$root/omp-calls.log" "and the omp binary was never called" + rm -rf "$root" + echo "# AUTODREAM_L1_ENGINE picks one L1 engine for every source, claude sessions included" + local c hc; root=$(setup_env); mk_session "$root" sess1; c="$root/projects/proj-a/sess1.jsonl"; hc=$(hash_of "$c") + o=$(mk_omp_session "$root" tttt); h=$(hash_of "$o") + export FANOUT=1 MOCK_CAPTURE_DIR="$root/cap" AUTODREAM_L1_ENGINE_OMP= AUTODREAM_L1_ENGINE=omp + TEST_OMP_L1_MODEL= AUTODREAM_L1_MODEL=omp/test-model run_dream_omp "$root" + unset FANOUT MOCK_CAPTURE_DIR AUTODREAM_L1_ENGINE_OMP AUTODREAM_L1_ENGINE + fd=$(fdir "$root") + assert_grep "$fd/run-stats.txt" '^l1_engine_claude: omp$' "the claude sessions are reviewed by omp" + assert_grep "$fd/run-stats.txt" '^l1_engine_omp: omp$' "and so are the omp sessions" + assert_grep "$root/cap/l1-args-$hc.txt" '^--allow-home$' "with the omp engine's flags" + assert_grep "$fd/run-stats.txt" '^l2_engine: claude$' "while the summary stays on the L2 engine" + rm -rf "$root" +} + +test_an_unusable_l1_engine_override_fails_closed(){ + echo "# an L1 engine override that names no accepted adapter: no model call at all, not the omp engine" + local root o h fd trip; root=$(setup_env); mk_session "$root" sess1 + o=$(mk_omp_session "$root" gggg); h=$(hash_of "$o"); trip=$(mk_omp_tripwire "$root") + export FANOUT=1 MOCK_CAPTURE_DIR="$root/cap" AUTODREAM_L1_ENGINE_OMP=nonesuch + TEST_OMP_BIN="$trip" run_dream_omp "$root" + unset FANOUT MOCK_CAPTURE_DIR AUTODREAM_L1_ENGINE_OMP + fd=$(fdir "$root") + assert_no_file "$root/cap/l1-args-$h.txt" "no worker started for the omp session" + assert_grep "$fd/$h.json" 'no L1 engine for this session' "it got the deterministic error record" + assert_grep "$root/run.out" 'names no accepted adapter' "the log says why" + assert_grep "$fd/run-stats.txt" '^l1_engine_omp: none$' "run-stats shows no engine" + assert_no_file "$root/omp-calls.log" "the omp binary was never called" + rm -rf "$root" +} + test_omp_session_roots_come_from_the_host_setting(){ echo "# AUTODREAM_SESSION_ROOTS_OMP replaces the omp manifest's default roots" local root o alt fd; root=$(setup_env); mk_session "$root" sess1 @@ -5445,6 +5530,9 @@ _run test_l1_precheck_probes_the_l1_provider _run test_worker_failure_probe_names_the_l1_provider _run test_one_dead_provider_does_not_hold_back_the_others _run test_omp_session_is_linearized_for_the_worker +_run test_omp_sessions_run_on_another_adapters_engine +_run test_the_runs_reviewer_reviews_every_source_by_default +_run test_an_unusable_l1_engine_override_fails_closed _run test_omp_session_roots_come_from_the_host_setting _run test_omp_nested_child_is_a_sidechain_of_its_bucket _run test_omp_session_that_cannot_be_linearized_is_an_error_record From 930f5137d0c9f7e018712c60f2f5fd20fd6200ac Mon Sep 17 00:00:00 2001 From: Daniel Young Date: Tue, 27 Oct 2026 10:41:08 -0400 Subject: [PATCH 03/11] feat: record who each review engine runs as and which model answered A run named the models it asked for and nothing else. Nothing showed which account or API the reviewing engines were signed in to, or which model actually answered each L1 call, so a report could not be traced to the model behind it and a substituted model would go unnoticed. Each engine the run uses, the L2 engine and every L1 engine, reports its login once through a new adapter verb, whoami: API provider, sign-in method, plan and account. For claude that is `claude auth status --json`, which makes no model call. It is logged and written to run-stats as login_. The claude L1 worker now runs with --output-format json, and the new served verb reads the models that answered from the result object. Every call is recorded in findings//l1-served.tsv and summed in run-stats (l1_served_models, l1_served_mismatch, and l1_cost_usd_estimate, the CLI's estimate at API prices). Both are record-only unless the host asks: AUTODREAM_REQUIRE_LOGIN="provider=firstParty auth=claude.ai plan=max" an L1 engine whose login does not match is handed no transcript, through the same error-record path as an unusable engine choice; an L2 engine that does not match stops the run before it starts AUTODREAM_L1_REQUIRE_SERVED=1 an L1 call answered by any model other than the requested one loses its findings, so nothing a substituted model wrote reaches L2 Not stream-json: --verbose echoes every Read result, the whole transcript, onto stdout, and a failed worker copies its stdout into the .err file L2 reads. --- adapters/claude/adapter.sh | 26 ++++++++++++ bin/run.sh | 86 +++++++++++++++++++++++++++++++++++++- tests/adapter-claude.sh | 38 ++++++++++++++--- tests/mock-claude.sh | 44 ++++++++++++++++--- tests/run-all.sh | 69 ++++++++++++++++++++++++++++++ 5 files changed, 251 insertions(+), 12 deletions(-) diff --git a/adapters/claude/adapter.sh b/adapters/claude/adapter.sh index 884dce2..566ab80 100755 --- a/adapters/claude/adapter.sh +++ b/adapters/claude/adapter.sh @@ -170,11 +170,15 @@ case "$cmd" in # adapter: tests/adapter-claude.sh pins it against that literal text, so a change here # that alters what the nightly runs has to say so. NO shell expansion of the paths in # the system prompt, hence the escaped dollar sign in the text. + # --output-format json makes stdout one small result object that names the models that + # answered (read back by `served`). Not stream-json: --verbose would echo every Read result, + # the whole transcript, onto stdout, and a failed worker copies its stdout into the .err. effort=$(l1_effort) || exit 2 set -- "$1" ${effort:+--effort "$effort"} model=$1; shift printf '%s\0' "${CLAUDE_BIN:-$HOME/.local/bin/claude}" \ --print \ + --output-format json \ --permission-mode bypassPermissions \ --model "$model" \ "$@" \ @@ -207,6 +211,28 @@ case "$cmd" in --system-prompt 'Reply with the single word ok and exit.' ;; + served) # $1=stdout of one L1 call -> "[,...]", exit 1 without a result + # The models come from the result object's modelUsage, which lists every model that answered + # any part of the call. The cost is the CLI's estimate at API prices, not what a subscription + # bills; it is kept because it compares models on real numbers. + [ "$#" -ge 1 ] && [ -r "$1" ] || exit 1 + line=$(jq -r 'select(type == "object" and .type == "result") + | [((.modelUsage // {}) | keys | sort | join(",")), ((.total_cost_usd // "") | tostring)] | @tsv' "$1" 2>/dev/null | head -1) + case "$line" in ""|$'\t'*) exit 1 ;; esac + printf '%s\n' "$line" + ;; + + whoami) # -> one JSON object {provider, auth, plan, account}, each "unknown" when not reported + # Who this engine's calls run as, read from the login they will use, with no model call: + # `claude auth status --json` reports the API provider (firstParty is Anthropic's own API), + # the sign-in method, the subscription plan and the account email. + out=$("${CLAUDE_BIN:-$HOME/.local/bin/claude}" auth status --json /dev/null) || out="" + [ -n "$out" ] || out="{}" + printf '%s' "$out" | jq -c '{provider: (.apiProvider // "unknown"), auth: (.authMethod // "unknown"), + plan: (.subscriptionType // "unknown"), account: (.email // "unknown")} | map_values(tostring)' 2>/dev/null \ + || printf '{"provider":"unknown","auth":"unknown","plan":"unknown","account":"unknown"}\n' + ;; + l1-env) # -> KEY=VALUE lines the engine needs in its environment # Lean-query env: keep subscription auth, strip per-call bloat (no CLAUDE.md auto-load, # no telemetry or error reporting). See CLAUDE.md, "How claude is invoked". diff --git a/bin/run.sh b/bin/run.sh index 03b0351..bdabab2 100755 --- a/bin/run.sh +++ b/bin/run.sh @@ -89,6 +89,14 @@ # another engine. The L1 model defaults to that engine's manifest l1_model. # AUTODREAM_SESSION_ROOTS_ colon-separated session roots for one adapter other than # claude, in place of its manifest's defaults default: the manifest's +# AUTODREAM_REQUIRE_LOGIN space-separated key=value pairs the login of every engine the run uses +# must match before it is handed anything; keys provider, auth, plan, +# account, as the engine's whoami verb reports them (claude: claude auth +# status). An L1 engine that fails triages nothing; a failing L2 engine +# stops the run before it starts default: record only +# (e.g. "provider=firstParty auth=claude.ai plan=max") +# AUTODREAM_L1_REQUIRE_SERVED 1 discards the findings of any L1 call answered by a model other +# than the requested one (the engine's served verb) default: 0, record only # AUTODREAM_MARKER_EPOCH first date whose report is REQUIRED to carry the # open-questions marker; earlier unmarked reports are treated # as complete (legacy) rather than abandoned @@ -742,6 +750,38 @@ review_engine() { resolve_enabled_adapters printf '%s' "${AUTODREAM_L2_ENGINE:-${ENABLED_ADAPTERS%% *}}" } +# Who an engine's calls run as, read once per engine before it is handed anything: the adapter's +# whoami verb reports the API provider, sign-in method, plan and account of its login (claude: +# `claude auth status`, no model call), and an engine without the verb reports unknown. Each answer +# is logged once and kept in ENGINE_LOGINS (engine, provider, auth, plan, account; tab-separated) +# for run-stats. AUTODREAM_REQUIRE_LOGIN lists key=value pairs that must all match; on a mismatch +# this returns 1 with the reason in LOGIN_REFUSAL. +ENGINE_LOGINS="" +LOGIN_REFUSAL="" +engine_login_ok() { # $1=engine -> 0 when its login meets AUTODREAM_REQUIRE_LOGIN, or nothing is required + local eng="$1" row who req k want have + row=$(printf '%s' "$ENGINE_LOGINS" | awk -F'\t' -v e="$eng" '$1 == e { print; exit }') + if [ -z "$row" ]; then + who=$(adapter_run "$eng" whoami 2>/dev/null) || who="" + printf '%s' "$who" | jq -e 'type == "object"' >/dev/null 2>&1 || who='{}' + row="$eng"$'\t'"$(printf '%s' "$who" | jq -r '[.provider, .auth, .plan, .account] | map(. // "unknown" | tostring) | @tsv')" + ENGINE_LOGINS="${ENGINE_LOGINS}${row}"$'\n' + log "login for the $eng engine: $(printf '%s' "$row" | awk -F'\t' '{ printf "provider=%s auth=%s plan=%s account=%s", $2, $3, $4, $5 }')" + fi + LOGIN_REFUSAL="" + for req in ${AUTODREAM_REQUIRE_LOGIN:-}; do + k=${req%%=*}; want=${req#*=} + case "$k" in + provider) have=$(printf '%s' "$row" | cut -f2) ;; + auth) have=$(printf '%s' "$row" | cut -f3) ;; + plan) have=$(printf '%s' "$row" | cut -f4) ;; + account) have=$(printf '%s' "$row" | cut -f5) ;; + *) LOGIN_REFUSAL="$req is not a provider, auth, plan or account requirement"; return 1 ;; + esac + [ "$have" = "$want" ] || { LOGIN_REFUSAL="$k is $have, not $want"; return 1; } + done + return 0 +} _enabled_adapters_uncached() { # "Genuinely absent" means the loader is not sourced OR the adapters tree does # not exist — a tarball or partial install. That is a legacy install and it @@ -2607,6 +2647,20 @@ dispatch_l1() { # one parallel pass; idempotent worker → only the still-missin rm -f "$2" fi + # Which model answered, from the engine result record (adapter verb served), for every call + # whatever it produced, so a night accounts for each one. Engines that report nothing are + # recorded as unknown. With AUTODREAM_L1_REQUIRE_SERVED=1 a call answered by anything other + # than the requested model loses its findings, so nothing a substituted model wrote reaches L2. + l1served=$("$ADAPTERS_DIR/$engine/adapter.sh" served "$4" 2>/dev/null) || l1served="" + l1smodels=$(printf "%s" "$l1served" | cut -f1) + l1scost=$(printf "%s" "$l1served" | cut -s -f2) + printf "%s\t%s\t%s\t%s\t%s\n" "$hash" "$engine" "$model" "${l1smodels:-unknown}" "$l1scost" >> "$FINDINGS_DIR/l1-served.tsv" + if [ "${AUTODREAM_L1_REQUIRE_SERVED:-0}" = 1 ] && [ -s "$2" ] && [ "$l1smodels" != "$model" ]; then + printf "the call was answered by [%s], not the requested model [%s]; its findings are discarded (AUTODREAM_L1_REQUIRE_SERVED=1)\n" "${l1smodels:-unknown}" "$model" >> "$3" + rm -f "$2" + return 1 + fi + # Non-empty is not the same as valid. A worker that writes malformed JSON, or JSON with no # .findings array, used to take the success branch below: both diagnostics were deleted and # the file was left for L2. The dispatcher validates .findings on its way IN, so the next @@ -3075,7 +3129,7 @@ run() { log_fatal "prompts/L2_SYSTEM.txt (read through bin/l2-prompt.sh) was not found beside run.sh, its link target or in AUTODREAM_DIR. Refusing to start." fatal_exit; return 1 fi - _l2e="${AUTODREAM_L2_ENGINE:-${ENABLED_ADAPTERS%% *}}" + _l2e=$(review_engine) if [ "$_l2e" != "claude" ] && [ -n "$_l2e" ]; then _l2m=$(adapter_l2_model "$_l2e" 2>/dev/null) || _l2m="" if ! adapter_run "$_l2e" l2-argv ${_l2m:+"$_l2m"} 2>/dev/null | head -c 1 | grep -q .; then @@ -3083,6 +3137,12 @@ run() { fatal_exit; return 1 fi fi + # The engine that writes the summary (and runs dream triage) receives every finding, so its login + # is held to AUTODREAM_REQUIRE_LOGIN before anything is read. + if ! engine_login_ok "$_l2e"; then + log_fatal "the $_l2e engine, which writes the summary, fails AUTODREAM_REQUIRE_LOGIN ($LOGIN_REFUSAL). Refusing to start." + fatal_exit; return 1 + fi # Exclude autodream's OWN headless worker/aggregator transcripts. New runs leave none # (--no-session-persistence), but runs predating that fix littered ~/.claude/projects/ @@ -3374,6 +3434,16 @@ EOF [ "$(adapter_manifest_get "$_src" '.normalize' 2>/dev/null)" = "true" ] \ && AUTODREAM_NORMALIZE_SOURCES="${AUTODREAM_NORMALIZE_SOURCES:+$AUTODREAM_NORMALIZE_SOURCES }$_src" done < <(printf '%s\n' "$AUTODREAM_SOURCE_MAP" | awk -F'\t' 'NF >= 2 && !seen[$2]++ { print $2 }') + # Every L1 engine is held to AUTODREAM_REQUIRE_LOGIN before any transcript is sent (engine_login_ok). + # One that fails is taken out of AUTODREAM_L1_ENGINES, so the sessions it would have read get error + # records and no model call, the same path as a choice naming no accepted adapter. + local _eng + while IFS= read -r _eng; do + [ -n "$_eng" ] || continue + engine_login_ok "$_eng" && continue + log "WARNING: the $_eng engine fails AUTODREAM_REQUIRE_LOGIN ($LOGIN_REFUSAL); the sessions it would read will not be triaged" + AUTODREAM_L1_ENGINES=$(printf '%s' "$AUTODREAM_L1_ENGINES" | awk -F'\t' -v e="$_eng" 'BEGIN { OFS = "\t" } $2 == e { $2 = "" } { print }')$'\n' + done < <(printf '%s' "$AUTODREAM_L1_ENGINES" | awk -F'\t' 'NF >= 2 && $2 != "" && !seen[$2]++ { print $2 }') export ADAPTERS_DIR AUTODREAM_SOURCE_MAP AUTODREAM_L1_MODELS AUTODREAM_L1_ENGINES AUTODREAM_L1_PROBE_URLS AUTODREAM_NORMALIZE_SOURCES # AUTODREAM_L1_ROUNDS is referenced by the dispatcher subshell to decide # whether this is the last retry round (gates the metadata-stub fallback). @@ -3883,6 +3953,20 @@ PY [ -n "$_msrc" ] || continue printf 'l1_engine_%s: %s\n' "$(printf '%s' "$_msrc" | tr '-' '_')" "${_mval:-none}" done <<< "${AUTODREAM_L1_ENGINES:-}" + # Who each engine the run used ran as (engine_login_ok), and which models answered the L1 calls + # recorded for this date in l1-served.tsv, so a report traces to the login and model behind it. + # The cost is the engine's own estimate at API prices, not what a subscription bills. + while IFS=$'\t' read -r _msrc _pprov _pauth _pplan _pacct; do + [ -n "$_msrc" ] || continue + printf 'login_%s: provider=%s auth=%s plan=%s account=%s\n' "$(printf '%s' "$_msrc" | tr '-' '_')" "$_pprov" "$_pauth" "$_pplan" "$_pacct" + done <<< "${ENGINE_LOGINS:-}" + if [ -s "$FINDINGS_DIR/l1-served.tsv" ]; then + printf 'l1_served_models: %s\n' "$(cut -f4 "$FINDINGS_DIR/l1-served.tsv" | sort | uniq -c | awk '{ printf "%s%s=%s", (NR > 1 ? " " : ""), $2, $1 }')" + printf 'l1_served_mismatch: %s\n' "$(awk -F'\t' '$4 != $3' "$FINDINGS_DIR/l1-served.tsv" | wc -l | tr -d ' ')" + printf 'l1_cost_usd_estimate: %s\n' "$(awk -F'\t' '$5 != "" { s += $5 } END { printf "%.4f", s }' "$FINDINGS_DIR/l1-served.tsv")" + else + printf 'l1_served_models: none\nl1_served_mismatch: 0\nl1_cost_usd_estimate: 0.0000\n' + fi # yes when the circuit breaker cut the retry budget, so a short l1_rounds_used is not # mistaken for a run that finished early and cleanly. printf 'l1_breaker_fired: %s\n' "${L1_BREAKER:-not_reached}" diff --git a/tests/adapter-claude.sh b/tests/adapter-claude.sh index 79b9317..b0b566b 100644 --- a/tests/adapter-claude.sh +++ b/tests/adapter-claude.sh @@ -252,15 +252,16 @@ case "$L2" in *AUTODREAM_REPORT_END*AUTODREAM_PINS_BEGIN*) ok "the system prompt echo "# l1-argv reproduces the invocation run.sh hard-coded before the engine moved behind the adapter" # The expected text is copied from run.sh as it stood at b473c6f6d (the dispatch_l1 call), not -# derived from the adapter, so this fails if the adapter and the old nightly ever disagree. +# derived from the adapter, so this fails if the adapter and the old nightly ever disagree. The one +# deliberate change since then is --output-format json, which `served` reads the answering model from. OLD_SYSPROMPT="Headless triage worker. Read the session transcript and write exactly one findings JSON object, via the Write tool, to the literal output path given on line 2 of the prompt. Those paths are literal strings, not shell variables — never \$-expand them. Print only the literal word done and exit." OLD_SETTINGS="{\"disableAllHooks\":true}" -printf '%s\0' /opt/test/claude --print --permission-mode bypassPermissions --model claude-haiku-5-5 --effort high \ +printf '%s\0' /opt/test/claude --print --output-format json --permission-mode bypassPermissions --model claude-haiku-5-5 --effort high \ --no-session-persistence --tools Read Write --disable-slash-commands --strict-mcp-config \ --settings "$OLD_SETTINGS" --system-prompt "$OLD_SYSPROMPT" > "$tmp/old-argv" CLAUDE_BIN=/opt/test/claude "$A" l1-argv claude-haiku-5-5 > "$tmp/new-argv" -if cmp -s "$tmp/old-argv" "$tmp/new-argv"; then ok "the claude L1 argv is the old hard-coded one with --effort high and --system-prompt" -else no "the claude L1 argv is the old hard-coded one with --effort high and --system-prompt"; fi +if cmp -s "$tmp/old-argv" "$tmp/new-argv"; then ok "the claude L1 argv is the old hard-coded one with --output-format json, --effort high and --system-prompt" +else no "the claude L1 argv is the old hard-coded one with --output-format json, --effort high and --system-prompt"; fi echo "# l1 effort: env override, off, and an invalid level" assert_eq "$(AUTODREAM_L1_EFFORT=low CLAUDE_BIN=/opt/test/claude "$A" l1-argv m | tr '\0' '\n' | sed -n '/^--effort$/{n;p;}')" "low" "AUTODREAM_L1_EFFORT sets the level" assert_eq "$(AUTODREAM_L1_EFFORT=low AUTODREAM_L1_EFFORT_CLAUDE=max CLAUDE_BIN=/opt/test/claude "$A" l1-argv m | tr '\0' '\n' | sed -n '/^--effort$/{n;p;}')" "max" "the per-adapter variable wins" @@ -274,8 +275,33 @@ echo "# warmup-argv: the worker's flags with a one-word system prompt" WA=$(CLAUDE_BIN=/opt/test/claude "$A" warmup-argv claude-haiku-5-5 | tr '\0' '\n') assert_eq "$(printf '%s\n' "$WA" | tail -1)" "Reply with the single word ok and exit." "the system prompt asks for one word" LA=$(CLAUDE_BIN=/opt/test/claude "$A" l1-argv claude-haiku-5-5 | tr '\0' '\n') -# Everything but the tools list and the system prompt must match the worker's, so the warmup takes the same auth path. -assert_eq "$(printf '%s\n' "$WA" | sed -n '1,/^--tools$/p' | sed '$d')" "$(printf '%s\n' "$LA" | sed -n '1,/^--tools$/p' | sed '$d')" "the flags before --tools match the worker's" +# Everything but the tools list, the system prompt and the worker's output format must match the +# worker's, so the warmup takes the same auth path. The warmup keeps plain text: it is judged on the +# word ok, and the output format changes only what stdout carries. +assert_eq "$(printf '%s\n' "$WA" | sed -n '1,/^--tools$/p' | sed '$d')" "$(printf '%s\n' "$LA" | sed '/^--output-format$/{N;d;}' | sed -n '1,/^--tools$/p' | sed '$d')" "the flags before --tools match the worker's" + +echo "# served: the models that answered one L1 call, from its result object" +printf '%s\n' '{"type":"result","is_error":false,"result":"done","modelUsage":{"claude-sonnet-5-5":{}},"total_cost_usd":0.25}' > "$tmp/r1" +assert_eq "$("$A" served "$tmp/r1")" "claude-sonnet-5-5 0.25" "one model and the cost estimate" +printf '%s\n' '{"type":"result","modelUsage":{"m-b":{},"m-a":{}},"total_cost_usd":1}' > "$tmp/r2" +assert_eq "$("$A" served "$tmp/r2")" "m-a,m-b 1" "every model that answered, sorted" +printf '%s\n' '{"type":"system","subtype":"init"}' '{"type":"result","modelUsage":{"m":{}}}' > "$tmp/r3" +assert_eq "$("$A" served "$tmp/r3")" "m " "a stream is read for its result line, and a missing cost is empty" +printf 'done\n' > "$tmp/r4" +"$A" served "$tmp/r4" >/dev/null 2>&1; assert_eq "$?" "1" "plain text carries no record" +printf '%s\n' '{"type":"result","is_error":true,"result":"Not logged in","modelUsage":{}}' > "$tmp/r5" +"$A" served "$tmp/r5" >/dev/null 2>&1; assert_eq "$?" "1" "a result no model answered carries no record" +"$A" served "$tmp/nonesuch" >/dev/null 2>&1; assert_eq "$?" "1" "a missing file carries no record" + +echo "# whoami: who the engine's calls run as, read from claude auth status" +printf '%s\n' '#!/bin/sh' '[ "$1 $2 $3" = "auth status --json" ] || exit 9' \ + "printf '%s\n' '{\"loggedIn\":true,\"authMethod\":\"claude.ai\",\"apiProvider\":\"firstParty\",\"email\":\"a@example.com\",\"subscriptionType\":\"max\",\"orgId\":\"x\"}'" > "$tmp/fake-claude" +chmod +x "$tmp/fake-claude" +assert_eq "$(CLAUDE_BIN="$tmp/fake-claude" "$A" whoami)" '{"provider":"firstParty","auth":"claude.ai","plan":"max","account":"a@example.com"}' "provider, sign-in, plan and account, nothing else" +printf '%s\n' '#!/bin/sh' "printf '%s\n' '{\"loggedIn\":false,\"authMethod\":\"none\",\"apiProvider\":\"firstParty\",\"subscriptionType\":null}'; exit 1" > "$tmp/fake-claude-out" +chmod +x "$tmp/fake-claude-out" +assert_eq "$(CLAUDE_BIN="$tmp/fake-claude-out" "$A" whoami)" '{"provider":"unknown","auth":"unknown","plan":"unknown","account":"unknown"}' "a failing status call reports unknown, not what it half-printed" +assert_eq "$(CLAUDE_BIN=/usr/bin/false "$A" whoami)" '{"provider":"unknown","auth":"unknown","plan":"unknown","account":"unknown"}' "no status at all reports unknown" printf '\npassed: %s failed: %s\n' "$pass" "$fail" [ "$fail" -eq 0 ] diff --git a/tests/mock-claude.sh b/tests/mock-claude.sh index 50714fe..e44966a 100755 --- a/tests/mock-claude.sh +++ b/tests/mock-claude.sh @@ -82,9 +82,43 @@ # model was (or was not) invoked for a given session # (e.g. a noise-gated session should never appear). +# MOCK_AUTH_STATUS= what `auth status --json` prints (the claude adapter's provenance verb); +# default a logged-in claude.ai max login on Anthropic's own API. +# MOCK_SERVED_MODEL= the model a successful L1 call says answered it, when the call asks for +# --output-format json (default: the requested --model). + +# `auth status` sends no stdin, so it is answered before stdin is read. +if [ "${1:-}" = "auth" ] && [ "${2:-}" = "status" ]; then + if [ -n "${MOCK_AUTH_STATUS:-}" ]; then + printf '%s\n' "$MOCK_AUTH_STATUS" + else + printf '%s\n' '{"loggedIn":true,"authMethod":"claude.ai","apiProvider":"firstParty","email":"test@example.com","subscriptionType":"max"}' + fi + exit 0 +fi + input=$(cat) mode="${MOCK_MODE:-good}" +out_format=""; req_model=""; prev="" +for a in "$@"; do + case "$prev" in + --output-format) out_format=$a ;; + --model) req_model=$a ;; + esac + prev=$a +done +# What a successful L1 call prints: the word done, or with --output-format json the result object +# whose modelUsage names the model that answered, as the real CLI prints it. +l1_reply() { + if [ "$out_format" = json ]; then + jq -cn --arg m "${MOCK_SERVED_MODEL:-$req_model}" \ + '{type: "result", subtype: "success", is_error: false, result: "done", modelUsage: {($m): {costUSD: 0.0125}}, total_cost_usd: 0.0125}' + else + echo done + fi +} + # $1=findings dir. Adds $MOCK_FORGED_SESSION to the runner's worklist files, the way a # prompt-injected model with the Write tool could. tamper_worklist() { @@ -162,7 +196,7 @@ if printf '%s' "$line1" | grep -q '^Session transcript'; then typed) printf '{"session_path":"x","findings":"oops"}' > "$out" ;; two) printf '{"error":"x","findings":[]}\n{"session_path":"x","findings":[]}\n' > "$out" ;; esac - echo done + l1_reply exit 0 fi write_findings() { printf '{"session_path":"x","project":"proj-a","turn_count":2,"tool_call_count":0,"tools_used":[],"skills_invoked":[],"models_used":[],"notable_initiatives":[],"underlying_goal":null,"outcome":"fully_achieved","satisfaction_signals":{"happy":0,"satisfied":1,"dissatisfied":0,"frustrated":0},"instructions_given":["always run tests after edits"],"findings":[]}' > "$out"; } @@ -179,13 +213,13 @@ if printf '%s' "$line1" | grep -q '^Session transcript'; then # The runner used to accept any non-empty file as success, delete both diagnostics, # and hand this to L2 on the final round. printf 'this is not json at all\n' > "$out" - echo done + l1_reply exit 0 ;; l1_wrongtype) # .findings present but a STRING, not an array. # jq -e .findings is truthy for this, so it used to pass both the idempotency read # and the outbound validation and reach L2 as a successful result. printf '{"session_path":"x","findings":"oops"}' > "$out" - echo done + l1_reply exit 0 ;; l1_noisy_fail) # writes nothing, but says WHY on stdout and exits # nonzero. This is the real shape: omp puts its diagnosis on stdout and only @@ -231,7 +265,7 @@ if printf '%s' "$line1" | grep -q '^Session transcript'; then fi ;; l1_nested_error) # a real finding that happens to carry an error key printf '{"session_path":"x","project":"proj-a","findings":[{"category":"tool_loop","error":"ENOENT while reading a file","severity":"low"}]}' > "$out" - echo done ;; + l1_reply ;; l1_badproject|pins|pins_partial|pins_tamper) write_badproject ;; # wrong project + real path — exercises normalization pins_tamper_l1) write_badproject; tamper_worklist "$(dirname "$out")" ;; pins_ledger_wipe) write_badproject; : > "$(dirname "$out")/pins-applied.tsv" ;; # a worker empties the ledger @@ -241,7 +275,7 @@ if printf '%s' "$line1" | grep -q '^Session transcript'; then if [ -f "$out.attempt" ]; then write_findings; else : > "$out.attempt"; fi ;; *) write_findings ;; esac - echo done + l1_reply elif printf '%s' "$input" | grep -q '^# Dream triage worker'; then # ---- Dream triage (bin/triage-dream.sh): its own capture files, never l2-*, so a test that # asserts on L2's args or stdin cannot be overwritten by this call ---- diff --git a/tests/run-all.sh b/tests/run-all.sh index 189c72f..54f2a05 100755 --- a/tests/run-all.sh +++ b/tests/run-all.sh @@ -4350,6 +4350,72 @@ test_omp_session_roots_come_from_the_host_setting(){ rm -rf "$root" } +test_l1_provenance_and_served_models_are_recorded(){ + echo "# who each engine runs as, and which model answered each call, reach the log and run-stats" + local root fd; root=$(setup_env); mk_session "$root" sess1; mk_session "$root" sess2 + run_dream "$root" + fd=$(fdir "$root") + assert_grep "$root/run.out" 'login for the claude engine: provider=firstParty auth=claude.ai plan=max account=test@example.com' "the log names the provider, sign-in, plan and account" + assert_grep "$fd/run-stats.txt" '^login_claude: provider=firstParty auth=claude.ai plan=max account=test@example.com$' "and so does run-stats" + assert_grep "$fd/run-stats.txt" '^l1_served_models: claude-haiku-5-5=2$' "each call is counted under the model that answered it" + assert_grep "$fd/run-stats.txt" '^l1_served_mismatch: 0$' "none was answered by another model" + assert_grep "$fd/run-stats.txt" '^l1_cost_usd_estimate: 0.0250$' "and the engine's cost estimates are summed" + rm -rf "$root" +} + +test_an_engine_failing_its_login_requirement_is_handed_nothing(){ + echo "# AUTODREAM_REQUIRE_LOGIN: a summary engine whose login does not match stops the run before it starts" + local root fd h o; root=$(setup_env); mk_session "$root" sess1 + AUTODREAM_REQUIRE_LOGIN="provider=firstParty plan=pro" MOCK_CALL_LOG="$root/calls.log" run_dream "$root" + assert_no_file "$root/calls.log" "no worker started" + assert_grep "$root/run.out" 'the claude engine, which writes the summary, fails AUTODREAM_REQUIRE_LOGIN (plan is max, not pro). Refusing to start.' "the log says which requirement failed" + assert_nogrep "$root/run.exit" '^0$' "and the run fails" + rm -rf "$root" + echo "# an L1 engine whose login does not match is handed no transcript, while the others run" + root=$(setup_env); mk_session "$root" sess1; h=$(hash_of "$root/projects/proj-a/sess1.jsonl") + o=$(mk_omp_session "$root" uuuu) + AUTODREAM_REQUIRE_LOGIN="provider=firstParty" run_dream_omp "$root" + fd=$(fdir "$root") + assert_grep "$root/run.out" 'login for the omp engine: provider=unknown auth=unknown plan=unknown account=unknown' "omp reports no login, so every field is unknown" + assert_grep "$root/run.out" 'the omp engine fails AUTODREAM_REQUIRE_LOGIN (provider is unknown, not firstParty)' "and fails the requirement" + assert_grep "$fd/$(hash_of "$o").json" 'no L1 engine for this session' "its session gets the deterministic error record" + assert_grep "$fd/$h.json" 'always run tests after edits' "while the claude session is triaged" + assert_grep "$fd/run-stats.txt" '^login_omp: provider=unknown auth=unknown plan=unknown account=unknown$' "run-stats records both logins" + rm -rf "$root" + echo "# a met requirement triages as usual, and a requirement on an unknown key fails closed" + root=$(setup_env); mk_session "$root" sess1; h=$(hash_of "$root/projects/proj-a/sess1.jsonl") + AUTODREAM_REQUIRE_LOGIN="provider=firstParty auth=claude.ai plan=max account=test@example.com" run_dream "$root" + assert_grep "$(fdir "$root")/$h.json" 'always run tests after edits' "a matching login triages the session" + rm -rf "$root" + root=$(setup_env); mk_session "$root" sess1 + AUTODREAM_REQUIRE_LOGIN="region=us" MOCK_CALL_LOG="$root/calls.log" run_dream "$root" + assert_no_file "$root/calls.log" "an unknown key starts no worker" + assert_grep "$root/run.out" 'region=us is not a provider, auth, plan or account requirement' "and says why" + rm -rf "$root" +} + +test_a_call_answered_by_another_model_loses_its_findings(){ + echo "# AUTODREAM_L1_REQUIRE_SERVED=1: findings from a call another model answered never reach L2" + local root fd h; root=$(setup_env); mk_session "$root" sess1 + h=$(hash_of "$root/projects/proj-a/sess1.jsonl") + AUTODREAM_L1_REQUIRE_SERVED=1 MOCK_SERVED_MODEL=claude-other-model run_dream "$root" + fd=$(fdir "$root") + assert_grep "$fd/l1-served.tsv" "^$h claude claude-haiku-5-5 claude-other-model " "the call is recorded under the model that answered it" + assert_grep "$fd/run-stats.txt" '^l1_served_mismatch: [1-9]' "and counted as a mismatch" + assert_nogrep "$fd/$h.json" 'always run tests after edits' "the substituted model's findings are gone" + if grep -qs 'answered by \[claude-other-model\], not the requested model \[claude-haiku-5-5\]' "$fd/$h.json" "$fd/$h.json.err"; then + ok "the failure names both models" + else no "the failure names both models"; fi + rm -rf "$root" + echo "# without the setting a mismatch is recorded and the findings are kept" + root=$(setup_env); mk_session "$root" sess1; h=$(hash_of "$root/projects/proj-a/sess1.jsonl") + MOCK_SERVED_MODEL=claude-other-model run_dream "$root" + fd=$(fdir "$root") + assert_grep "$fd/$h.json" 'always run tests after edits' "the findings are kept" + assert_grep "$fd/run-stats.txt" '^l1_served_mismatch: 1$' "and the mismatch is counted" + rm -rf "$root" +} + test_omp_stats_describe_the_live_branch_only(){ echo "# omp stats are computed on the live branch, so abandoned turns cannot lift a session past the noise gate" local root; root=$(setup_env); mk_session "$root" sess1 @@ -5534,6 +5600,9 @@ _run test_omp_sessions_run_on_another_adapters_engine _run test_the_runs_reviewer_reviews_every_source_by_default _run test_an_unusable_l1_engine_override_fails_closed _run test_omp_session_roots_come_from_the_host_setting +_run test_l1_provenance_and_served_models_are_recorded +_run test_an_engine_failing_its_login_requirement_is_handed_nothing +_run test_a_call_answered_by_another_model_loses_its_findings _run test_omp_nested_child_is_a_sidechain_of_its_bucket _run test_omp_session_that_cannot_be_linearized_is_an_error_record _run test_omp_stats_describe_the_live_branch_only From 5ee2c64040e109579cf6a38702cc050e720d072d Mon Sep 17 00:00:00 2001 From: Daniel Young Date: Thu, 29 Oct 2026 11:07:47 -0400 Subject: [PATCH 04/11] fix: key the L1 model pin by the engine that runs it, not the source AUTODREAM_L1_MODEL_ was still read for the source after the engine choice was split from it, but a model id belongs to an engine. With claude as the reviewer and AUTODREAM_L1_MODEL_OMP set to a deepseek model, omp sessions went to the claude CLI with --model deepseek/...; going the other way, an omp pin never reached claude sessions routed to the omp engine. adapter_l1_model now resolves the engine first and reads AUTODREAM_L1_MODEL_, then AUTODREAM_L1_MODEL, then that engine's manifest default, the same keying AUTODREAM_L2_MODEL_ already has. The engine choice stays per source. Tests: tests/adapters.sh asserts a source pin no longer reaches another engine and an engine pin reaches every source it reads; run-all checks the worker argv for the review repro in both directions. --- CLAUDE.md | 2 +- README.md | 12 ++++++++---- bin/adapters.sh | 28 ++++++++++++++-------------- bin/run.sh | 5 ++++- tests/adapters.sh | 6 ++++-- tests/run-all.sh | 19 +++++++++++++++---- 6 files changed, 46 insertions(+), 26 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index edd6acd..07f5a66 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -108,7 +108,7 @@ plus env `CLAUDE_CODE_DISABLE_CLAUDE_MDS=1 DISABLE_TELEMETRY=1 DISABLE_ERROR_REP - **The source map is held in the environment, not read from the findings dir.** `AUTODREAM_SOURCE_MAP` (hash and adapter per session) and `AUTODREAM_L1_MODELS` (model per adapter) are built before the first model call. An L1 worker holds the Write tool and could rewrite `sessions-source.txt`, and the source decides which binary the next worker executes. `test_l1_engine_cannot_be_redirected_by_a_worker` has a worker do exactly that and checks the next session still runs on the real engine; reading the file instead fails it. - **Haiku 5.5, effort and a replaced system prompt (2026-10-07).** The claude manifest's `l1_model` is `claude-haiku-5-5` (verified: `--model claude-haiku-5-5` and the `haiku` alias both answer as it; an id the CLI does not know falls back silently, so re-check `modelUsage` in `--output-format json` when changing it). `l1-argv` and `warmup-argv` add `--effort` from `AUTODREAM_L1_EFFORT_CLAUDE`, then `AUTODREAM_L1_EFFORT`, then `high`; `off` omits it and an unknown level exits 2. They also pass `--system-prompt` instead of `--append-system-prompt`, which drops Claude Code's default prompt: one-word call 6.9k -> 5.0k input tokens, and a real triage run on a 250 KB slice cost $0.159 against $0.215 with the same number of findings. L2 is untouched. -- **The model** resolves `AUTODREAM_L1_MODEL_`, then `AUTODREAM_L1_MODEL`, then the manifest's `l1_model` (`adapter_l1_model`). A model id belongs to one engine, which is why a host with two engines pins them per adapter. +- **The engine, then the model.** A source's L1 engine is `AUTODREAM_L1_ENGINE_`, then `AUTODREAM_L1_ENGINE`, then the run's reviewer (the L2 engine) (`adapter_l1_engine`). The model is keyed by that engine, not by the source: `AUTODREAM_L1_MODEL_`, then `AUTODREAM_L1_MODEL`, then the engine's manifest `l1_model` (`adapter_l1_model`). A model id belongs to one engine, so a source-keyed pin sent omp sessions to the claude CLI with `--model deepseek/...` once a claude reviewer read them (PR #165 review). The engine choice is per source and the model pin per engine, the same as `AUTODREAM_L2_MODEL_`. - **No engine is a deterministic error.** A session whose adapter is not executable, or resolves no model, gets a findings record with `error: no L1 engine for this session`, left in place and counted by `l1_findings_with_error`. Retrying cannot change the answer, and the run still reports. ## Self-pollution (the eating-its-own-tail bug) diff --git a/README.md b/README.md index a9e6e51..3bc6be6 100644 --- a/README.md +++ b/README.md @@ -78,9 +78,11 @@ cd ~/git/autodream ``` `--adapters` lists the harnesses to read, and `--l2-engine` names the one whose CLI runs the -aggregation pass. Pin a model per harness in `config` with `AUTODREAM_L1_MODEL_` and -`AUTODREAM_L2_MODEL_` (for example `AUTODREAM_L1_MODEL_OMP`). A host with only one harness -installed needs none of this. +aggregation pass. That engine also reviews every harness's sessions in L1 unless `config` says +otherwise: `AUTODREAM_L1_ENGINE_=` keeps one harness on its own engine (for example +`AUTODREAM_L1_ENGINE_CLAUDE=claude`). Pin a model per engine with `AUTODREAM_L1_MODEL_` and +`AUTODREAM_L2_MODEL_` (for example `AUTODREAM_L1_MODEL_OMP`, the model of every L1 call the +omp engine makes). A host with only one harness installed needs none of this. This symlinks `bin/*.sh`, `prompts/*.md` and the `adapters/` tree into `~/.claude/autodream/`, creates `~/.claude/dreams/`, and on macOS installs and bootstraps the nightly launchd @@ -223,7 +225,9 @@ Commands, fallbacks and how to test it are in [`mods/autodream-band/README.md`]( the changelog window, the review popup dedup) and the OMP session reader is `adapters/omp/`. To move an existing OMP install: unload its launchd jobs, run `./install.sh --adapters claude,omp --l2-engine omp`, and carry its L1 model over as -`AUTODREAM_L1_MODEL_OMP`. Do not copy its `SESSION_ROOTS` line: the omp adapter finds +`AUTODREAM_L1_MODEL_OMP`. With omp as the L2 engine it reviews the Claude Code sessions too, on +that model; add `AUTODREAM_L1_ENGINE_CLAUDE=claude` to keep them on Claude Code's own engine and +model. Do not copy its `SESSION_ROOTS` line: the omp adapter finds `~/.omp/agent/sessions` itself, and a stray root would make the Claude adapter parse OMP files. The full runbook, with rollback, is in `docs/plans/2026-10-03-omp-adapter.md`. diff --git a/bin/adapters.sh b/bin/adapters.sh index 2bbbd32..7ce3e58 100755 --- a/bin/adapters.sh +++ b/bin/adapters.sh @@ -219,24 +219,24 @@ adapter_l1_engine() { # $1=source $2=the run's reviewer -> engine adapter on std printf '%s' "$v" } -# The model one adapter's L1 workers run, resolved in a fixed order so a host can pin one -# harness without touching another: -# AUTODREAM_L1_MODEL_ that adapter only (the name upper-cased, dashes to underscores) -# AUTODREAM_L1_MODEL every adapter -# the manifest's l1_model the default of the adapter whose engine runs it (adapter_l1_engine) -# A model id belongs to one engine (neuralwatt/glm-5.3-flash means nothing to claude), which -# is why the per-adapter form exists: the generic one is an override for hosts with one engine. +# The model one source's L1 calls run. A model id belongs to one engine (neuralwatt/glm-5.3-flash +# means nothing to claude), so the engine is resolved first (adapter_l1_engine) and the model is +# keyed by it, never by the source, in a fixed order so a host can pin one engine without touching +# another: +# AUTODREAM_L1_MODEL_ that engine only, whichever source's sessions it reads (the name +# upper-cased, dashes to underscores), like AUTODREAM_L2_MODEL_ +# AUTODREAM_L1_MODEL every engine: an override for hosts with one engine +# the manifest's l1_model that engine's default +# With the reviewer on claude, AUTODREAM_L1_MODEL_OMP therefore does not reach omp sessions, and +# an omp model id never reaches the claude CLI. adapter_l1_model() { # $1=source $2=the run's reviewer -> model on stdout, exit 1 when none resolves - local name="$1" up var v engine - _adapter_name_safe "$name" || return 1 - up=$(printf '%s' "$name" | tr 'a-z-' 'A-Z_') + local engine up var v + engine=$(adapter_l1_engine "$1" "${2:-}") || return 1 + up=$(printf '%s' "$engine" | tr 'a-z-' 'A-Z_') var="AUTODREAM_L1_MODEL_$up" v="${!var:-}" [ -n "$v" ] || v="${AUTODREAM_L1_MODEL:-}" - if [ -z "$v" ]; then - engine=$(adapter_l1_engine "$name" "${2:-}") || return 1 - v=$(adapter_manifest_get "$engine" '.l1_model' 2>/dev/null) || v="" - fi + [ -n "$v" ] || v=$(adapter_manifest_get "$engine" '.l1_model' 2>/dev/null) || v="" [ -n "$v" ] || return 1 printf '%s' "$v" } diff --git a/bin/run.sh b/bin/run.sh index bdabab2..e50cef1 100755 --- a/bin/run.sh +++ b/bin/run.sh @@ -78,7 +78,10 @@ # One extra read-only call on the L2 engine and model, after everything # else (bin/triage-dream.sh; never fatal). Off, nothing changes. # AUTODREAM_L2_MODEL pin the L2 aggregator model (every engine) default: the adapter's own (claude: the CLI default) -# AUTODREAM_L2_MODEL_ / AUTODREAM_L1_MODEL_ the same for one adapter only +# AUTODREAM_L2_MODEL_ / AUTODREAM_L1_MODEL_ the same for one engine only. Keyed by +# the engine that makes the call, not the source: AUTODREAM_L1_MODEL_OMP is +# the model of every L1 call on the omp engine, whichever harness holds the +# session, and never reaches the claude CLI # AUTODREAM_L1_ENGINE accepted adapter whose engine reviews every source's sessions in L1 # default: the L2 engine # AUTODREAM_L1_ENGINE_ the same for one source only, e.g. AUTODREAM_L1_ENGINE_OMP=omp diff --git a/tests/adapters.sh b/tests/adapters.sh index e87a315..8ebea45 100644 --- a/tests/adapters.sh +++ b/tests/adapters.sh @@ -158,7 +158,7 @@ rm -f "$(_adapter_reject_broken_marker)" # shellcheck disable=SC2086 rm -rf $TMPROOTS -echo "# adapter_l1_model: per-adapter override, then the generic one, then the manifest default" +echo "# adapter_l1_model: per-engine override, then the generic one, then the engine's manifest default" l1counts=$(mktemp "${TMPDIR:-/tmp}/l1model.XXXXXX") ( export ADAPTERS_ROOT="$REPO/adapters" @@ -185,7 +185,9 @@ l1counts=$(mktemp "${TMPDIR:-/tmp}/l1model.XXXXXX") AUTODREAM_L1_ENGINE_OMP=claude assert_eq "$(adapter_l1_engine omp)" "claude" "the per-source engine choice moves it" assert_eq "$(adapter_l1_model omp)" "claude-haiku-5-5" "and its default model is the engine's, not its own" - assert_eq "$(AUTODREAM_L1_MODEL_OMP=claude-x adapter_l1_model omp)" "claude-x" "an explicit model still wins" + assert_eq "$(AUTODREAM_L1_MODEL_CLAUDE=claude-x adapter_l1_model omp)" "claude-x" "a pin on the engine reaches the sessions it reads" + assert_eq "$(AUTODREAM_L1_MODEL_OMP=deepseek/deepseek-flash adapter_l1_model omp)" "claude-haiku-5-5" "a pin on the source does not: its model id belongs to another engine" + assert_eq "$(AUTODREAM_L1_MODEL_CLAUDE=claude-x AUTODREAM_L1_MODEL_OMP=omp/x adapter_l1_model claude omp)" "omp/x" "the other way round: claude sessions the omp engine reads take the omp pin" AUTODREAM_L1_ENGINE_OMP=nonesuch adapter_l1_engine omp >/dev/null 2>&1; assert_eq "$?" "1" "an override naming no accepted adapter resolves nothing" adapter_l1_model omp >/dev/null 2>&1; assert_eq "$?" "1" "and leaves no default model to fall back on" diff --git a/tests/run-all.sh b/tests/run-all.sh index 54f2a05..790d922 100755 --- a/tests/run-all.sh +++ b/tests/run-all.sh @@ -4267,13 +4267,15 @@ test_omp_sessions_run_on_another_adapters_engine(){ echo "# AUTODREAM_L1_ENGINE_OMP=claude: the claude engine reads omp sessions, and omp never runs" local root o h fd trip; root=$(setup_env); mk_session "$root" sess1 o=$(mk_omp_session "$root" ffff); h=$(hash_of "$o"); trip=$(mk_omp_tripwire "$root") - export FANOUT=1 MOCK_CAPTURE_DIR="$root/cap" AUTODREAM_L1_ENGINE_OMP=claude - TEST_OMP_BIN="$trip" TEST_OMP_L1_MODEL=claude-test-model run_dream_omp "$root" - unset FANOUT MOCK_CAPTURE_DIR AUTODREAM_L1_ENGINE_OMP + # An omp pin is set too, and must not follow the omp sessions onto the claude CLI. + export FANOUT=1 MOCK_CAPTURE_DIR="$root/cap" AUTODREAM_L1_ENGINE_OMP=claude AUTODREAM_L1_MODEL_CLAUDE=claude-test-model + TEST_OMP_BIN="$trip" TEST_OMP_L1_MODEL=deepseek/deepseek-flash run_dream_omp "$root" + unset FANOUT MOCK_CAPTURE_DIR AUTODREAM_L1_ENGINE_OMP AUTODREAM_L1_MODEL_CLAUDE fd=$(fdir "$root") assert_grep "$fd/sessions-source.txt" "^$h omp$" "the session is still the omp adapter's" assert_grep "$root/cap/l1-read-$h.txt" 'LIVE_BRANCH_MARKER' "the omp adapter still linearized it" - assert_grep "$root/cap/l1-args-$h.txt" '^claude-test-model$' "the worker ran the configured model" + assert_grep "$root/cap/l1-args-$h.txt" '^claude-test-model$' "the worker ran the claude engine's pinned model" + assert_nogrep "$root/cap/l1-args-$h.txt" 'deepseek' "not the omp pin: a model id belongs to the engine, not the source" assert_grep "$root/cap/l1-args-$h.txt" '^--no-session-persistence$' "with the claude engine's flags" assert_nogrep "$root/cap/l1-args-$h.txt" '^--allow-home$' "and not the omp engine's" assert_nogrep "$fd/$h.json" '"error"' "the session was triaged" @@ -4281,6 +4283,15 @@ test_omp_sessions_run_on_another_adapters_engine(){ assert_grep "$root/run.out" 'L1 model for omp: claude-test-model (on the claude engine)' "and so does the log" assert_no_file "$root/omp-calls.log" "the omp binary was never called" rm -rf "$root" + echo "# the other way round: claude sessions on the omp engine take the omp pin, never a claude one" + local c hc; root=$(setup_env); mk_session "$root" sess1; c="$root/projects/proj-a/sess1.jsonl"; hc=$(hash_of "$c") + export FANOUT=1 MOCK_CAPTURE_DIR="$root/cap" AUTODREAM_L1_ENGINE_CLAUDE=omp AUTODREAM_L1_MODEL_CLAUDE=claude-test-model + TEST_OMP_L1_MODEL=omp/pinned run_dream_omp "$root" + unset FANOUT MOCK_CAPTURE_DIR AUTODREAM_L1_ENGINE_CLAUDE AUTODREAM_L1_MODEL_CLAUDE + assert_grep "$root/cap/l1-args-$hc.txt" '^omp/pinned$' "the omp engine ran its own pin for the claude session" + assert_nogrep "$root/cap/l1-args-$hc.txt" 'claude-test-model' "and the claude pin stayed with the claude engine" + assert_grep "$root/run.out" 'L1 model for claude: omp/pinned (on the omp engine)' "the log says so" + rm -rf "$root" } test_the_runs_reviewer_reviews_every_source_by_default(){ From 6d0f36631cebef5c94ec9eaddbe2cedd0de108fb Mon Sep 17 00:00:00 2001 From: Daniel Young Date: Thu, 29 Oct 2026 11:12:21 -0400 Subject: [PATCH 05/11] fix: put the L1 engine in the chunk answer cache key A chunk answer is reused only for the same chunk read under the same instructions, and the digest in its cache name said so for the source and the model but left out the engine. Now that the engine is a choice of its own, one model id can be served by two engines with different system prompts and tools, so a chunk answered on one engine was reused after the session moved to another. Tests: run-all leaves a chunk answer behind on the claude engine, moves the session to the omp engine under the same model id, and checks the chunk is called again rather than reused. --- bin/run.sh | 7 ++++--- tests/run-all.sh | 23 +++++++++++++++++++++++ 2 files changed, 27 insertions(+), 3 deletions(-) diff --git a/bin/run.sh b/bin/run.sh index e50cef1..3c43025 100755 --- a/bin/run.sh +++ b/bin/run.sh @@ -2766,11 +2766,12 @@ dispatch_l1() { # one parallel pass; idempotent worker → only the still-missin # triage prompt, the harness addendum, the engine and model, the chunk count and omitted count the # chunk note states, and the stats block the worker # copies from (merge-chunks.sh takes the stats fields from chunk 1). All of it is in the - # cache name, so a retry after the prompt, the model or the session changed redoes the chunk - # instead of merging a stale answer with fresh ones. + # cache name, so a retry after the prompt, the engine, the model or the session changed redoes + # the chunk instead of merging a stale answer with fresh ones. The engine is there in its own + # right: one model id can be served by two engines, through different system prompts and tools. ccfg=$( { cat "$AUTODREAM_DIR/SESSION_TRIAGE.md" 2>/dev/null if [ -n "$src" ] && [ -r "$ADAPTERS_DIR/$src/triage.md" ]; then cat "$ADAPTERS_DIR/$src/triage.md"; fi - printf "%s %s %s %s\n" "$src" "$model" "$nchunks" "$elided" + printf "%s %s %s %s %s\n" "$src" "$engine" "$model" "$nchunks" "$elided" cat "$FINDINGS_DIR/$hash.stats.json" 2>/dev/null; } | shasum -a 1 2>/dev/null | cut -c1-8 ) while [ "$ci" -le "$nchunks" ]; do cin=$(printf "%s/in/chunk-%02d.jsonl" "$chunkroot" "$ci") diff --git a/tests/run-all.sh b/tests/run-all.sh index 790d922..207b0dc 100755 --- a/tests/run-all.sh +++ b/tests/run-all.sh @@ -6370,6 +6370,28 @@ test_a_cached_chunk_answer_is_not_reused_after_the_instructions_or_the_session_c done } +test_a_cached_chunk_answer_is_not_reused_on_another_engine(){ + echo "# chunking: a finished chunk answer is reused only on the engine that wrote it, even under the same model id" + local root; root=$(setup_env) + local f; f=$(mk_chunky "$root" big 60); local h; h=$(hash_of "$f"); local fd; fd=$(fdir "$root") + mkdir -p "$root/home" + # Run 1, on the claude engine: a permanent refusal on chunk 2 defers the date and leaves chunk 1. + export MOCK_MODE=chunked MOCK_CALL_LOG="$root/calls1.log" MOCK_FAIL_CHUNK=2 MOCK_FAIL_KIND=refusal AUTODREAM_L1_ROUNDS=1 AUTODREAM_L1_MODEL=shared/model + run_chunked "$root"; unset MOCK_MODE MOCK_CALL_LOG MOCK_FAIL_CHUNK MOCK_FAIL_KIND AUTODREAM_L1_ROUNDS + assert_eq "$(ls "$fd/.chunks/$h"/01-*.chunkout 2>/dev/null | wc -l | tr -d ' ')" "1" "run 1 on the claude engine left chunk 1's answer behind" + # Run 2: the same model id, the same prompt and stats, but the session moves to the omp engine. + # A subshell, so the sandbox HOME (the omp adapter enumerates ~/.omp) cannot outlive the run. + ( export MOCK_MODE=chunked MOCK_CALL_LOG="$root/calls2.log" AUTODREAM_L1_ROUNDS=1 HOME="$root/home" \ + AUTODREAM_ADAPTERS=claude,omp OMP_BIN="$MOCK" AUTODREAM_L1_ENGINE_CLAUDE=omp + run_chunked "$root" ) + unset AUTODREAM_L1_MODEL + assert_grep "$fd/run-stats.txt" '^l1_engine_claude: omp$' "run 2 put the session on the omp engine" + assert_eq "$(grep -c '^01-' <(sed 's#.*/##' "$root/calls2.log"))" "1" "chunk 1 was called again on the new engine, not served from the old answer" + assert_nogrep "$root/run.out" "reuse: chunk 1/" "and the log does not say it reused it" + assert_eq "$(jq -r 'has("error")' "$fd/$h.json" 2>/dev/null)" "false" "the session finished" + rm -rf "$root" +} + test_a_bad_chunk_answer_is_never_merged_around(){ echo "# chunking: an error, partial or malformed chunk answer blocks the merge and is redone alone" local kind @@ -6554,6 +6576,7 @@ _run test_a_failed_chunker_never_hands_a_worker_the_whole_slim _run test_a_failed_chunker_on_a_transcript_that_was_never_slimmed_is_still_bounded _run test_a_chunk_that_fails_is_retried_alone_and_finished_chunks_are_reused _run test_a_cached_chunk_answer_is_not_reused_after_the_instructions_or_the_session_change +_run test_a_cached_chunk_answer_is_not_reused_on_another_engine _run test_a_bad_chunk_answer_is_never_merged_around _run test_a_chunk_failure_is_classified_like_any_worker_failure _run test_a_provider_refusal_on_a_chunk_defers_the_date_and_stops_the_other_chunks From 1f6f503c37eb42bd60329fafb1c019f37bd714af Mon Sep 17 00:00:00 2001 From: Daniel Young Date: Thu, 29 Oct 2026 11:16:37 -0400 Subject: [PATCH 06/11] fix: let the engine's adapter judge whether the requested model answered The served check compared the requested string to the id the CLI reports. The CLI reports canonical ids, so a host that asked for haiku saw every call answered by claude-haiku-5-5 counted as a mismatch, and AUTODREAM_L1_REQUIRE_SERVED=1 discarded all of them. The served verb now takes the requested model and returns a verdict, match or mismatch, beside the models and the cost. Only the adapter knows its CLI's naming: for claude a bare family name matches any id of that family (claude--...), a full id only itself, and a context suffix such as [1m] is ignored. The verdict is a sixth column in l1-served.tsv, and both the REQUIRE_SERVED check and l1_served_mismatch read it, so the two cannot disagree. An engine with no served verb (omp) gets the verdict unmeasured. It was counted as a mismatch before; it is now counted on its own line, l1_served_unmeasured, and REQUIRE_SERVED=1 still discards its calls, with a message saying the engine reported no model. Tests: tests/adapter-claude.sh covers aliases, full ids, prefixes, suffixes and unreadable aliases. run-all checks that a haiku request answered as claude-haiku-5-5 keeps its findings under REQUIRE_SERVED, and that an omp call is unmeasured and discarded under it. --- adapters/claude/adapter.sh | 19 ++++++++++++++++--- bin/run.sh | 36 ++++++++++++++++++++++++++---------- tests/adapter-claude.sh | 17 ++++++++++++++--- tests/run-all.sh | 25 +++++++++++++++++++++++++ 4 files changed, 81 insertions(+), 16 deletions(-) diff --git a/adapters/claude/adapter.sh b/adapters/claude/adapter.sh index 566ab80..c342f37 100755 --- a/adapters/claude/adapter.sh +++ b/adapters/claude/adapter.sh @@ -211,13 +211,26 @@ case "$cmd" in --system-prompt 'Reply with the single word ok and exit.' ;; - served) # $1=stdout of one L1 call -> "[,...]", exit 1 without a result + served) # $1=stdout of one L1 call $2=the model it asked for + # -> "[,...]", exit 1 without a result # The models come from the result object's modelUsage, which lists every model that answered # any part of the call. The cost is the CLI's estimate at API prices, not what a subscription # bills; it is kept because it compares models on real numbers. + # The verdict is match when every model that answered is the one asked for, judged here because + # only this adapter knows the CLI's naming: modelUsage carries canonical ids (claude-haiku-5-5) + # while a host may ask by alias (haiku). A bare family name matches any id of that family + # (claude--...), a full id only itself, and a context suffix such as [1m] is ignored on + # both sides. An alias the family rule cannot read (default, opusplan) is a mismatch: the + # runner then cannot vouch for the model, which is what AUTODREAM_L1_REQUIRE_SERVED asks. [ "$#" -ge 1 ] && [ -r "$1" ] || exit 1 - line=$(jq -r 'select(type == "object" and .type == "result") - | [((.modelUsage // {}) | keys | sort | join(",")), ((.total_cost_usd // "") | tostring)] | @tsv' "$1" 2>/dev/null | head -1) + line=$(jq -r --arg want "${2:-}" 'select(type == "object" and .type == "result") + | ((.modelUsage // {}) | keys | sort) as $m + | ($want | sub("\\[[^\\]]*\\]$"; "")) as $w + | [($m | join(",")), ((.total_cost_usd // "") | tostring), + (if ($m | length) > 0 and all($m[]; sub("\\[[^\\]]*\\]$"; "") as $s + | $s == $w or (($w | test("^[a-z]+$")) and ($s | startswith("claude-" + $w + "-")))) + then "match" else "mismatch" end)] + | @tsv' "$1" 2>/dev/null | head -1) case "$line" in ""|$'\t'*) exit 1 ;; esac printf '%s\n' "$line" ;; diff --git a/bin/run.sh b/bin/run.sh index 3c43025..4a78e18 100755 --- a/bin/run.sh +++ b/bin/run.sh @@ -99,7 +99,10 @@ # stops the run before it starts default: record only # (e.g. "provider=firstParty auth=claude.ai plan=max") # AUTODREAM_L1_REQUIRE_SERVED 1 discards the findings of any L1 call answered by a model other -# than the requested one (the engine's served verb) default: 0, record only +# than the requested one, as the engine's served verb judges it (claude: an +# alias such as haiku matches the claude-haiku-... id it answers as). An +# engine with no served verb (omp) cannot be vouched for, so its calls are +# discarded too default: 0, record only # AUTODREAM_MARKER_EPOCH first date whose report is REQUIRED to carry the # open-questions marker; earlier unmarked reports are treated # as complete (legacy) rather than abandoned @@ -2651,15 +2654,24 @@ dispatch_l1() { # one parallel pass; idempotent worker → only the still-missin fi # Which model answered, from the engine result record (adapter verb served), for every call - # whatever it produced, so a night accounts for each one. Engines that report nothing are - # recorded as unknown. With AUTODREAM_L1_REQUIRE_SERVED=1 a call answered by anything other - # than the requested model loses its findings, so nothing a substituted model wrote reaches L2. - l1served=$("$ADAPTERS_DIR/$engine/adapter.sh" served "$4" 2>/dev/null) || l1served="" + # whatever it produced, so a night accounts for each one. Whether that is the requested model + # is the adapter verdict, not a string compare here: the CLI answers with a canonical id + # (claude-haiku-5-5) for a model asked for by alias (haiku), and only the adapter knows its + # naming. An engine that reports nothing is recorded as unknown and unmeasured, never as a + # match or a mismatch. With AUTODREAM_L1_REQUIRE_SERVED=1 a call whose verdict is not match + # loses its findings, so nothing a substituted or unverifiable model wrote reaches L2. + l1served=$("$ADAPTERS_DIR/$engine/adapter.sh" served "$4" "$model" 2>/dev/null) || l1served="" l1smodels=$(printf "%s" "$l1served" | cut -f1) l1scost=$(printf "%s" "$l1served" | cut -s -f2) - printf "%s\t%s\t%s\t%s\t%s\n" "$hash" "$engine" "$model" "${l1smodels:-unknown}" "$l1scost" >> "$FINDINGS_DIR/l1-served.tsv" - if [ "${AUTODREAM_L1_REQUIRE_SERVED:-0}" = 1 ] && [ -s "$2" ] && [ "$l1smodels" != "$model" ]; then - printf "the call was answered by [%s], not the requested model [%s]; its findings are discarded (AUTODREAM_L1_REQUIRE_SERVED=1)\n" "${l1smodels:-unknown}" "$model" >> "$3" + l1sverdict=$(printf "%s" "$l1served" | cut -s -f3) + case "$l1sverdict" in match|mismatch) ;; *) l1sverdict=unmeasured ;; esac + printf "%s\t%s\t%s\t%s\t%s\t%s\n" "$hash" "$engine" "$model" "${l1smodels:-unknown}" "$l1scost" "$l1sverdict" >> "$FINDINGS_DIR/l1-served.tsv" + if [ "${AUTODREAM_L1_REQUIRE_SERVED:-0}" = 1 ] && [ -s "$2" ] && [ "$l1sverdict" != match ]; then + if [ "$l1sverdict" = mismatch ]; then + printf "the call was answered by [%s], not the requested model [%s]; its findings are discarded (AUTODREAM_L1_REQUIRE_SERVED=1)\n" "$l1smodels" "$model" >> "$3" + else + printf "the %s engine did not report which model answered, so the call cannot be held to the requested model [%s]; its findings are discarded (AUTODREAM_L1_REQUIRE_SERVED=1)\n" "$engine" "$model" >> "$3" + fi rm -f "$2" return 1 fi @@ -3964,12 +3976,16 @@ PY [ -n "$_msrc" ] || continue printf 'login_%s: provider=%s auth=%s plan=%s account=%s\n' "$(printf '%s' "$_msrc" | tr '-' '_')" "$_pprov" "$_pauth" "$_pplan" "$_pacct" done <<< "${ENGINE_LOGINS:-}" + # Mismatch is the adapter verdict (column 6), the same one AUTODREAM_L1_REQUIRE_SERVED acts on. + # A call the engine reported no model for (omp has no served verb), or a row written before the + # verdict existed, is unmeasured: counted on its own line, never as a match or a mismatch. if [ -s "$FINDINGS_DIR/l1-served.tsv" ]; then printf 'l1_served_models: %s\n' "$(cut -f4 "$FINDINGS_DIR/l1-served.tsv" | sort | uniq -c | awk '{ printf "%s%s=%s", (NR > 1 ? " " : ""), $2, $1 }')" - printf 'l1_served_mismatch: %s\n' "$(awk -F'\t' '$4 != $3' "$FINDINGS_DIR/l1-served.tsv" | wc -l | tr -d ' ')" + printf 'l1_served_mismatch: %s\n' "$(awk -F'\t' '$6 == "mismatch"' "$FINDINGS_DIR/l1-served.tsv" | wc -l | tr -d ' ')" + printf 'l1_served_unmeasured: %s\n' "$(awk -F'\t' '$6 != "match" && $6 != "mismatch"' "$FINDINGS_DIR/l1-served.tsv" | wc -l | tr -d ' ')" printf 'l1_cost_usd_estimate: %s\n' "$(awk -F'\t' '$5 != "" { s += $5 } END { printf "%.4f", s }' "$FINDINGS_DIR/l1-served.tsv")" else - printf 'l1_served_models: none\nl1_served_mismatch: 0\nl1_cost_usd_estimate: 0.0000\n' + printf 'l1_served_models: none\nl1_served_mismatch: 0\nl1_served_unmeasured: 0\nl1_cost_usd_estimate: 0.0000\n' fi # yes when the circuit breaker cut the retry budget, so a short l1_rounds_used is not # mistaken for a run that finished early and cleanly. diff --git a/tests/adapter-claude.sh b/tests/adapter-claude.sh index b0b566b..157d40b 100644 --- a/tests/adapter-claude.sh +++ b/tests/adapter-claude.sh @@ -282,16 +282,27 @@ assert_eq "$(printf '%s\n' "$WA" | sed -n '1,/^--tools$/p' | sed '$d')" "$(print echo "# served: the models that answered one L1 call, from its result object" printf '%s\n' '{"type":"result","is_error":false,"result":"done","modelUsage":{"claude-sonnet-5-5":{}},"total_cost_usd":0.25}' > "$tmp/r1" -assert_eq "$("$A" served "$tmp/r1")" "claude-sonnet-5-5 0.25" "one model and the cost estimate" +assert_eq "$("$A" served "$tmp/r1" claude-sonnet-5-5)" "claude-sonnet-5-5 0.25 match" "one model, the cost estimate, and a match for the model asked for" printf '%s\n' '{"type":"result","modelUsage":{"m-b":{},"m-a":{}},"total_cost_usd":1}' > "$tmp/r2" -assert_eq "$("$A" served "$tmp/r2")" "m-a,m-b 1" "every model that answered, sorted" +assert_eq "$("$A" served "$tmp/r2" m-a)" "m-a,m-b 1 mismatch" "every model that answered, sorted, and a second model is a mismatch" printf '%s\n' '{"type":"system","subtype":"init"}' '{"type":"result","modelUsage":{"m":{}}}' > "$tmp/r3" -assert_eq "$("$A" served "$tmp/r3")" "m " "a stream is read for its result line, and a missing cost is empty" +assert_eq "$("$A" served "$tmp/r3" m)" "m match" "a stream is read for its result line, and a missing cost is empty" printf 'done\n' > "$tmp/r4" "$A" served "$tmp/r4" >/dev/null 2>&1; assert_eq "$?" "1" "plain text carries no record" printf '%s\n' '{"type":"result","is_error":true,"result":"Not logged in","modelUsage":{}}' > "$tmp/r5" "$A" served "$tmp/r5" >/dev/null 2>&1; assert_eq "$?" "1" "a result no model answered carries no record" "$A" served "$tmp/nonesuch" >/dev/null 2>&1; assert_eq "$?" "1" "a missing file carries no record" +echo "# served: the verdict reads the CLI's naming, so an alias matches the canonical id it answers as" +printf '%s\n' '{"type":"result","modelUsage":{"claude-haiku-5-5":{}},"total_cost_usd":0.01}' > "$tmp/r6" +assert_eq "$("$A" served "$tmp/r6" haiku | cut -f3)" "match" "the haiku alias matches claude-haiku-5-5" +assert_eq "$("$A" served "$tmp/r6" claude-haiku-5-5 | cut -f3)" "match" "the full id matches itself" +assert_eq "$("$A" served "$tmp/r6" sonnet | cut -f3)" "mismatch" "another family's alias does not" +assert_eq "$("$A" served "$tmp/r6" claude-haiku-5 | cut -f3)" "mismatch" "nor does a full id that is only a prefix" +assert_eq "$("$A" served "$tmp/r6" default | cut -f3)" "mismatch" "an alias the family rule cannot read is not vouched for" +assert_eq "$("$A" served "$tmp/r6" | cut -f3)" "mismatch" "and nor is a call with no requested model" +printf '%s\n' '{"type":"result","modelUsage":{"claude-opus-5-5[1m]":{}}}' > "$tmp/r7" +assert_eq "$("$A" served "$tmp/r7" 'opus[1m]' | cut -f3)" "match" "a context suffix is ignored on both sides" +assert_eq "$("$A" served "$tmp/r7" claude-opus-5-5 | cut -f3)" "match" "including on the answering side alone" echo "# whoami: who the engine's calls run as, read from claude auth status" printf '%s\n' '#!/bin/sh' '[ "$1 $2 $3" = "auth status --json" ] || exit 9' \ diff --git a/tests/run-all.sh b/tests/run-all.sh index 207b0dc..0bf730a 100755 --- a/tests/run-all.sh +++ b/tests/run-all.sh @@ -4370,8 +4370,33 @@ test_l1_provenance_and_served_models_are_recorded(){ assert_grep "$fd/run-stats.txt" '^login_claude: provider=firstParty auth=claude.ai plan=max account=test@example.com$' "and so does run-stats" assert_grep "$fd/run-stats.txt" '^l1_served_models: claude-haiku-5-5=2$' "each call is counted under the model that answered it" assert_grep "$fd/run-stats.txt" '^l1_served_mismatch: 0$' "none was answered by another model" + assert_grep "$fd/run-stats.txt" '^l1_served_unmeasured: 0$' "and every call reported its model" assert_grep "$fd/run-stats.txt" '^l1_cost_usd_estimate: 0.0250$' "and the engine's cost estimates are summed" rm -rf "$root" + echo "# a model asked for by alias matches the canonical id the CLI answers as (the adapter verdict)" + local h; root=$(setup_env); mk_session "$root" sess1; h=$(hash_of "$root/projects/proj-a/sess1.jsonl") + AUTODREAM_L1_MODEL=haiku MOCK_SERVED_MODEL=claude-haiku-5-5 AUTODREAM_L1_REQUIRE_SERVED=1 run_dream "$root" + fd=$(fdir "$root") + assert_grep "$fd/l1-served.tsv" "^$h claude haiku claude-haiku-5-5 [0-9.]* match$" "haiku answered as claude-haiku-5-5 is recorded as a match" + assert_grep "$fd/run-stats.txt" '^l1_served_mismatch: 0$' "not counted as a mismatch" + assert_grep "$fd/$h.json" 'always run tests after edits' "and AUTODREAM_L1_REQUIRE_SERVED=1 keeps its findings" + rm -rf "$root" + echo "# an engine that reports no model is unmeasured, never a mismatch, and REQUIRE_SERVED cannot vouch for it" + local o; root=$(setup_env); o=$(mk_omp_session "$root" vvvv); h=$(hash_of "$o") + AUTODREAM_ADAPTERS=omp run_dream_omp "$root" + fd=$(fdir "$root") + assert_grep "$fd/l1-served.tsv" "^$h omp omp/test-model unknown unmeasured$" "the omp call is recorded with no model and no verdict" + assert_grep "$fd/run-stats.txt" '^l1_served_mismatch: 0$' "it is not a mismatch" + assert_grep "$fd/run-stats.txt" '^l1_served_unmeasured: 1$' "it is counted as unmeasured" + rm -rf "$root" + root=$(setup_env); o=$(mk_omp_session "$root" wwww); h=$(hash_of "$o") + AUTODREAM_ADAPTERS=omp AUTODREAM_L1_REQUIRE_SERVED=1 run_dream_omp "$root" + fd=$(fdir "$root") + assert_nogrep "$fd/$h.json" 'always run tests after edits' "with AUTODREAM_L1_REQUIRE_SERVED=1 its findings are discarded" + if grep -qs 'the omp engine did not report which model answered' "$fd/$h.json" "$fd/$h.json.err"; then + ok "and the failure says the engine reported no model" + else no "and the failure says the engine reported no model"; fi + rm -rf "$root" } test_an_engine_failing_its_login_requirement_is_handed_nothing(){ From 084dffbb6c875b71cc2fc45625717978b05053f6 Mon Sep 17 00:00:00 2001 From: Daniel Young Date: Thu, 29 Oct 2026 11:28:19 -0400 Subject: [PATCH 07/11] fix: hold cached L1 results to AUTODREAM_L1_REQUIRE_SERVED as well The served check ran only on fresh calls. A findings JSON left by an earlier run is reused by the dispatcher's idempotency read, and a finished chunk answer by the chunk cache, so with the requirement turned on after a night without it, or added on an install whose findings predate it, a substituted model's answer reached L2 anyway. The runner now stores the verdict beside every answer it accepts, as .served (engine, requested model, models that answered, verdict), after the engine has exited. A merged session is match only when every chunk was. Before L1, drop_unacceptable_l1_cache removes any cached findings JSON or chunk answer whose stored verdict is not match, or that has none, so it is triaged again. It runs in the parent before the pre-L1 snapshot, so the dispatcher, l1_missing_count and l1_sessions_already_done_at_start see the same set. Records carrying error or skipped are the runner's own and are kept. run-stats gains l1_cache_dropped. Tests: run-all reruns a date under the requirement and checks that findings another model wrote, and findings with no stored verdict, are triaged again, that matching ones and runner stubs are reused, and that a cached chunk answer is redone or reused by the same rule. --- CLAUDE.md | 1 + bin/run.sh | 72 ++++++++++++++++++++++++++++++++++++++++++++++-- tests/run-all.sh | 63 ++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 133 insertions(+), 3 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index 07f5a66..923435a 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -109,6 +109,7 @@ plus env `CLAUDE_CODE_DISABLE_CLAUDE_MDS=1 DISABLE_TELEMETRY=1 DISABLE_ERROR_REP - **The source map is held in the environment, not read from the findings dir.** `AUTODREAM_SOURCE_MAP` (hash and adapter per session) and `AUTODREAM_L1_MODELS` (model per adapter) are built before the first model call. An L1 worker holds the Write tool and could rewrite `sessions-source.txt`, and the source decides which binary the next worker executes. `test_l1_engine_cannot_be_redirected_by_a_worker` has a worker do exactly that and checks the next session still runs on the real engine; reading the file instead fails it. - **Haiku 5.5, effort and a replaced system prompt (2026-10-07).** The claude manifest's `l1_model` is `claude-haiku-5-5` (verified: `--model claude-haiku-5-5` and the `haiku` alias both answer as it; an id the CLI does not know falls back silently, so re-check `modelUsage` in `--output-format json` when changing it). `l1-argv` and `warmup-argv` add `--effort` from `AUTODREAM_L1_EFFORT_CLAUDE`, then `AUTODREAM_L1_EFFORT`, then `high`; `off` omits it and an unknown level exits 2. They also pass `--system-prompt` instead of `--append-system-prompt`, which drops Claude Code's default prompt: one-word call 6.9k -> 5.0k input tokens, and a real triage run on a 250 KB slice cost $0.159 against $0.215 with the same number of findings. L2 is untouched. - **The engine, then the model.** A source's L1 engine is `AUTODREAM_L1_ENGINE_`, then `AUTODREAM_L1_ENGINE`, then the run's reviewer (the L2 engine) (`adapter_l1_engine`). The model is keyed by that engine, not by the source: `AUTODREAM_L1_MODEL_`, then `AUTODREAM_L1_MODEL`, then the engine's manifest `l1_model` (`adapter_l1_model`). A model id belongs to one engine, so a source-keyed pin sent omp sessions to the claude CLI with `--model deepseek/...` once a claude reviewer read them (PR #165 review). The engine choice is per source and the model pin per engine, the same as `AUTODREAM_L2_MODEL_`. +- **What answered, and whether it counts.** Every L1 call's stdout goes through the engine adapter's `served` verb with the requested model: it returns the models that answered, the cost estimate and a verdict, `match` or `mismatch`, judged by the adapter because only it knows its CLI's naming (claude: `haiku` answers as `claude-haiku-5-5`). An engine with no `served` verb (omp) is `unmeasured`. The verdict is the sixth column of `l1-served.tsv` (run-stats `l1_served_mismatch`, `l1_served_unmeasured`) and is stored beside every accepted answer, findings JSON or chunk answer, as `.served`. With `AUTODREAM_L1_REQUIRE_SERVED=1` a call whose verdict is not `match` loses its answer, and before L1 `drop_unacceptable_l1_cache` holds the cache to the same rule: a findings JSON or chunk answer from an earlier run with no verdict or a non-`match` one is dropped and triaged again (`l1_cache_dropped`). A record with `error` or `skipped` is kept, because the runner writes those itself and the post-L1 passes add fields to them, so their shape alone cannot tell them from a worker's. - **No engine is a deterministic error.** A session whose adapter is not executable, or resolves no model, gets a findings record with `error: no L1 engine for this session`, left in place and counted by `l1_findings_with_error`. Retrying cannot change the answer, and the run still reports. ## Self-pollution (the eating-its-own-tail bug) diff --git a/bin/run.sh b/bin/run.sh index 4a78e18..bd8e399 100755 --- a/bin/run.sh +++ b/bin/run.sh @@ -2157,6 +2157,43 @@ reconcile_findings_with_worklist() { log "WARNING: $FINDINGS_OUTSIDE_WORKLIST findings JSON(s) belong to no session in this run's worklist; moved to $_q so L2 does not read them: $OUTSIDE_FINDINGS_NAMES" fi } +# A cached L1 result is reused only if this run would accept it now. With +# AUTODREAM_L1_REQUIRE_SERVED=1, a findings JSON or chunk answer a model wrote is held to the +# verdict the runner stored beside it (.served, written in dispatch_l1): one with no +# verdict, or any verdict but match, came from a run that did not require it, or from before the +# verdict existed, and is dropped so this run triages it again. A record carrying error or skipped +# is kept: the runner writes those itself with no model behind it (a noise-gate skip, an unreadable +# session, a final-round stub), and the post-L1 passes add fields to them, so their shape cannot +# be told from a worker's. Runs once, after reconcile_findings_with_worklist and before the pre-L1 +# cache snapshot, so the dispatcher, l1_missing_count and L1_PRECACHED all see the same set. +# Counted in run-stats as l1_cache_dropped and named in the log. +L1_CACHE_DROPPED=0 +drop_unacceptable_l1_cache() { + L1_CACHE_DROPPED=0 + [ "${AUTODREAM_L1_REQUIRE_SERVED:-0}" = 1 ] || return 0 + local _h _f _names="" _chunks=0 + while IFS= read -r _h; do + [ -n "$_h" ] || continue + _f="$FINDINGS_DIR/$_h.json" + [ -f "$_f" ] || continue + l1_record_is_runner_only "$_f" && continue + [ "$(served_verdict "$_f")" = match ] && continue + rm -f "$_f" "$_f.served" + L1_CACHE_DROPPED=$((L1_CACHE_DROPPED + 1)); _names="${_names:+$_names }$_h" + done < <(worklist_hashes) + for _f in "$FINDINGS_DIR"/.chunks/*/*.chunkout; do + [ -f "$_f" ] || continue + [ "$(served_verdict "$_f")" = match ] && continue + rm -f "$_f" "$_f.served" + L1_CACHE_DROPPED=$((L1_CACHE_DROPPED + 1)); _chunks=$((_chunks + 1)) + done + [ "$L1_CACHE_DROPPED" -eq 0 ] || log "AUTODREAM_L1_REQUIRE_SERVED=1: dropped $L1_CACHE_DROPPED cached L1 result(s) no served verdict vouches for, to triage again (sessions: ${_names:-none}; chunk answers: $_chunks)" +} +# 0 for a record that says the session was not triaged (error) or was not sent to a model (skipped). +l1_record_is_runner_only() { # $1=findings JSON + jq -e 'has("error") or has("skipped")' "$1" >/dev/null 2>&1 +} +served_verdict() { cut -s -f4 "$1.served" 2>/dev/null | head -1; } # $1=answer -> its stored verdict scan_worklist_leftovers() { L1_ERR_ORPHANED=0; ORPHAN_ERR_NAMES="" local _hashes _f _h @@ -2672,7 +2709,7 @@ dispatch_l1() { # one parallel pass; idempotent worker → only the still-missin else printf "the %s engine did not report which model answered, so the call cannot be held to the requested model [%s]; its findings are discarded (AUTODREAM_L1_REQUIRE_SERVED=1)\n" "$engine" "$model" >> "$3" fi - rm -f "$2" + rm -f "$2" "$2.served" return 1 fi @@ -2693,7 +2730,16 @@ dispatch_l1() { # one parallel pass; idempotent worker → only the still-missin rm -f "$2" fi - [ -s "$2" ] && return 0 + # The verdict travels with the answer it describes, in .served (engine, requested + # model, models that answered, verdict), written by the runner after the engine has exited, + # so whatever the worker wrote there itself is replaced. A later run reads it before reusing + # the answer: with AUTODREAM_L1_REQUIRE_SERVED=1 a cached findings JSON or chunk answer is + # reused only when this says match (drop_unacceptable_l1_cache in the parent). + rm -f "$2.served" + if [ -s "$2" ]; then + printf "%s\t%s\t%s\t%s\n" "$engine" "$model" "${l1smodels:-unknown}" "$l1sverdict" > "$2.served" + return 0 + fi # Worker exited without writing findings JSON. Record a diagnostic so the # failure is visible. printf "worker produced no findings JSON for %s (incomplete run: the engine exited without writing output)\n" "${l1_subject:-$session}" >> "$3" @@ -2816,8 +2862,24 @@ dispatch_l1() { # one parallel pass; idempotent worker → only the still-missin if bash "$MERGER" --session "$session" --elided "$elided" "${parts[@]}" > "$output.merge" 2>> "$errlog" \ && [ -s "$output.merge" ] && jq -e ".findings | arrays" "$output.merge" >/dev/null 2>&1; then mv -f "$output.merge" "$output" + # The merged answer is vouched for only as far as its chunks were: match when every chunk + # answer says match, mismatch when any says mismatch, unmeasured otherwise (a chunk + # whose engine reports no model, or an answer cached before the verdict existed). + mverdict=match; mmodels="" + for cpart in "${parts[@]}"; do + cpv=$(cut -s -f4 "$cpart.served" 2>/dev/null | head -1) + cpm=$(cut -s -f3 "$cpart.served" 2>/dev/null | head -1) + mmodels="$mmodels,${cpm:-unknown}" + case "$cpv" in + match) ;; + mismatch) mverdict=mismatch ;; + *) [ "$mverdict" = mismatch ] || mverdict=unmeasured ;; + esac + done + mmodels=$(printf "%s\n" "${mmodels#,}" | tr "," "\n" | sort -u | paste -s -d "," -) + printf "%s\t%s\t%s\t%s\n" "$engine" "$model" "$mmodels" "$mverdict" > "$output.served" else - rm -f "$output.merge" "${parts[@]}" + rm -f "$output.merge" "${parts[@]}" "${parts[@]/%/.served}" printf "merging the %s chunk answers of %s failed; they were discarded so the next round redoes them\n" "$nchunks" "$session" >> "$errlog" fi else @@ -3276,6 +3338,7 @@ run() { printf 'l1_errored_provider: 0\n' printf 'l1_errored_unclassified: 0\n' printf 'l1_sessions_already_done_at_start: 0\n' + printf 'l1_cache_dropped: 0\n' printf 'l1_sessions_freshly_processed: 0\n' printf 'l1_elapsed_seconds: 0\n' printf 'oversized_total: 0\n' @@ -3484,6 +3547,7 @@ EOF sweep_killed_leftovers reconcile_findings_with_worklist + drop_unacceptable_l1_cache clean_work_bucket # start clean: drop any stub left by a prior run's workers @@ -4037,6 +4101,8 @@ PY # Cached vs. fresh: lets the aggregator distinguish a sub-second "elapsed" # caused by everything already being done from a broken timer. printf 'l1_sessions_already_done_at_start: %s\n' "$L1_PRECACHED" + # Cached results this run would not reuse (drop_unacceptable_l1_cache), triaged again instead. + printf 'l1_cache_dropped: %s\n' "$L1_CACHE_DROPPED" printf 'l1_sessions_freshly_processed: %s\n' "$L1_FRESHLY_PROCESSED" printf 'l1_elapsed_seconds: %s\n' "$L1_ELAPSED" # Global cross-session overlap stat (#14) — see compute_overlap_stats above. diff --git a/tests/run-all.sh b/tests/run-all.sh index 0bf730a..3922ef0 100755 --- a/tests/run-all.sh +++ b/tests/run-all.sh @@ -4332,6 +4332,40 @@ test_the_runs_reviewer_reviews_every_source_by_default(){ rm -rf "$root" } +test_require_served_holds_cached_findings_to_their_verdict(){ + echo "# AUTODREAM_L1_REQUIRE_SERVED=1 also holds cached findings: one another model wrote is triaged again" + local root fd h; root=$(setup_env); mk_session "$root" sess1; h=$(hash_of "$root/projects/proj-a/sess1.jsonl") + MOCK_SERVED_MODEL=claude-other-model run_dream "$root" + fd=$(fdir "$root") + assert_grep "$fd/$h.json" 'always run tests after edits' "run 1, not requiring it, kept the substituted model's findings" + assert_grep "$fd/$h.json.served" ' claude-other-model mismatch$' "and stored the mismatch beside them" + AUTODREAM_FORCE=1 AUTODREAM_L1_REQUIRE_SERVED=1 MOCK_CALL_LOG="$root/calls2.log" run_dream "$root" + assert_eq "$(grep -c . "$root/calls2.log" 2>/dev/null)" "1" "run 2, requiring it, called the engine for the session again" + assert_grep "$root/run.out" 'dropped 1 cached L1 result(s) no served verdict vouches for' "the log says why" + assert_grep "$fd/run-stats.txt" '^l1_cache_dropped: 1$' "run-stats counts it" + assert_grep "$fd/run-stats.txt" '^l1_sessions_already_done_at_start: 0$' "and it is not counted as already done" + assert_grep "$fd/$h.json.served" ' claude-haiku-5-5 match$' "the new findings carry a match" + assert_grep "$fd/$h.json" 'always run tests after edits' "and are kept" + rm -rf "$root" + echo "# findings with no stored verdict (written before it existed) are triaged again; a match is reused" + root=$(setup_env); mk_session "$root" sess1; h=$(hash_of "$root/projects/proj-a/sess1.jsonl") + run_dream "$root"; fd=$(fdir "$root") + AUTODREAM_FORCE=1 AUTODREAM_L1_REQUIRE_SERVED=1 MOCK_CALL_LOG="$root/calls2.log" run_dream "$root" + assert_no_file "$root/calls2.log" "findings whose verdict is match are reused, with no call" + assert_grep "$fd/run-stats.txt" '^l1_cache_dropped: 0$' "and nothing is dropped" + rm -f "$fd/$h.json.served" + AUTODREAM_FORCE=1 AUTODREAM_L1_REQUIRE_SERVED=1 MOCK_CALL_LOG="$root/calls3.log" run_dream "$root" + assert_eq "$(grep -c . "$root/calls3.log" 2>/dev/null)" "1" "findings with no verdict are triaged again" + rm -rf "$root" + echo "# a record the runner wrote with no model behind it is kept: it is not re-dispatched every run" + root=$(setup_env); mk_session "$root" sess1; h=$(hash_of "$root/projects/proj-a/sess1.jsonl") + MOCK_MODE=l1_incomplete AUTODREAM_L1_ROUNDS=1 run_dream "$root"; fd=$(fdir "$root") + assert_grep "$fd/$h.json" 'worker exited without findings JSON' "run 1 left the final-round stub" + AUTODREAM_FORCE=1 AUTODREAM_L1_REQUIRE_SERVED=1 MOCK_CALL_LOG="$root/calls2.log" run_dream "$root" + assert_no_file "$root/calls2.log" "run 2 kept it and called nothing" + assert_grep "$fd/run-stats.txt" '^l1_cache_dropped: 0$' "and dropped nothing" + rm -rf "$root" +} test_an_unusable_l1_engine_override_fails_closed(){ echo "# an L1 engine override that names no accepted adapter: no model call at all, not the omp engine" local root o h fd trip; root=$(setup_env); mk_session "$root" sess1 @@ -5639,6 +5673,7 @@ _run test_omp_session_roots_come_from_the_host_setting _run test_l1_provenance_and_served_models_are_recorded _run test_an_engine_failing_its_login_requirement_is_handed_nothing _run test_a_call_answered_by_another_model_loses_its_findings +_run test_require_served_holds_cached_findings_to_their_verdict _run test_omp_nested_child_is_a_sidechain_of_its_bucket _run test_omp_session_that_cannot_be_linearized_is_an_error_record _run test_omp_stats_describe_the_live_branch_only @@ -6417,6 +6452,33 @@ test_a_cached_chunk_answer_is_not_reused_on_another_engine(){ rm -rf "$root" } +test_require_served_holds_cached_chunk_answers_to_their_verdict(){ + echo "# AUTODREAM_L1_REQUIRE_SERVED=1 also holds cached chunk answers: one another model wrote is redone" + local scenario + for scenario in substituted matched; do + local root; root=$(setup_env) + local f; f=$(mk_chunky "$root" big 60); local h; h=$(hash_of "$f"); local fd; fd=$(fdir "$root") + # Run 1, not requiring it: a permanent refusal on chunk 2 defers the date and leaves chunk 1's answer. + export MOCK_MODE=chunked MOCK_CALL_LOG="$root/calls1.log" MOCK_FAIL_CHUNK=2 MOCK_FAIL_KIND=refusal AUTODREAM_L1_ROUNDS=1 + [ "$scenario" = substituted ] && export MOCK_SERVED_MODEL=claude-other-model + run_chunked "$root"; unset MOCK_MODE MOCK_CALL_LOG MOCK_FAIL_CHUNK MOCK_FAIL_KIND AUTODREAM_L1_ROUNDS MOCK_SERVED_MODEL + assert_eq "$(ls "$fd/.chunks/$h"/01-*.chunkout 2>/dev/null | wc -l | tr -d ' ')" "1" "[$scenario] run 1 left chunk 1's answer behind" + export MOCK_MODE=chunked MOCK_CALL_LOG="$root/calls2.log" AUTODREAM_L1_ROUNDS=1 AUTODREAM_L1_REQUIRE_SERVED=1 + run_chunked "$root"; unset MOCK_MODE MOCK_CALL_LOG AUTODREAM_L1_ROUNDS AUTODREAM_L1_REQUIRE_SERVED + if [ "$scenario" = substituted ]; then + assert_eq "$(grep -c '^01-' <(sed 's#.*/##' "$root/calls2.log"))" "1" "[$scenario] chunk 1 was called again, not reused" + assert_nogrep "$root/run.out" "reuse: chunk 1/" "[$scenario] and the log does not say it reused it" + assert_grep "$fd/run-stats.txt" '^l1_cache_dropped: 1$' "[$scenario] run-stats counts the dropped answer" + else + assert_eq "$(grep -c '^01-' <(sed 's#.*/##' "$root/calls2.log"))" "0" "[$scenario] chunk 1 was reused" + assert_grep "$fd/run-stats.txt" '^l1_cache_dropped: 0$' "[$scenario] and nothing was dropped" + fi + assert_eq "$(jq -r 'has("error")' "$fd/$h.json" 2>/dev/null)" "false" "[$scenario] the session finished" + assert_grep "$fd/$h.json.served" ' match$' "[$scenario] and the merged findings carry a match" + rm -rf "$root" + done +} + test_a_bad_chunk_answer_is_never_merged_around(){ echo "# chunking: an error, partial or malformed chunk answer blocks the merge and is redone alone" local kind @@ -6602,6 +6664,7 @@ _run test_a_failed_chunker_on_a_transcript_that_was_never_slimmed_is_still_bound _run test_a_chunk_that_fails_is_retried_alone_and_finished_chunks_are_reused _run test_a_cached_chunk_answer_is_not_reused_after_the_instructions_or_the_session_change _run test_a_cached_chunk_answer_is_not_reused_on_another_engine +_run test_require_served_holds_cached_chunk_answers_to_their_verdict _run test_a_bad_chunk_answer_is_never_merged_around _run test_a_chunk_failure_is_classified_like_any_worker_failure _run test_a_provider_refusal_on_a_chunk_defers_the_date_and_stops_the_other_chunks From 9f0907d16c4679afea7be21cb9f8d927bd0162b1 Mon Sep 17 00:00:00 2001 From: Daniel Young Date: Thu, 29 Oct 2026 11:31:38 -0400 Subject: [PATCH 08/11] fix: retry a no-engine session once a later run resolves its engine A session with no usable L1 engine gets an error record that counts as done, on the rule that retrying cannot change the answer. That held when the only cause was an adapter missing on disk. Now an engine choice naming no accepted adapter, a missing model or a login failing AUTODREAM_REQUIRE_LOGIN writes the same record, and all of those are config: fixing it and running the date again (a catch-up or AUTODREAM_FORCE=1) still skipped the session. The record now carries no_engine: true. The pass that runs before L1, drop_unacceptable_l1_cache, drops it when this run resolves an engine and a model for the session's source, read from AUTODREAM_SOURCE_MAP rather than the record, so the session is triaged. Within one run the record still stands, so the retry rounds do not spin on it and the report still ships. Records written before the marker are recognised by their error text. Tests: run-all writes the record with a bad engine choice and with a failing login, then reruns the date: the same bad config keeps it and calls nothing, a fixed one triages the session. --- CLAUDE.md | 2 +- bin/run.sh | 74 ++++++++++++++++++++++++++++++++++++------------ tests/run-all.sh | 35 ++++++++++++++++++++++- 3 files changed, 91 insertions(+), 20 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index 923435a..1036880 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -110,7 +110,7 @@ plus env `CLAUDE_CODE_DISABLE_CLAUDE_MDS=1 DISABLE_TELEMETRY=1 DISABLE_ERROR_REP - **Haiku 5.5, effort and a replaced system prompt (2026-10-07).** The claude manifest's `l1_model` is `claude-haiku-5-5` (verified: `--model claude-haiku-5-5` and the `haiku` alias both answer as it; an id the CLI does not know falls back silently, so re-check `modelUsage` in `--output-format json` when changing it). `l1-argv` and `warmup-argv` add `--effort` from `AUTODREAM_L1_EFFORT_CLAUDE`, then `AUTODREAM_L1_EFFORT`, then `high`; `off` omits it and an unknown level exits 2. They also pass `--system-prompt` instead of `--append-system-prompt`, which drops Claude Code's default prompt: one-word call 6.9k -> 5.0k input tokens, and a real triage run on a 250 KB slice cost $0.159 against $0.215 with the same number of findings. L2 is untouched. - **The engine, then the model.** A source's L1 engine is `AUTODREAM_L1_ENGINE_`, then `AUTODREAM_L1_ENGINE`, then the run's reviewer (the L2 engine) (`adapter_l1_engine`). The model is keyed by that engine, not by the source: `AUTODREAM_L1_MODEL_`, then `AUTODREAM_L1_MODEL`, then the engine's manifest `l1_model` (`adapter_l1_model`). A model id belongs to one engine, so a source-keyed pin sent omp sessions to the claude CLI with `--model deepseek/...` once a claude reviewer read them (PR #165 review). The engine choice is per source and the model pin per engine, the same as `AUTODREAM_L2_MODEL_`. - **What answered, and whether it counts.** Every L1 call's stdout goes through the engine adapter's `served` verb with the requested model: it returns the models that answered, the cost estimate and a verdict, `match` or `mismatch`, judged by the adapter because only it knows its CLI's naming (claude: `haiku` answers as `claude-haiku-5-5`). An engine with no `served` verb (omp) is `unmeasured`. The verdict is the sixth column of `l1-served.tsv` (run-stats `l1_served_mismatch`, `l1_served_unmeasured`) and is stored beside every accepted answer, findings JSON or chunk answer, as `.served`. With `AUTODREAM_L1_REQUIRE_SERVED=1` a call whose verdict is not `match` loses its answer, and before L1 `drop_unacceptable_l1_cache` holds the cache to the same rule: a findings JSON or chunk answer from an earlier run with no verdict or a non-`match` one is dropped and triaged again (`l1_cache_dropped`). A record with `error` or `skipped` is kept, because the runner writes those itself and the post-L1 passes add fields to them, so their shape alone cannot tell them from a worker's. -- **No engine is a deterministic error.** A session whose adapter is not executable, or resolves no model, gets a findings record with `error: no L1 engine for this session`, left in place and counted by `l1_findings_with_error`. Retrying cannot change the answer, and the run still reports. +- **No engine is an error for this run's config, not forever.** A session whose engine is not executable, resolves no model, names no accepted adapter or fails `AUTODREAM_REQUIRE_LOGIN` gets a findings record with `no_engine: true` and `error: no L1 engine for this session`, counted by `l1_findings_with_error` and skipped by the retry rounds, since retrying in the same run cannot change the answer, and the run still reports. Every one of those causes is config, which can change between runs, so `drop_unacceptable_l1_cache` drops the record in a later run (a catch-up or `AUTODREAM_FORCE=1`) once that run resolves an engine and a model for the session's source, read from `AUTODREAM_SOURCE_MAP` and never from the record. Records written before the marker are recognised by their error text. ## Self-pollution (the eating-its-own-tail bug) diff --git a/bin/run.sh b/bin/run.sh index bd8e399..214b8a6 100755 --- a/bin/run.sh +++ b/bin/run.sh @@ -2157,37 +2157,71 @@ reconcile_findings_with_worklist() { log "WARNING: $FINDINGS_OUTSIDE_WORKLIST findings JSON(s) belong to no session in this run's worklist; moved to $_q so L2 does not read them: $OUTSIDE_FINDINGS_NAMES" fi } -# A cached L1 result is reused only if this run would accept it now. With -# AUTODREAM_L1_REQUIRE_SERVED=1, a findings JSON or chunk answer a model wrote is held to the +# A cached L1 result is reused only if this run would accept it now, for two reasons. +# +# A no-engine record (no_engine, written by dispatch_l1 when a source had no usable engine) is +# final only for the config that wrote it: an engine choice naming no accepted adapter, a missing +# model, a login failing AUTODREAM_REQUIRE_LOGIN. Once this run resolves an engine and a model for +# the session's source, the record is dropped and the session triaged. The source comes from +# AUTODREAM_SOURCE_MAP, never from the record, which a worker could have written. +# +# With AUTODREAM_L1_REQUIRE_SERVED=1, a findings JSON or chunk answer a model wrote is held to the # verdict the runner stored beside it (.served, written in dispatch_l1): one with no # verdict, or any verdict but match, came from a run that did not require it, or from before the # verdict existed, and is dropped so this run triages it again. A record carrying error or skipped # is kept: the runner writes those itself with no model behind it (a noise-gate skip, an unreadable # session, a final-round stub), and the post-L1 passes add fields to them, so their shape cannot -# be told from a worker's. Runs once, after reconcile_findings_with_worklist and before the pre-L1 -# cache snapshot, so the dispatcher, l1_missing_count and L1_PRECACHED all see the same set. -# Counted in run-stats as l1_cache_dropped and named in the log. +# be told from a worker's. +# +# Runs once, after reconcile_findings_with_worklist and before the pre-L1 cache snapshot, so the +# dispatcher, l1_missing_count and L1_PRECACHED all see the same set. Counted in run-stats as +# l1_cache_dropped and named in the log. L1_CACHE_DROPPED=0 drop_unacceptable_l1_cache() { L1_CACHE_DROPPED=0 - [ "${AUTODREAM_L1_REQUIRE_SERVED:-0}" = 1 ] || return 0 - local _h _f _names="" _chunks=0 + local _h _f _src _unrouted="" _unverified="" _chunks=0 _served=0 + [ "${AUTODREAM_L1_REQUIRE_SERVED:-0}" = 1 ] && _served=1 while IFS= read -r _h; do [ -n "$_h" ] || continue _f="$FINDINGS_DIR/$_h.json" [ -f "$_f" ] || continue + if l1_record_has_no_engine "$_f"; then + _src=$(printf '%s\n' "$AUTODREAM_SOURCE_MAP" | awk -F'\t' -v h="$_h" '$1 == h { print $2; exit }') + l1_source_has_engine "$_src" || continue + rm -f "$_f" "$_f.served" + L1_CACHE_DROPPED=$((L1_CACHE_DROPPED + 1)); _unrouted="${_unrouted:+$_unrouted }$_h" + continue + fi + [ "$_served" = 1 ] || continue l1_record_is_runner_only "$_f" && continue [ "$(served_verdict "$_f")" = match ] && continue rm -f "$_f" "$_f.served" - L1_CACHE_DROPPED=$((L1_CACHE_DROPPED + 1)); _names="${_names:+$_names }$_h" + L1_CACHE_DROPPED=$((L1_CACHE_DROPPED + 1)); _unverified="${_unverified:+$_unverified }$_h" done < <(worklist_hashes) - for _f in "$FINDINGS_DIR"/.chunks/*/*.chunkout; do - [ -f "$_f" ] || continue - [ "$(served_verdict "$_f")" = match ] && continue - rm -f "$_f" "$_f.served" - L1_CACHE_DROPPED=$((L1_CACHE_DROPPED + 1)); _chunks=$((_chunks + 1)) - done - [ "$L1_CACHE_DROPPED" -eq 0 ] || log "AUTODREAM_L1_REQUIRE_SERVED=1: dropped $L1_CACHE_DROPPED cached L1 result(s) no served verdict vouches for, to triage again (sessions: ${_names:-none}; chunk answers: $_chunks)" + if [ "$_served" = 1 ]; then + for _f in "$FINDINGS_DIR"/.chunks/*/*.chunkout; do + [ -f "$_f" ] || continue + [ "$(served_verdict "$_f")" = match ] && continue + rm -f "$_f" "$_f.served" + L1_CACHE_DROPPED=$((L1_CACHE_DROPPED + 1)); _chunks=$((_chunks + 1)) + done + fi + [ -z "$_unrouted" ] || log "dropped no-engine record(s) whose source now has an L1 engine, to triage again: $_unrouted" + [ -z "$_unverified" ] && [ "$_chunks" -eq 0 ] \ + || log "AUTODREAM_L1_REQUIRE_SERVED=1: dropped cached L1 result(s) no served verdict vouches for, to triage again (sessions: ${_unverified:-none}; chunk answers: $_chunks)" +} +# 0 for a no-engine record, by its marker or, for one written before the marker, by its error. +l1_record_has_no_engine() { # $1=findings JSON + jq -e '.no_engine == true or ((.error // "") | tostring | startswith("no L1 engine for this session"))' "$1" >/dev/null 2>&1 +} +# 0 when this run resolved an engine and a model for the source, as dispatch_l1 will read them. +l1_source_has_engine() { # $1=source adapter + local _e _m + case "$1" in ""|*[!abcdefghijklmnopqrstuvwxyz0123456789_-]*) return 1 ;; esac + _e=$(printf '%s\n' "$AUTODREAM_L1_ENGINES" | awk -F'\t' -v s="$1" '$1 == s { print $2; exit }') + _m=$(printf '%s\n' "$AUTODREAM_L1_MODELS" | awk -F'\t' -v s="$1" '$1 == s { print $2; exit }') + case "$_e" in ""|*[!abcdefghijklmnopqrstuvwxyz0123456789_-]*) return 1 ;; esac + [ -n "$_m" ] && [ -x "$ADAPTERS_DIR/$_e/adapter.sh" ] } # 0 for a record that says the session was not triaged (error) or was not sent to a model (skipped). l1_record_is_runner_only() { # $1=findings JSON @@ -2539,10 +2573,14 @@ dispatch_l1() { # one parallel pass; idempotent worker → only the still-missin while IFS= read -r line; do [ -n "$line" ] && envs+=("$line"); done < <("$ADAPTERS_DIR/$engine/adapter.sh" l1-env 2>/dev/null) fi if [ "${#argv[@]}" -eq 0 ]; then - # Deterministic, so a structured error record that is left in place and skipped on re-run - # (retrying would not change the answer) and counted by l1_findings_with_error. + # Deterministic for this run: retrying in a later round would not change the answer, so a + # structured error record is left in place, skipped by the rounds that follow and counted by + # l1_findings_with_error. It is not deterministic across runs. The engine choice, the model + # pin and the login check all come from the config, so no_engine marks the record and a later + # run whose config resolves an engine for this source drops it and triages the session + # (drop_unacceptable_l1_cache in the parent). rm -f "$slimfile" "$normfile" "$dayfile" - jq -cn --arg p "$session" --arg s "$src" --arg m "$model" "{session_path: \$p, error: (\"no L1 engine for this session (source [\" + \$s + \"], model [\" + \$m + \"])\"), findings: []}" > "$output" + jq -cn --arg p "$session" --arg s "$src" --arg m "$model" "{session_path: \$p, no_engine: true, error: (\"no L1 engine for this session (source [\" + \$s + \"], model [\" + \$m + \"])\"), findings: []}" > "$output" rm -f "$errlog" echo "skip (no engine): $session ($hash)" >&2 exit 0 diff --git a/tests/run-all.sh b/tests/run-all.sh index 3922ef0..b9eca87 100755 --- a/tests/run-all.sh +++ b/tests/run-all.sh @@ -4341,7 +4341,7 @@ test_require_served_holds_cached_findings_to_their_verdict(){ assert_grep "$fd/$h.json.served" ' claude-other-model mismatch$' "and stored the mismatch beside them" AUTODREAM_FORCE=1 AUTODREAM_L1_REQUIRE_SERVED=1 MOCK_CALL_LOG="$root/calls2.log" run_dream "$root" assert_eq "$(grep -c . "$root/calls2.log" 2>/dev/null)" "1" "run 2, requiring it, called the engine for the session again" - assert_grep "$root/run.out" 'dropped 1 cached L1 result(s) no served verdict vouches for' "the log says why" + assert_grep "$root/run.out" "dropped cached L1 result(s) no served verdict vouches for, to triage again (sessions: $h;" "the log says why" assert_grep "$fd/run-stats.txt" '^l1_cache_dropped: 1$' "run-stats counts it" assert_grep "$fd/run-stats.txt" '^l1_sessions_already_done_at_start: 0$' "and it is not counted as already done" assert_grep "$fd/$h.json.served" ' claude-haiku-5-5 match$' "the new findings carry a match" @@ -4366,6 +4366,38 @@ test_require_served_holds_cached_findings_to_their_verdict(){ assert_grep "$fd/run-stats.txt" '^l1_cache_dropped: 0$' "and dropped nothing" rm -rf "$root" } +test_a_no_engine_record_is_retried_once_an_engine_resolves(){ + echo "# a no-engine record left by a bad engine choice is triaged by a later run whose config resolves one" + local root o h fd; root=$(setup_env); o=$(mk_omp_session "$root" xxxx); h=$(hash_of "$o") + AUTODREAM_L1_ENGINE_OMP=nonesuch run_dream_omp "$root" + fd=$(fdir "$root") + assert_eq "$(jq -r '.no_engine' "$fd/$h.json" 2>/dev/null)" "true" "run 1 wrote the no-engine record, marked" + AUTODREAM_FORCE=1 AUTODREAM_L1_ENGINE_OMP=nonesuch MOCK_CALL_LOG="$root/calls2.log" run_dream_omp "$root" + assert_no_file "$root/calls2.log" "a run with the same bad choice keeps it and calls nothing" + assert_eq "$(jq -r '.no_engine' "$fd/$h.json" 2>/dev/null)" "true" "and the record stays" + AUTODREAM_FORCE=1 MOCK_CALL_LOG="$root/calls3.log" run_dream_omp "$root" + assert_eq "$(grep -c . "$root/calls3.log" 2>/dev/null)" "1" "a run whose config resolves the engine triages the session" + assert_grep "$root/run.out" "dropped no-engine record(s) whose source now has an L1 engine, to triage again: $h" "the log says why" + assert_grep "$fd/run-stats.txt" '^l1_cache_dropped: 1$' "run-stats counts it" + assert_nogrep "$fd/$h.json" 'no L1 engine' "and the record is replaced by findings" + rm -rf "$root" + echo "# so is one left by an engine that failed AUTODREAM_REQUIRE_LOGIN, once its login passes" + root=$(setup_env); o=$(mk_omp_session "$root" yyyy); h=$(hash_of "$o") + AUTODREAM_ADAPTERS=claude,omp AUTODREAM_REQUIRE_LOGIN="provider=firstParty" run_dream_omp "$root" + fd=$(fdir "$root") + assert_eq "$(jq -r '.no_engine' "$fd/$h.json" 2>/dev/null)" "true" "run 1 handed the omp engine nothing" + AUTODREAM_FORCE=1 MOCK_CALL_LOG="$root/calls2.log" run_dream_omp "$root" + assert_eq "$(grep -c . "$root/calls2.log" 2>/dev/null)" "1" "run 2, with no login requirement, triages it" + rm -rf "$root" + echo "# a no-engine record written before the marker existed is recognised by its error" + root=$(setup_env); o=$(mk_omp_session "$root" zzzz); h=$(hash_of "$o") + AUTODREAM_L1_ENGINE_OMP=nonesuch run_dream_omp "$root" + fd=$(fdir "$root") + jq -c 'del(.no_engine)' "$fd/$h.json" > "$fd/$h.json.tmp" && mv "$fd/$h.json.tmp" "$fd/$h.json" + AUTODREAM_FORCE=1 MOCK_CALL_LOG="$root/calls2.log" run_dream_omp "$root" + assert_eq "$(grep -c . "$root/calls2.log" 2>/dev/null)" "1" "it is triaged once the engine resolves" + rm -rf "$root" +} test_an_unusable_l1_engine_override_fails_closed(){ echo "# an L1 engine override that names no accepted adapter: no model call at all, not the omp engine" local root o h fd trip; root=$(setup_env); mk_session "$root" sess1 @@ -5669,6 +5701,7 @@ _run test_omp_session_is_linearized_for_the_worker _run test_omp_sessions_run_on_another_adapters_engine _run test_the_runs_reviewer_reviews_every_source_by_default _run test_an_unusable_l1_engine_override_fails_closed +_run test_a_no_engine_record_is_retried_once_an_engine_resolves _run test_omp_session_roots_come_from_the_host_setting _run test_l1_provenance_and_served_models_are_recorded _run test_an_engine_failing_its_login_requirement_is_handed_nothing From e75c04a1864093d518517261ce68ef223345c1ef Mon Sep 17 00:00:00 2001 From: Daniel Young Date: Thu, 29 Oct 2026 11:33:37 -0400 Subject: [PATCH 09/11] fix: count the L1 calls with no cost instead of reading them as zero l1_cost_usd_estimate sums the cost each engine reports, and an engine with no served verb (omp) reports none, so an all-omp night read 0.0000: a degraded measurement that looks like a real one, which the self-audit rule in CLAUDE.md forbids. run-stats now prints l1_cost_unmeasured beside the sum, the number of calls it leaves out, so a zero reads as "nothing measured" when it is. Tests: run-all checks l1_cost_unmeasured is 0 on a claude night and 1 on a one-session omp night whose estimate is 0.0000. --- bin/run.sh | 5 ++++- tests/run-all.sh | 3 +++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/bin/run.sh b/bin/run.sh index 214b8a6..711941f 100755 --- a/bin/run.sh +++ b/bin/run.sh @@ -4086,8 +4086,11 @@ PY printf 'l1_served_mismatch: %s\n' "$(awk -F'\t' '$6 == "mismatch"' "$FINDINGS_DIR/l1-served.tsv" | wc -l | tr -d ' ')" printf 'l1_served_unmeasured: %s\n' "$(awk -F'\t' '$6 != "match" && $6 != "mismatch"' "$FINDINGS_DIR/l1-served.tsv" | wc -l | tr -d ' ')" printf 'l1_cost_usd_estimate: %s\n' "$(awk -F'\t' '$5 != "" { s += $5 } END { printf "%.4f", s }' "$FINDINGS_DIR/l1-served.tsv")" + # The calls the sum above leaves out because their engine reported no cost (omp, or a claude + # call with no result object), so an all-omp night reads 0.0000 beside a count, not alone. + printf 'l1_cost_unmeasured: %s\n' "$(awk -F'\t' '$5 == ""' "$FINDINGS_DIR/l1-served.tsv" | wc -l | tr -d ' ')" else - printf 'l1_served_models: none\nl1_served_mismatch: 0\nl1_served_unmeasured: 0\nl1_cost_usd_estimate: 0.0000\n' + printf 'l1_served_models: none\nl1_served_mismatch: 0\nl1_served_unmeasured: 0\nl1_cost_usd_estimate: 0.0000\nl1_cost_unmeasured: 0\n' fi # yes when the circuit breaker cut the retry budget, so a short l1_rounds_used is not # mistaken for a run that finished early and cleanly. diff --git a/tests/run-all.sh b/tests/run-all.sh index b9eca87..6468744 100755 --- a/tests/run-all.sh +++ b/tests/run-all.sh @@ -4438,6 +4438,7 @@ test_l1_provenance_and_served_models_are_recorded(){ assert_grep "$fd/run-stats.txt" '^l1_served_mismatch: 0$' "none was answered by another model" assert_grep "$fd/run-stats.txt" '^l1_served_unmeasured: 0$' "and every call reported its model" assert_grep "$fd/run-stats.txt" '^l1_cost_usd_estimate: 0.0250$' "and the engine's cost estimates are summed" + assert_grep "$fd/run-stats.txt" '^l1_cost_unmeasured: 0$' "with no call left out of the sum" rm -rf "$root" echo "# a model asked for by alias matches the canonical id the CLI answers as (the adapter verdict)" local h; root=$(setup_env); mk_session "$root" sess1; h=$(hash_of "$root/projects/proj-a/sess1.jsonl") @@ -4454,6 +4455,8 @@ test_l1_provenance_and_served_models_are_recorded(){ assert_grep "$fd/l1-served.tsv" "^$h omp omp/test-model unknown unmeasured$" "the omp call is recorded with no model and no verdict" assert_grep "$fd/run-stats.txt" '^l1_served_mismatch: 0$' "it is not a mismatch" assert_grep "$fd/run-stats.txt" '^l1_served_unmeasured: 1$' "it is counted as unmeasured" + assert_grep "$fd/run-stats.txt" '^l1_cost_usd_estimate: 0.0000$' "an all-omp night has no cost to sum" + assert_grep "$fd/run-stats.txt" '^l1_cost_unmeasured: 1$' "and says how many calls that leaves out, so 0.0000 never reads as free" rm -rf "$root" root=$(setup_env); o=$(mk_omp_session "$root" wwww); h=$(hash_of "$o") AUTODREAM_ADAPTERS=omp AUTODREAM_L1_REQUIRE_SERVED=1 run_dream_omp "$root" From ed8be439a2278807e004a449b96a55c40c3e4b11 Mon Sep 17 00:00:00 2001 From: Daniel Young Date: Thu, 29 Oct 2026 11:34:14 -0400 Subject: [PATCH 10/11] docs: changelog entry for the reviewer split, its provenance and the fixes Includes the upgrade note for a host running both harnesses with omp as the L2 engine: the default reviewer moves its Claude Code sessions from Haiku on the claude CLI to omp's L1 model unless it pins AUTODREAM_L1_ENGINE_CLAUDE=claude. --- CHANGELOG.md | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9c8a4ef..e0de137 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,18 @@ All notable changes to cc-autodream. Format loosely follows Keep a Changelog. +## 2026-10-29 + +### Changed +- **The engine that writes the summary now reviews every harness's sessions in L1 by default.** Which harness holds a transcript and which engine reads it used to be one choice: each adapter's sessions went to its own engine. A run now has one reviewer, `AUTODREAM_L2_ENGINE` (else the first enabled adapter), and it reads every source's sessions unless `AUTODREAM_L1_ENGINE` (every source) or `AUTODREAM_L1_ENGINE_` (one source) picks another. The source's own adapter still enumerates, linearizes and adds its triage notes; only the model call, its warmup and the network probe move. A host with one harness behaves exactly as before. **Upgrade note:** a host with both `claude` and `omp` enabled and `AUTODREAM_L2_ENGINE=omp` moves every Claude Code session from Haiku on the claude CLI to omp's L1 model on this upgrade. Set `AUTODREAM_L1_ENGINE_CLAUDE=claude` to keep them where they were. +- **`AUTODREAM_L1_MODEL_` is keyed by the engine that makes the call, not by the harness that holds the session.** A model id belongs to an engine, so `AUTODREAM_L1_MODEL_OMP` is the model of every L1 call on the omp engine, whichever harness the session came from, and never reaches the claude CLI. A host whose harnesses each review their own sessions sees no change. One that moves a harness to another engine pins that engine's model, the same keying `AUTODREAM_L2_MODEL_` already had. +- **A session with no usable L1 engine is retried once a later run gives it one.** Its error record counted as done, which held while the only cause was an adapter missing on disk. An engine choice naming no accepted adapter, a missing model and a failed login check are config, so the record now carries `no_engine: true`, and a later run of the date (a catch-up or `AUTODREAM_FORCE=1`) that resolves an engine and a model for the session's source drops it and triages the session. + +### Added +- **`AUTODREAM_SESSION_ROOTS_`** lists the session roots of one adapter other than claude, colon-separated, in place of its manifest's defaults, for a harness that keeps one store per profile. +- **Who each engine runs as, and which model answered each L1 call.** Each engine the run uses reports its login once through a new adapter verb, `whoami` (claude: `claude auth status --json`, no model call), logged and written to run-stats as `login_`. The claude L1 worker runs with `--output-format json`, and the new `served` verb reads the models that answered, the CLI's cost estimate at API prices and a verdict, `match` or `mismatch`, that knows the CLI's naming (`haiku` answers as `claude-haiku-5-5`). Every call goes to `findings//l1-served.tsv`. run-stats gains `l1_engine_`, `l1_served_models`, `l1_served_mismatch`, `l1_served_unmeasured` (calls whose engine reports no model, such as omp), `l1_cost_usd_estimate` and `l1_cost_unmeasured` (calls left out of that sum, so an all-omp night does not read as free). +- **Both are record-only unless the host asks.** `AUTODREAM_REQUIRE_LOGIN` (space-separated `provider=`, `auth=`, `plan=`, `account=` pairs) hands an L1 engine whose login does not match no transcript, through the no-engine record above, and stops the run before it starts when the L2 engine does not match. `AUTODREAM_L1_REQUIRE_SERVED=1` discards the findings of any L1 call whose verdict is not `match`, an engine that cannot report one included. The verdict is stored beside every accepted answer (`.served`), and before L1 the run drops cached findings and chunk answers from earlier runs that no stored verdict vouches for and triages them again (`l1_cache_dropped`). + ## 2026-10-27 ### Fixed From e142066e1200fc23c8452d7be27a18a93ceb7880 Mon Sep 17 00:00:00 2001 From: Daniel Young Date: Thu, 29 Oct 2026 13:36:01 -0400 Subject: [PATCH 11/11] fix: hold any cached answer with findings to its served verdict drop_unacceptable_l1_cache kept any cached findings JSON carrying error or skipped without reading its stored verdict, on the grounds that the runner writes those records itself. A worker holds the Write tool and can put either key in its own answer, so under AUTODREAM_L1_REQUIRE_SERVED=1 a substituted model's {"skipped":true,"findings":[...]} from an earlier run was kept, passed the worker's idempotency read and reached L2. The no-engine marker had the same gap: a forged no_engine record with findings was kept for as long as its source had no engine. Every record the runner writes has findings: [], so both predicates now require it. A record with findings in it is model output whatever else it carries and goes through the verdict. Tests: run-all forges skipped, error and no_engine answers with findings over a mismatched verdict, reruns under the requirement and checks each is dropped and triaged again (or replaced by the runner's own no-engine record when the source has no engine). --- CLAUDE.md | 4 ++-- bin/run.sh | 17 +++++++++++------ tests/run-all.sh | 23 +++++++++++++++++++++++ 3 files changed, 36 insertions(+), 8 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index 1036880..a969823 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -109,8 +109,8 @@ plus env `CLAUDE_CODE_DISABLE_CLAUDE_MDS=1 DISABLE_TELEMETRY=1 DISABLE_ERROR_REP - **The source map is held in the environment, not read from the findings dir.** `AUTODREAM_SOURCE_MAP` (hash and adapter per session) and `AUTODREAM_L1_MODELS` (model per adapter) are built before the first model call. An L1 worker holds the Write tool and could rewrite `sessions-source.txt`, and the source decides which binary the next worker executes. `test_l1_engine_cannot_be_redirected_by_a_worker` has a worker do exactly that and checks the next session still runs on the real engine; reading the file instead fails it. - **Haiku 5.5, effort and a replaced system prompt (2026-10-07).** The claude manifest's `l1_model` is `claude-haiku-5-5` (verified: `--model claude-haiku-5-5` and the `haiku` alias both answer as it; an id the CLI does not know falls back silently, so re-check `modelUsage` in `--output-format json` when changing it). `l1-argv` and `warmup-argv` add `--effort` from `AUTODREAM_L1_EFFORT_CLAUDE`, then `AUTODREAM_L1_EFFORT`, then `high`; `off` omits it and an unknown level exits 2. They also pass `--system-prompt` instead of `--append-system-prompt`, which drops Claude Code's default prompt: one-word call 6.9k -> 5.0k input tokens, and a real triage run on a 250 KB slice cost $0.159 against $0.215 with the same number of findings. L2 is untouched. - **The engine, then the model.** A source's L1 engine is `AUTODREAM_L1_ENGINE_`, then `AUTODREAM_L1_ENGINE`, then the run's reviewer (the L2 engine) (`adapter_l1_engine`). The model is keyed by that engine, not by the source: `AUTODREAM_L1_MODEL_`, then `AUTODREAM_L1_MODEL`, then the engine's manifest `l1_model` (`adapter_l1_model`). A model id belongs to one engine, so a source-keyed pin sent omp sessions to the claude CLI with `--model deepseek/...` once a claude reviewer read them (PR #165 review). The engine choice is per source and the model pin per engine, the same as `AUTODREAM_L2_MODEL_`. -- **What answered, and whether it counts.** Every L1 call's stdout goes through the engine adapter's `served` verb with the requested model: it returns the models that answered, the cost estimate and a verdict, `match` or `mismatch`, judged by the adapter because only it knows its CLI's naming (claude: `haiku` answers as `claude-haiku-5-5`). An engine with no `served` verb (omp) is `unmeasured`. The verdict is the sixth column of `l1-served.tsv` (run-stats `l1_served_mismatch`, `l1_served_unmeasured`) and is stored beside every accepted answer, findings JSON or chunk answer, as `.served`. With `AUTODREAM_L1_REQUIRE_SERVED=1` a call whose verdict is not `match` loses its answer, and before L1 `drop_unacceptable_l1_cache` holds the cache to the same rule: a findings JSON or chunk answer from an earlier run with no verdict or a non-`match` one is dropped and triaged again (`l1_cache_dropped`). A record with `error` or `skipped` is kept, because the runner writes those itself and the post-L1 passes add fields to them, so their shape alone cannot tell them from a worker's. -- **No engine is an error for this run's config, not forever.** A session whose engine is not executable, resolves no model, names no accepted adapter or fails `AUTODREAM_REQUIRE_LOGIN` gets a findings record with `no_engine: true` and `error: no L1 engine for this session`, counted by `l1_findings_with_error` and skipped by the retry rounds, since retrying in the same run cannot change the answer, and the run still reports. Every one of those causes is config, which can change between runs, so `drop_unacceptable_l1_cache` drops the record in a later run (a catch-up or `AUTODREAM_FORCE=1`) once that run resolves an engine and a model for the session's source, read from `AUTODREAM_SOURCE_MAP` and never from the record. Records written before the marker are recognised by their error text. +- **What answered, and whether it counts.** Every L1 call's stdout goes through the engine adapter's `served` verb with the requested model: it returns the models that answered, the cost estimate and a verdict, `match` or `mismatch`, judged by the adapter because only it knows its CLI's naming (claude: `haiku` answers as `claude-haiku-5-5`). An engine with no `served` verb (omp) is `unmeasured`. The verdict is the sixth column of `l1-served.tsv` (run-stats `l1_served_mismatch`, `l1_served_unmeasured`) and is stored beside every accepted answer, findings JSON or chunk answer, as `.served`. With `AUTODREAM_L1_REQUIRE_SERVED=1` a call whose verdict is not `match` loses its answer, and before L1 `drop_unacceptable_l1_cache` holds the cache to the same rule: a findings JSON or chunk answer from an earlier run with no verdict or a non-`match` one is dropped and triaged again (`l1_cache_dropped`). A record with `error` or `skipped` and `findings: []` is kept, because the runner writes those itself and the post-L1 passes add fields to them, so their shape alone cannot tell them from a worker's. The empty array is the test, not the key: a worker can write either key, but every record the runner writes has no findings, so one with findings is model output and is held to its verdict. +- **No engine is an error for this run's config, not forever.** A session whose engine is not executable, resolves no model, names no accepted adapter or fails `AUTODREAM_REQUIRE_LOGIN` gets a findings record with `no_engine: true` and `error: no L1 engine for this session`, counted by `l1_findings_with_error` and skipped by the retry rounds, since retrying in the same run cannot change the answer, and the run still reports. Every one of those causes is config, which can change between runs, so `drop_unacceptable_l1_cache` drops the record in a later run (a catch-up or `AUTODREAM_FORCE=1`) once that run resolves an engine and a model for the session's source, read from `AUTODREAM_SOURCE_MAP` and never from the record. Records written before the marker are recognised by their error text. Either way the record must have `findings: []`: one with findings is model output, and with `AUTODREAM_L1_REQUIRE_SERVED=1` it is held to its verdict like any other. ## Self-pollution (the eating-its-own-tail bug) diff --git a/bin/run.sh b/bin/run.sh index 711941f..f0adf61 100755 --- a/bin/run.sh +++ b/bin/run.sh @@ -2169,9 +2169,12 @@ reconcile_findings_with_worklist() { # verdict the runner stored beside it (.served, written in dispatch_l1): one with no # verdict, or any verdict but match, came from a run that did not require it, or from before the # verdict existed, and is dropped so this run triages it again. A record carrying error or skipped -# is kept: the runner writes those itself with no model behind it (a noise-gate skip, an unreadable -# session, a final-round stub), and the post-L1 passes add fields to them, so their shape cannot -# be told from a worker's. +# with an empty findings array is kept: the runner writes those itself with no model behind it (a +# noise-gate skip, an unreadable session, a final-round stub), and the post-L1 passes add fields to +# them, so their shape cannot be told from a worker's. Neither can their keys, since a worker can +# write any of them, so the empty array is what is trusted: every record the runner writes has +# findings: [], and one with findings in it is model output whatever else it carries. That holds +# for the no-engine marker as well, so a no_engine record with findings goes through the verdict. # # Runs once, after reconcile_findings_with_worklist and before the pre-L1 cache snapshot, so the # dispatcher, l1_missing_count and L1_PRECACHED all see the same set. Counted in run-stats as @@ -2211,8 +2214,9 @@ drop_unacceptable_l1_cache() { || log "AUTODREAM_L1_REQUIRE_SERVED=1: dropped cached L1 result(s) no served verdict vouches for, to triage again (sessions: ${_unverified:-none}; chunk answers: $_chunks)" } # 0 for a no-engine record, by its marker or, for one written before the marker, by its error. +# Findings in it make it model output (see l1_record_is_runner_only). l1_record_has_no_engine() { # $1=findings JSON - jq -e '.no_engine == true or ((.error // "") | tostring | startswith("no L1 engine for this session"))' "$1" >/dev/null 2>&1 + jq -e '(.no_engine == true or ((.error // "") | tostring | startswith("no L1 engine for this session"))) and .findings == []' "$1" >/dev/null 2>&1 } # 0 when this run resolved an engine and a model for the source, as dispatch_l1 will read them. l1_source_has_engine() { # $1=source adapter @@ -2223,9 +2227,10 @@ l1_source_has_engine() { # $1=source adapter case "$_e" in ""|*[!abcdefghijklmnopqrstuvwxyz0123456789_-]*) return 1 ;; esac [ -n "$_m" ] && [ -x "$ADAPTERS_DIR/$_e/adapter.sh" ] } -# 0 for a record that says the session was not triaged (error) or was not sent to a model (skipped). +# 0 for a record that says the session was not triaged (error) or was not sent to a model (skipped) +# and has no findings. A worker can write either key, so a record with findings is model output. l1_record_is_runner_only() { # $1=findings JSON - jq -e 'has("error") or has("skipped")' "$1" >/dev/null 2>&1 + jq -e '(has("error") or has("skipped")) and .findings == []' "$1" >/dev/null 2>&1 } served_verdict() { cut -s -f4 "$1.served" 2>/dev/null | head -1; } # $1=answer -> its stored verdict scan_worklist_leftovers() { diff --git a/tests/run-all.sh b/tests/run-all.sh index 6468744..07d3980 100755 --- a/tests/run-all.sh +++ b/tests/run-all.sh @@ -4357,6 +4357,29 @@ test_require_served_holds_cached_findings_to_their_verdict(){ AUTODREAM_FORCE=1 AUTODREAM_L1_REQUIRE_SERVED=1 MOCK_CALL_LOG="$root/calls3.log" run_dream "$root" assert_eq "$(grep -c . "$root/calls3.log" 2>/dev/null)" "1" "findings with no verdict are triaged again" rm -rf "$root" + echo "# a model answer with findings is held to its verdict even when it carries the runner's skipped or error key" + local key + for key in skipped error; do + root=$(setup_env); mk_session "$root" sess1; h=$(hash_of "$root/projects/proj-a/sess1.jsonl") + MOCK_SERVED_MODEL=claude-other-model run_dream "$root"; fd=$(fdir "$root") + jq -c --arg k "$key" '{session_path} + {($k): true, findings: [{what: "forged by the substituted model"}]}' \ + "$fd/$h.json" > "$fd/$h.json.tmp" && mv "$fd/$h.json.tmp" "$fd/$h.json" + AUTODREAM_FORCE=1 AUTODREAM_L1_REQUIRE_SERVED=1 MOCK_CALL_LOG="$root/calls2.log" run_dream "$root" + assert_eq "$(grep -c . "$root/calls2.log" 2>/dev/null)" "1" "$key: run 2 triaged the session again" + assert_grep "$fd/run-stats.txt" '^l1_cache_dropped: 1$' "$key: run-stats counts the drop" + assert_nogrep "$fd/$h.json" 'forged by the substituted model' "$key: the forged findings are gone" + rm -rf "$root" + done + echo "# so is one carrying no_engine, while its source has no engine to triage it again" + root=$(setup_env); mk_session "$root" sess1; h=$(hash_of "$root/projects/proj-a/sess1.jsonl") + MOCK_SERVED_MODEL=claude-other-model run_dream "$root"; fd=$(fdir "$root") + jq -c '{session_path, no_engine: true, findings: [{what: "forged by the substituted model"}]}' \ + "$fd/$h.json" > "$fd/$h.json.tmp" && mv "$fd/$h.json.tmp" "$fd/$h.json" + AUTODREAM_FORCE=1 AUTODREAM_L1_REQUIRE_SERVED=1 AUTODREAM_L1_ENGINE_CLAUDE=nonesuch run_dream "$root" + assert_grep "$fd/run-stats.txt" '^l1_cache_dropped: 1$' "the forged record is dropped" + assert_nogrep "$fd/$h.json" 'forged by the substituted model' "its findings do not reach L2" + assert_grep "$fd/$h.json" 'no L1 engine for this session' "the runner writes its own no-engine record in its place" + rm -rf "$root" echo "# a record the runner wrote with no model behind it is kept: it is not re-dispatched every run" root=$(setup_env); mk_session "$root" sess1; h=$(hash_of "$root/projects/proj-a/sess1.jsonl") MOCK_MODE=l1_incomplete AUTODREAM_L1_ROUNDS=1 run_dream "$root"; fd=$(fdir "$root")