diff --git a/.github/workflows/hermetic-evals.yml b/.github/workflows/hermetic-evals.yml index 17a95296d6..e68887c544 100644 --- a/.github/workflows/hermetic-evals.yml +++ b/.github/workflows/hermetic-evals.yml @@ -36,6 +36,7 @@ jobs: run: | bash -n evals/hermetic/run.sh bash -n evals/agent_adoption/run.sh + bash -n evals/agent_adoption/claude_tools.sh bash -n scripts/build-eval-binary.sh - name: Run scorer and shell-runner regressions run: >- @@ -46,3 +47,5 @@ jobs: //evals/agent_adoption:selftest //scripts:build_eval_binary_test --test_output=errors + - name: Execute stock-edit oracle regressions + run: python3 evals/agent_adoption/stock_check_selftest.py diff --git a/crates/tracedecay-agent-hosts/src/agents/claude/tests.rs b/crates/tracedecay-agent-hosts/src/agents/claude/tests.rs index 707bc9da32..07725b1ec1 100644 --- a/crates/tracedecay-agent-hosts/src/agents/claude/tests.rs +++ b/crates/tracedecay-agent-hosts/src/agents/claude/tests.rs @@ -148,6 +148,43 @@ fn native_cache_content_drift_and_binary_relocation_require_refresh() { .unwrap() ); copy_rendered_bundle_to_native_cache(home.path(), new_bin); + let rendered = rendered_plugin_files(new_bin, &ServeArgs::default()).unwrap(); + let relatives = rendered + .iter() + .map(|(relative, _)| (*relative).to_string()) + .collect::>(); + for root in [ + plugin_deploy_dir(home.path()), + claude_current_cached_plugin_root(home.path()), + ] { + for (relative, expected) in &rendered { + let path = root.join(relative); + let actual = std::fs::read(&path).unwrap(); + assert!( + actual == expected.as_bytes(), + "relocated bundle differs at {}: actual bytes={}, expected bytes={}, first differing byte={:?}", + path.display(), + actual.len(), + expected.len(), + actual + .iter() + .zip(expected.as_bytes()) + .position(|(actual, expected)| actual != expected) + ); + } + assert_eq!( + super::super::unexpected_bundle_entrypoints( + &root, + &relatives, + &[".claude-plugin", "agents", "commands", "hooks", "skills"], + ) + .unwrap(), + Vec::::new(), + "relocation left auto-discovered entries in {}", + root.display() + ); + } + assert!(claude_plugin_registration_is_active(home.path()).unwrap()); assert!( claude_plugin_is_natively_active(home.path(), Some((new_bin, &ServeArgs::default()))) .unwrap() diff --git a/crates/tracedecay-code-index-runtime/src/code_index_scheduler.rs b/crates/tracedecay-code-index-runtime/src/code_index_scheduler.rs index baca65a33f..a325e59813 100644 --- a/crates/tracedecay-code-index-runtime/src/code_index_scheduler.rs +++ b/crates/tracedecay-code-index-runtime/src/code_index_scheduler.rs @@ -300,8 +300,7 @@ pub use serving::{ use serving::{ CodeIndexCommittedProgressSampleV1, CodeTextProjectionSlotV1, map_sealed_page_source_error, sha256_private_file_and_size, text_artifact_admitted_build_budget, - text_artifact_builder_budget, text_artifact_resident_memory_charge, - text_artifact_source_batch_limits, + text_artifact_builder_budget, text_artifact_source_batch_limits, }; #[cfg(test)] use tracedecay_query::retrieval::ports::TEXT_ARTIFACT_PAGE_CHUNKS_V1; diff --git a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/serving.rs b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/serving.rs index de14bd949d..719d44539f 100644 --- a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/serving.rs +++ b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/serving.rs @@ -5,7 +5,6 @@ mod family_report; use std::{ collections::{BTreeSet, VecDeque}, io::Read, - num::NonZeroU64, path::{Path, PathBuf}, sync::{ Arc, Condvar, Mutex, MutexGuard, OnceLock, PoisonError, RwLock, @@ -230,6 +229,29 @@ impl CodeIndexBuildProgressSlotStateV1 { let _ = self.publish(&generation_id, self.owner_epoch, snapshot); } + /// A text refusal belongs to its pending owner. A caller without the + /// projection claim must also identify the progress it observed before + /// attempting admission, so a delayed refusal cannot overwrite new work. + pub(super) fn block_owned_text( + &mut self, + generation_id: &CodeGenerationId, + owner_epoch: u64, + reason: CodeIndexBuildBlockedReasonV1, + expected_progress_epoch: Option, + ) -> bool { + let Some(current) = self.snapshot() else { + return false; + }; + if current.phase == CodeIndexBuildPhaseV1::Ready + || expected_progress_epoch.is_some_and(|epoch| current.progress_epoch != epoch) + { + return false; + } + let mut snapshot = current.as_ref().clone(); + snapshot.blocked_reason = Some(reason); + self.publish(generation_id, owner_epoch, snapshot) + } + pub fn snapshot(&self) -> Option> { self.snapshot.as_ref().map(Arc::clone) } @@ -947,29 +969,6 @@ pub struct DaemonCodeTextArtifactStoreV1 { worktree_id: WorktreeId, } -/// The bytes one artifact admission reserves: its ceiling plus the band above -/// the admission watermark, so the reserve call enforces that watermark -/// against both the ledger and the measured resident set. The band is not -/// memory the artifact owns and is released right after the reserve; -/// retaining it in every overlapping build would charge the same -/// process-wide margin repeatedly. The process's live set is not charged: -/// the reserve already compares the request with the measured resident set, -/// which contains it. -pub(super) fn text_artifact_resident_memory_charge( - requested: NonZeroU64, - watermark_headroom: u64, -) -> Result { - requested - .get() - .checked_add(watermark_headroom) - .and_then(NonZeroU64::new) - .ok_or_else(|| { - RetrievalPortError::Contract( - "text-artifact resident-memory accounting overflowed".to_owned(), - ) - }) -} - pub(super) fn text_artifact_admitted_build_budget( preferred_bytes: u64, minimum_bytes: u64, @@ -1170,10 +1169,8 @@ impl DaemonCodeTextArtifactStoreV1 { } } - /// Reserve one artifact memory ceiling. The atomic reserve also includes - /// the process-wide high-watermark headroom, then releases that - /// check-only margin before returning while the component ceiling stays - /// charged. + /// Charge the artifact's ceiling below the process admission watermark. + /// Sizing and charging share one measured headroom reading. fn reserve_resident_memory( &self, generation_id: &CodeGenerationId, @@ -1184,32 +1181,6 @@ impl DaemonCodeTextArtifactStoreV1 { .map(|(reservation, _)| reservation) } - /// Measure headroom and size the build: `(observed, watermark headroom, - /// admitted)` bytes. - fn text_artifact_admission( - &self, - preferred: NonZeroU64, - minimum: NonZeroU64, - ) -> Result<(u64, u64), RetrievalPortError> { - let admission_watermark = self.resident_memory.admission_watermark_bytes(); - let headroom = self.resident_memory.headroom_below(admission_watermark); - let watermark_headroom = self - .resident_memory - .snapshot() - .limit_bytes - .saturating_sub(admission_watermark); - let admitted_bytes = text_artifact_admitted_build_budget( - preferred.get(), - minimum.get(), - headroom.available_bytes, - ) - .inspect_err(|_| { - self.resident_memory - .wait_for_headroom(minimum.get(), admission_watermark); - })?; - Ok((watermark_headroom, admitted_bytes)) - } - fn reserve_resident_memory_up_to( &self, generation_id: &CodeGenerationId, @@ -1235,57 +1206,59 @@ impl DaemonCodeTextArtifactStoreV1 { "text-artifact minimum resident-memory reservation must be nonzero".to_owned(), ) })?; - let (watermark_headroom, admitted_bytes) = - match self.text_artifact_admission(preferred, minimum) { - Err(RetrievalPortError::ResidentMemoryRefused(detail)) => { - let released = self - .resident_owners - .shed(minimum.get(), std::time::Instant::now()); - for release in &released { - log_resident_owner_release_v1(release); - } - // Freed pages sit in the pool workers' allocator heaps - // whether or not an owner was shed. - let trim = release_process_allocator_memory_v1(); - tracing::info!( - event = "code_text_artifact_build_shed_retained_state", - released_owners = released.len(), - trimmed_bytes = trim.released_bytes(), - refused = %detail, - "text-artifact build shed retained state before re-measuring headroom" - ); - self.text_artifact_admission(preferred, minimum) + let admission_watermark = self.resident_memory.admission_watermark_bytes(); + let reserve = || { + let reservation = self + .resident_memory + .reserve_up_to( + ResidentMemoryKeyV1 { + project_id: self.project_id.clone(), + worktree_id: self.worktree_id.clone(), + generation_id: generation_id.clone(), + component, + }, + preferred, + admission_watermark, + ) + .map_err(|error| { + RetrievalPortError::ResidentMemoryRefused(format!( + "text-artifact resident-memory admission was refused: {error}" + )) + })?; + let admitted = text_artifact_admitted_build_budget( + preferred.get(), + minimum.get(), + reservation.reserved_bytes(), + )?; + Ok((reservation, admitted)) + }; + let (reservation, admitted) = match reserve() { + Err(RetrievalPortError::ResidentMemoryRefused(detail)) => { + let released = self + .resident_owners + .shed(minimum.get(), std::time::Instant::now()); + for release in &released { + log_resident_owner_release_v1(release); } - admission => admission, - }?; - let admitted = NonZeroU64::new(admitted_bytes).ok_or_else(|| { - RetrievalPortError::Contract( - "text-artifact admitted resident-memory reservation must be nonzero".to_owned(), - ) - })?; - let accounted = text_artifact_resident_memory_charge(admitted, watermark_headroom)?; - let mut reservation = self - .resident_memory - .reserve( - ResidentMemoryKeyV1 { - project_id: self.project_id.clone(), - worktree_id: self.worktree_id.clone(), - generation_id: generation_id.clone(), - component, - }, - accounted, - ) - .map_err(|error| { - RetrievalPortError::ResidentMemoryRefused(format!( - "text-artifact resident-memory admission was refused: {error}" - )) - })?; - reservation.shrink_to(admitted.get()).map_err(|error| { - RetrievalPortError::Contract(format!( - "text-artifact resident-memory headroom release failed: {error}" - )) + let trim = release_process_allocator_memory_v1(); + tracing::info!( + event = "code_text_artifact_build_shed_retained_state", + released_owners = released.len(), + trimmed_bytes = trim.released_bytes(), + refused = %detail, + "text-artifact build shed retained state before re-measuring headroom" + ); + reserve() + } + result => result, + } + .inspect_err(|error| { + if matches!(error, RetrievalPortError::ResidentMemoryRefused(_)) { + self.resident_memory + .wait_for_headroom(minimum.get(), admission_watermark); + } })?; - let admitted = usize::try_from(admitted.get()).map_err(|error| { + let admitted = usize::try_from(admitted).map_err(|error| { RetrievalPortError::Contract(format!( "text-artifact admitted reservation exceeds the platform limit: {error}" )) @@ -2723,9 +2696,13 @@ impl LatestCodeTextGenerationV1 { // its phase before it re-attempts the work that was refused, so // clearing here erased the reason between two identical refusals // and status reported `blocked_reason: null` throughout a stall. - // A committed boundary is the honest clear -- it builds a fresh - // snapshot with no blocked reason -- and only that runs after work - // actually landed. + // A committed boundary clears the refusal. Ready is also a + // successful boundary: it is published only after the verified + // query owners are installed, even if finalization completed + // without another pending boundary in between. + if phase == CodeIndexBuildPhaseV1::Ready { + snapshot.blocked_reason = None; + } let _ = slot.publish(generation_id, self.text_progress_owner_epoch, snapshot); } }; @@ -2742,18 +2719,26 @@ impl LatestCodeTextGenerationV1 { fn publish_text_artifact_block(&self, error: &CodeLexicalArtifactErrorV1) { match error { CodeLexicalArtifactErrorV1::Unreserved(_) => { - self.publish_text_progress_blocked(CodeIndexBuildBlockedReasonV1::ResidentMemory); + self.publish_text_progress_blocked( + CodeIndexBuildBlockedReasonV1::ResidentMemory, + None, + ); } CodeLexicalArtifactErrorV1::Io(_) | CodeLexicalArtifactErrorV1::Missing(_) => { self.publish_text_progress_blocked( CodeIndexBuildBlockedReasonV1::ArtifactStoreUnavailable, + None, ); } _ => {} } } - fn publish_text_progress_blocked(&self, reason: CodeIndexBuildBlockedReasonV1) { + fn publish_text_progress_blocked( + &self, + reason: CodeIndexBuildBlockedReasonV1, + expected_progress_epoch: Option, + ) { let generation_id = &self.metadata.manifest().generation_id; { let _span = tracing::trace_span!("query.artifact.progress.publish").entered(); @@ -2762,12 +2747,12 @@ impl LatestCodeTextGenerationV1 { .text_progress_slot .write() .unwrap_or_else(std::sync::PoisonError::into_inner); - let Some(current) = slot.snapshot() else { - return; - }; - let mut snapshot = current.as_ref().clone(); - snapshot.blocked_reason = Some(reason); - let _ = slot.publish(generation_id, self.text_progress_owner_epoch, snapshot); + let _ = slot.block_owned_text( + generation_id, + self.text_progress_owner_epoch, + reason, + expected_progress_epoch, + ); } }; } @@ -2811,7 +2796,13 @@ impl LatestCodeTextGenerationV1 { maximum_work: usize, control: &dyn CodeIndexExecutionControlV1, ) -> Result { - let _workers = self + let progress_before_admission = self + .text_progress_slot + .read() + .unwrap_or_else(PoisonError::into_inner) + .snapshot() + .map(|snapshot| snapshot.progress_epoch); + let workers = self .text_artifact_store .worker_runtime .enter(&self.text_artifact_store.resident_memory) @@ -2824,8 +2815,23 @@ impl LatestCodeTextGenerationV1 { ), ) => RetrievalPortError::ResidentMemoryRefused(error.to_string()), other => RetrievalPortError::AuthorityUnavailable(other.to_string()), - })?; - let result = self.advance_text_serving_inner(maximum_work, control); + }); + let result = match workers { + Ok(_workers) => self.advance_text_serving_inner(maximum_work, control), + Err(error) => { + if matches!(&error, RetrievalPortError::ResidentMemoryRefused(_)) + && let Some(epoch) = progress_before_admission + { + // Worker admission owns no projection claim. Only the + // unchanged progress it observed can inherit this refusal. + self.publish_text_progress_blocked( + CodeIndexBuildBlockedReasonV1::ResidentMemory, + Some(epoch), + ); + } + Err(error) + } + }; if result.as_ref().is_err_and(|error| { matches!( error, @@ -3347,7 +3353,17 @@ impl LatestCodeTextGenerationV1 { let outcome = { let _span = tracing::trace_span!("query.artifact.head_open").entered(); self.open_published_head_or_begin_build(control) - }?; + } + .inspect_err(|error| { + if matches!(error, RetrievalPortError::ResidentMemoryRefused(_)) { + // Keep the claim until this attempt's refusal is visible; + // no later wake can publish progress ahead of this block. + self.publish_text_progress_blocked( + CodeIndexBuildBlockedReasonV1::ResidentMemory, + None, + ); + } + })?; match outcome { TextHeadOpenOutcomeV1::Served => return Ok(true), TextHeadOpenOutcomeV1::Build(initialized) => { @@ -3638,7 +3654,15 @@ impl LatestCodeTextGenerationV1 { &content_key, build_reservation, control, - )?; + ) + .inspect_err(|error| { + if matches!(error, RetrievalPortError::ResidentMemoryRefused(_)) { + self.publish_text_progress_blocked( + CodeIndexBuildBlockedReasonV1::ResidentMemory, + None, + ); + } + })?; Ok(true) } diff --git a/crates/tracedecay-code-index-runtime/tests/code_index_scheduler/serving.rs b/crates/tracedecay-code-index-runtime/tests/code_index_scheduler/serving.rs index 05bf0e812e..e5c882effc 100644 --- a/crates/tracedecay-code-index-runtime/tests/code_index_scheduler/serving.rs +++ b/crates/tracedecay-code-index-runtime/tests/code_index_scheduler/serving.rs @@ -3,7 +3,11 @@ use std::{ fmt::Write as _, num::NonZeroU64, path::{Path, PathBuf}, - sync::{Arc, mpsc}, + sync::{ + Arc, Mutex, RwLock, + atomic::{AtomicBool, Ordering}, + mpsc, + }, thread, time::{Duration, Instant}, }; @@ -20,6 +24,7 @@ use tracedecay_contracts::{ PageRequest, PhraseSearchRequest, QualifiedNameRequest, ResolvedScope, ResultProjection, RetrievalOrder, RetrievalPortContext, RetrievalPortOutcome, RetrievalRequestMeta, SourceMetadataRequest, callable_code_operation, + code_index_freshness::{CodeIndexBuildBlockedReasonV1, CodeIndexBuildPhaseV1}, retrieval::{ CodeFacetDimension, CodeFacetRequest, CodeHierarchyRequest, CodeImpactRequest, CodeImplementationsRequest, CodeNavigationRequest, CodeTimelineRequest, @@ -49,8 +54,8 @@ use tracedecay_query::retrieval::{ }; use tracedecay_runtime_core::resident_memory::{ DEFAULT_PROCESS_RESIDENT_MEMORY_LIMIT_V1, ProcessResidentMemoryV1, ProcessResidentSampleV1, - RESIDENT_MEMORY_PRESSURE_HIGH_WATERMARK_PERMILLE_V1, ResidentMemoryPressureV1, - resident_memory_watermark_bytes_v1, + RESIDENT_MEMORY_PRESSURE_HIGH_WATERMARK_PERMILLE_V1, ResidentMemoryComponentIdV1, + ResidentMemoryPressureV1, resident_memory_watermark_bytes_v1, }; use tracedecay_session_temporal_store::SessionTemporalAccess; @@ -2086,42 +2091,260 @@ fn reader_reservation_refusal_precedes_missing_artifact_access() { } #[test] -fn overlapping_text_builds_share_one_admission_watermark_headroom() { - let limit_bytes = 1_000_u64; - let watermark_headroom = 100_u64; - let requested = NonZeroU64::new(300).expect("nonzero build request"); - let mut used_bytes = 0_u64; - - for _ in 0..3 { - let accounted = - super::super::text_artifact_resident_memory_charge(requested, watermark_headroom) - .expect("bounded admission accounting"); - assert!( - used_bytes + accounted.get() <= limit_bytes, - "each overlapping build fits beneath the same 900-byte high watermark" +fn retained_text_admission_reports_its_block_and_recovers_after_headroom_release() { + let fixture = GitFixture::new(&[("src/lib.rs", "pub fn reserved_until_release() {}\n")]); + let store = TempDir::new().expect("store root"); + let limit = NonZeroU64::new(4 * 1024 * 1024 * 1024).expect("test process limit"); + let pressure = Arc::new(ResidentMemoryPressureV1::with_sampler( + limit, + Arc::new(|| { + Some(ProcessResidentSampleV1 { + resident_bytes: 0, + unreclaimable_bytes: 0, + swapped_bytes: 0, + cgroup_headroom_bytes: None, + }) + }), + )); + let memory = Arc::new(ProcessResidentMemoryV1::with_pressure( + limit, + Arc::clone(&pressure), + )); + { + let mut scheduler = scheduler( + &fixture, + store.path().to_path_buf(), + Arc::new(SharedCodeIndexBytePoolV1::default()), ); - used_bytes += requested.get(); + scheduler.bind_resident_memory(Arc::clone(&memory)); + published(scheduler.reconcile_now().expect("seal source generation")); } + let mut scheduler = scheduler( + &fixture, + store.path().to_path_buf(), + Arc::new(SharedCodeIndexBytePoolV1::default()), + ); + scheduler.bind_resident_memory(Arc::clone(&memory)); + let text = scheduler + .servable_retained_text_generation() + .expect("publication store") + .expect("retained sealed text generation"); + let scanned = build_progress_snapshot(&scheduler); + assert_eq!(scanned.phase, CodeIndexBuildPhaseV1::SourceScan); + assert_eq!(scanned.blocked_reason, None); + let build_floor = u64::try_from(CODE_LEXICAL_ARTIFACT_BUILD_MEMORY_BUDGET_BYTES_V1) + .expect("build floor fits u64"); + let held_bytes = memory.admission_watermark_bytes() - build_floor + 1; + let competing = memory + .reserve_process_shared( + ResidentMemoryComponentIdV1::new("held-text-headroom").expect("component"), + NonZeroU64::new(held_bytes).expect("competing reservation"), + ) + .expect("hold actual ledger headroom"); + let mut headroom = pressure.subscribe_headroom(); - assert_eq!( - used_bytes, 900, - "the retained ledger owns the three build ceilings, not three margins" + for claimed_path in [true, false] { + let refused = if claimed_path { + text.advance_artifact_text_serving(1, &text.text_execution_control()) + } else { + text.advance_text_serving(1) + }; + assert!(matches!( + refused, + Err(RetrievalPortError::ResidentMemoryRefused(_)) + )); + let blocked = build_progress_snapshot(&scheduler); + assert_eq!(blocked.generation_id, scanned.generation_id); + assert_eq!(blocked.phase, CodeIndexBuildPhaseV1::SourceScan); + assert_eq!(blocked.committed_pages, 0); + assert_eq!( + blocked.blocked_reason, + Some(CodeIndexBuildBlockedReasonV1::ResidentMemory) + ); + assert!(text.text_projection_needs_work()); + assert_eq!(memory.snapshot().used_bytes, held_bytes); + } + + drop(headroom.borrow_and_update()); + drop(competing); + assert!(headroom.has_changed()); + assert_eq!(memory.snapshot().used_bytes, 0); + text.production_query_owners() + .expect("same owner builds after headroom is released"); + let ready = build_progress_snapshot(&scheduler); + assert_eq!(ready.generation_id, scanned.generation_id); + assert_eq!(ready.phase, CodeIndexBuildPhaseV1::Ready); + assert!(ready.committed_pages > 0); + assert_eq!(ready.blocked_reason, None); + assert!(text.query_owners_are_ready()); + assert!(active_text_artifact_path(store.path()).is_file()); +} + +#[test] +fn ready_publication_clears_a_finalization_refusal_without_another_pending_boundary() { + let fixture = GitFixture::new(&[("src/lib.rs", "pub fn ready_after_refusal() {}\n")]); + let store = TempDir::new().expect("store root"); + let limit = NonZeroU64::new(4 * 1024 * 1024 * 1024).expect("test process limit"); + let pressure = Arc::new(ResidentMemoryPressureV1::with_sampler( + limit, + Arc::new(|| { + Some(ProcessResidentSampleV1 { + resident_bytes: 0, + unreclaimable_bytes: 0, + swapped_bytes: 0, + cgroup_headroom_bytes: None, + }) + }), + )); + let mut scheduler = scheduler( + &fixture, + store.path().to_path_buf(), + Arc::new(SharedCodeIndexBytePoolV1::default()), ); - for overflow in [ - super::super::text_artifact_resident_memory_charge( - NonZeroU64::new(u64::MAX).expect("maximum nonzero request"), - 1, - ), - super::super::text_artifact_resident_memory_charge(requested, u64::MAX), - ] { + scheduler.bind_resident_memory(Arc::new(ProcessResidentMemoryV1::with_pressure( + limit, pressure, + ))); + published(scheduler.reconcile_now().expect("seal source generation")); + let _workers = scheduler + .ensure_worker_plan() + .expect("fixture worker runtime"); + let text = scheduler.latest_complete().expect("text generation"); + let control = text.text_execution_control(); + assert!( + !text + .advance_text_serving(0) + .expect("initialize pending text progress") + ); + let finalization_deadline = Instant::now() + SERVING_SEAT_FAILURE_CEILING; + while !matches!( + build_progress_snapshot(&scheduler).phase, + CodeIndexBuildPhaseV1::IndexBuild | CodeIndexBuildPhaseV1::Verification + ) { assert!( - matches!( - overflow, - Err(tracedecay_query::retrieval::RetrievalPortError::Contract(_)) - ), - "overflow must remain a typed contract refusal: {overflow:?}" + Instant::now() < finalization_deadline, + "text build did not reach finalization" + ); + assert!( + !text.advance_text_serving(1).expect("begin finalization"), + "the bounded build must expose a pending finalization boundary" ); } + { + let mut slot = text.text_projection_build.lock_slot(); + let super::super::CodeTextProjectionSlotV1::Building(build) = &mut *slot else { + panic!("resumable text builder"); + }; + while !matches!( + build + .builder + .advance_started_finalization(1_000, &control) + .expect("finish the durable artifact before publication"), + CodeLexicalArtifactFinalizationStepV1::Ready(_) + ) { + assert!( + Instant::now() < finalization_deadline, + "durable finalization did not finish" + ); + } + } + text.publish_text_progress_phase(CodeIndexBuildPhaseV1::Verification, 0, 0); + let before = build_progress_snapshot(&scheduler); + let progress = scheduler.build_progress_slot(); + assert!( + progress + .write() + .expect("progress authority") + .block_owned_text( + &text.metadata().manifest().generation_id, + text.text_progress_owner_epoch, + CodeIndexBuildBlockedReasonV1::ResidentMemory, + Some(before.progress_epoch), + ) + ); + + // A resumed finalization can go directly from its prior refusal to + // installing query owners. No intervening Pending snapshot clears it. + assert!( + text.advance_artifact_text_serving(1, &control) + .expect("publish the finalized artifact") + ); + let ready = build_progress_snapshot(&scheduler); + assert!(text.query_owners_are_ready()); + assert_eq!(ready.phase, CodeIndexBuildPhaseV1::Ready); + assert_eq!(ready.blocked_reason, None); + assert_eq!(ready.committed_pages, before.committed_pages); +} + +#[test] +fn concurrent_text_admission_refusal_cannot_block_a_ready_owner() { + let fixture = GitFixture::new(&[("src/lib.rs", "pub fn concurrent_text_owner() {}\n")]); + let store = TempDir::new().expect("store root"); + let limit = NonZeroU64::new(4 * 1024 * 1024 * 1024).expect("process limit"); + let (sampling, sampled) = mpsc::channel(); + let (resume, resumed) = mpsc::channel(); + let resumed = Mutex::new(resumed); + let paused = AtomicBool::new(false); + let pressure = Arc::new(ResidentMemoryPressureV1::with_sampler( + limit, + Arc::new(move || { + let refused = thread::current().name() == Some("refused-text-advance"); + if refused && !paused.swap(true, Ordering::AcqRel) { + sampling.send(()).expect("announce refused sample"); + resumed + .lock() + .expect("sample resume receiver") + .recv_timeout(SERVING_SEAT_FAILURE_CEILING) + .expect("resume refused sample"); + } + Some(ProcessResidentSampleV1 { + resident_bytes: 0, + unreclaimable_bytes: 0, + swapped_bytes: 0, + cgroup_headroom_bytes: refused.then_some(0), + }) + }), + )); + let memory = Arc::new(ProcessResidentMemoryV1::with_pressure(limit, pressure)); + let mut scheduler = scheduler( + &fixture, + store.path().to_path_buf(), + Arc::new(SharedCodeIndexBytePoolV1::default()), + ); + scheduler.bind_resident_memory(memory); + published(scheduler.reconcile_now().expect("seal source generation")); + let text = scheduler.latest_complete().expect("text generation"); + thread::scope(|scope| { + let refused_text = text.clone(); + let refused = thread::Builder::new() + .name("refused-text-advance".to_owned()) + .spawn_scoped(scope, move || refused_text.advance_text_serving(1)) + .expect("spawn refused advance"); + sampled + .recv_timeout(SERVING_SEAT_FAILURE_CEILING) + .expect("first advance reaches memory admission"); + let ready_text = text.clone(); + let (starting, started) = mpsc::channel(); + let ready = scope.spawn(move || { + starting.send(()).expect("announce concurrent advance"); + ready_text.production_query_owners() + }); + started + .recv_timeout(SERVING_SEAT_FAILURE_CEILING) + .expect("second advance starts while the first is admitted"); + resume.send(()).expect("release refused sample"); + assert!(matches!( + refused.join().expect("refused advance joins"), + Err(RetrievalPortError::ResidentMemoryRefused(_)) + )); + ready + .join() + .expect("ready advance joins") + .expect("concurrent owner serves after the refusal"); + }); + let ready = build_progress_snapshot(&scheduler); + assert!(text.query_owners_are_ready()); + assert_eq!(ready.phase, CodeIndexBuildPhaseV1::Ready); + assert_eq!(ready.blocked_reason, None); } #[test] @@ -2758,6 +2981,98 @@ fn progress_owner_epoch_rejects_original_a_after_a_b_a_replacement() { after_stale_original.progress_epoch, replacement_a.progress_epoch, "matching generation identity must not let the original A owner replace the newer A epoch" ); + assert!(!slot.block_owned_text( + &generation_a, + original_a_owner, + CodeIndexBuildBlockedReasonV1::ResidentMemory, + Some(after_stale_original.progress_epoch), + )); + assert_eq!( + slot.snapshot().expect("replacement owner").blocked_reason, + None + ); +} + +#[test] +fn delayed_memory_refusal_cannot_replace_newer_pending_or_ready_progress() { + for phase in [ + CodeIndexBuildPhaseV1::BulkCommit, + CodeIndexBuildPhaseV1::Ready, + ] { + let generation = + CodeGenerationId::new("generation.progress-refusal").expect("generation identity"); + let mut state = super::super::CodeIndexBuildProgressSlotStateV1::default(); + let owner = state.replace_generation(generation.clone()); + assert!(state.publish( + &generation, + owner, + progress_snapshot_for_generation(&generation, 1), + )); + let slot = Arc::new(RwLock::new(state)); + let (observed_tx, observed_rx) = mpsc::sync_channel(1); + let (resume_tx, resume_rx) = mpsc::sync_channel(1); + let delayed_slot = Arc::clone(&slot); + let delayed_generation = generation.clone(); + let delayed = thread::spawn(move || { + let observed_epoch = delayed_slot + .read() + .expect("observe pending progress") + .snapshot() + .expect("pending snapshot") + .progress_epoch; + observed_tx.send(()).expect("admission began"); + resume_rx + .recv_timeout(SERVING_SEAT_FAILURE_CEILING) + .expect("admission refused after concurrent progress"); + delayed_slot + .write() + .expect("publish refusal") + .block_owned_text( + &delayed_generation, + owner, + CodeIndexBuildBlockedReasonV1::ResidentMemory, + Some(observed_epoch), + ) + }); + observed_rx + .recv_timeout(SERVING_SEAT_FAILURE_CEILING) + .expect("worker observed its admission epoch"); + let newer = { + let mut current = slot.write().expect("publish concurrent progress"); + let mut next = progress_snapshot_for_generation(&generation, 2); + next.phase = phase; + assert!(current.publish(&generation, owner, next)); + current.snapshot().expect("newer progress") + }; + resume_tx.send(()).expect("release delayed refusal"); + assert!(!delayed.join().expect("delayed worker")); + + let mut current = slot.write().expect("read surviving progress"); + let after = current.snapshot().expect("current progress"); + assert_eq!(after.progress_epoch, newer.progress_epoch); + assert_eq!(after.phase, phase); + assert_eq!(after.committed_pages, 2); + assert_eq!(after.blocked_reason, None); + let accepts_current_refusal = current.block_owned_text( + &generation, + owner, + CodeIndexBuildBlockedReasonV1::ResidentMemory, + Some(after.progress_epoch), + ); + assert_eq!( + accepts_current_refusal, + phase != CodeIndexBuildPhaseV1::Ready + ); + assert_eq!( + current + .snapshot() + .expect("current refusal verdict") + .blocked_reason, + (phase != CodeIndexBuildPhaseV1::Ready) + .then_some(CodeIndexBuildBlockedReasonV1::ResidentMemory), + "an unchanged pending owner must still report its genuine wait" + ); + } } #[test] diff --git a/crates/tracedecay-runtime-core/src/resident_memory.rs b/crates/tracedecay-runtime-core/src/resident_memory.rs index 34a9c445be..a16e42ca7d 100644 --- a/crates/tracedecay-runtime-core/src/resident_memory.rs +++ b/crates/tracedecay-runtime-core/src/resident_memory.rs @@ -1531,6 +1531,62 @@ impl ProcessResidentMemoryV1 { Err(self.refused(self.admission_failure(requested_bytes))) } + /// Size and charge new work from the same measured headroom reading. + /// The returned ceiling can be smaller than `preferred_bytes`; consumers + /// with a minimum working set must release a reservation below that floor. + /// Concurrent ledger charges are checked under the charge's own lock. + /// The consumer registers its working-set minimum with `wait_for_headroom` + /// when the returned budget cannot support its next operation. + #[tracing::instrument( + name = "runtime_core.resident.reserve_up_to", + level = "trace", + skip_all + )] + pub fn reserve_up_to( + self: &Arc, + key: ResidentMemoryKeyV1, + preferred_bytes: NonZeroU64, + ceiling: u64, + ) -> Result { + self.pressure.measure_admission_bytes(); + let ceiling = ceiling.min(self.limit_bytes.get()); + let mut state = self.lock_state(); + let headroom = self.headroom_from_state(&state, ceiling); + let admitted = preferred_bytes + .get() + .min(headroom.available_bytes) + .min(ceiling.saturating_sub(state.used_bytes)); + if admitted == 0 { + let failure = if self.pressure.state().is_over_budget() + || headroom.observed_bytes >= ceiling.min(self.pressure.limit_bytes()) + { + ResidentMemoryAdmissionFailureV1::ObservedOverBudget { + observed_bytes: headroom.observed_bytes, + limit_bytes: self.pressure.limit_bytes(), + high_watermark_bytes: self.pressure.high_watermark_bytes(), + requested_bytes: preferred_bytes.get(), + floor_bytes: RESIDENT_MEMORY_PRESSURE_ADMISSION_FLOOR_BYTES_V1, + } + } else { + ResidentMemoryAdmissionFailureV1::ReservationCeiling { + used_bytes: state.used_bytes, + requested_bytes: preferred_bytes.get(), + limit_bytes: ceiling, + } + }; + drop(state); + return Err(failure); + } + // `admitted` is at most ceiling - used_bytes, so both ledger sums fit. + state.used_bytes += admitted; + *state.charges.entry(key.clone()).or_default() += admitted; + Ok(ResidentMemoryReservationV1 { + authority: Arc::clone(self), + key, + reserved_bytes: admitted, + }) + } + /// Reserves one process-shared component without fabricating a project, /// worktree, or code-generation owner. These reservations use the same /// process ceiling and RAII release authority as project generations. @@ -1596,12 +1652,20 @@ impl ProcessResidentMemoryV1 { /// [`Self::reserve`] holds them. pub fn headroom_below(&self, ceiling: u64) -> ResidentMemoryHeadroomV1 { self.pressure.measure_admission_bytes(); + self.headroom_from_state(&self.lock_state(), ceiling) + } + + fn headroom_from_state( + &self, + state: &ResidentMemoryStateV1, + ceiling: u64, + ) -> ResidentMemoryHeadroomV1 { let measured_ceiling = ceiling.min(self.pressure.limit_bytes()); let observed_bytes = self .pressure .observed_bytes_for_limit(measured_ceiling) .unwrap_or(0); - let used_bytes = self.lock_state().used_bytes; + let used_bytes = state.used_bytes; let available_bytes = if self.pressure.state().is_over_budget() { 0 } else { diff --git a/crates/tracedecay-runtime-core/src/resident_memory/tests.rs b/crates/tracedecay-runtime-core/src/resident_memory/tests.rs index 557293b4ec..10fe13f033 100644 --- a/crates/tracedecay-runtime-core/src/resident_memory/tests.rs +++ b/crates/tracedecay-runtime-core/src/resident_memory/tests.rs @@ -1335,6 +1335,35 @@ fn cgroup_headroom_respects_a_larger_but_fuller_ancestor() { ); } +#[test] +fn sized_reservations_charge_only_the_measured_cgroup_headroom() { + const MIB: u64 = 1024 * 1024; + let limit = bytes(800 * MIB); + let pressure = Arc::new(ResidentMemoryPressureV1::with_sampler( + limit, + Arc::new(|| { + Some(ProcessResidentSampleV1 { + resident_bytes: 100 * MIB, + unreclaimable_bytes: 100 * MIB, + swapped_bytes: 0, + cgroup_headroom_bytes: Some(200 * MIB), + }) + }), + )); + let authority = Arc::new(ProcessResidentMemoryV1::with_pressure(limit, pressure)); + let watermark = authority.admission_watermark_bytes(); + let headroom = authority.headroom_below(watermark); + assert_eq!(headroom.available_bytes, 200 * MIB); + let owner = key("project-a", "worktree-a", "generation-a", "text-build"); + let held = authority + .reserve_up_to(owner, bytes(headroom.available_bytes + 1), watermark) + .expect("the actual allocation fits without charging the 80 MiB watermark band"); + assert_eq!(held.reserved_bytes(), 200 * MIB); + assert_eq!(authority.snapshot().used_bytes, 200 * MIB); + drop(held); + assert_eq!(authority.snapshot().used_bytes, 0); +} + #[test] fn narrowed_budgets_and_waiters_share_cgroup_capacity_without_charging_peers() { const MIB: u64 = 1024 * 1024; @@ -1619,3 +1648,101 @@ fn allocator_reclaim_preserves_cgroup_pressure_until_sample_recovers() { .is_ok() ); } + +#[test] +fn sized_admission_remeasures_and_charges_the_supported_smaller_budget() { + const MIB: u64 = 1024 * 1024; + let measured = Arc::new(AtomicU64::new(300 * MIB)); + let sampled = Arc::clone(&measured); + let reads = Arc::new(AtomicU64::new(0)); + let sampled_reads = Arc::clone(&reads); + let pressure = Arc::new(ResidentMemoryPressureV1::with_sampler( + bytes(1000 * MIB), + Arc::new(move || { + sampled_reads.fetch_add(1, Ordering::SeqCst); + let observed = sampled.load(Ordering::SeqCst); + Some(ProcessResidentSampleV1 { + resident_bytes: observed, + unreclaimable_bytes: observed, + swapped_bytes: 0, + cgroup_headroom_bytes: None, + }) + }), + )); + let authority = Arc::new(ProcessResidentMemoryV1::with_pressure( + bytes(1000 * MIB), + Arc::clone(&pressure), + )); + let watermark = authority.admission_watermark_bytes(); + assert_eq!(watermark, 900 * MIB); + assert_eq!( + authority.headroom_below(watermark).available_bytes, + 600 * MIB + ); + measured.store(700 * MIB, Ordering::SeqCst); + let before = reads.load(Ordering::SeqCst); + let reservation = authority + .reserve_up_to( + key("project-a", "worktree-a", "generation-a", "text-build"), + bytes(600 * MIB), + watermark, + ) + .expect("the smaller build fits despite growth since planning"); + assert_eq!(reads.load(Ordering::SeqCst), before + 1); + assert_eq!(reservation.reserved_bytes(), 200 * MIB); + assert_eq!(authority.snapshot().used_bytes, 200 * MIB); + drop(reservation); + assert_eq!(authority.snapshot().used_bytes, 0); + + measured.store(950 * MIB, Ordering::SeqCst); + let refused = authority + .reserve_up_to( + key("project-a", "worktree-a", "generation-b", "text-build"), + bytes(600 * MIB), + watermark, + ) + .expect_err("sizing must still refuse measured pressure"); + assert!(refused.is_observed_over_budget()); + assert_eq!(authority.snapshot().used_bytes, 0); +} + +#[test] +fn sized_admissions_share_the_watermark_without_charging_its_margin() { + let pressure = Arc::new(ResidentMemoryPressureV1::with_sampler( + bytes(1000), + Arc::new(|| None), + )); + let authority = Arc::new(ProcessResidentMemoryV1::with_pressure( + bytes(1000), + pressure, + )); + let watermark = authority.admission_watermark_bytes(); + let owner = key("project-a", "worktree-a", "generation-a", "text-build"); + let mut reservations = (0..3) + .map(|_| { + authority + .reserve_up_to(owner.clone(), bytes(300), watermark) + .unwrap() + }) + .collect::>(); + assert_eq!(authority.snapshot().used_bytes, 900); + let refused = authority + .reserve_up_to(owner.clone(), bytes(300), watermark) + .unwrap_err(); + assert_eq!( + refused, + ResidentMemoryAdmissionFailureV1::ReservationCeiling { + used_bytes: 900, + requested_bytes: 300, + limit_bytes: 900, + } + ); + drop(reservations.pop()); + let maximum = authority + .reserve_up_to(owner, bytes(u64::MAX), watermark) + .expect("bounded sizing cannot overflow while reserving available bytes"); + assert_eq!(maximum.reserved_bytes(), 300); + assert_eq!(authority.snapshot().used_bytes, 900); + drop((reservations, maximum)); + assert_eq!(authority.snapshot().used_bytes, 0); +} diff --git a/crates/tracedecay/src/daemon/core_hooks.rs b/crates/tracedecay/src/daemon/core_hooks.rs index b1994fe82a..09cbabe674 100644 --- a/crates/tracedecay/src/daemon/core_hooks.rs +++ b/crates/tracedecay/src/daemon/core_hooks.rs @@ -16,9 +16,11 @@ use tracedecay_daemon_identity::{ ResolvedDaemonConnection, client_connection, current_daemon_connection, }; use tracedecay_daemon_protocol::{DaemonClientStream, SOCKET_ENV}; +use tracedecay_domain::canonical_text::sha256_hex; use tracedecay_domain::errors::{Result, TraceDecayError}; use tracedecay_mcp::hook_events::{self, HookEventKind, HookEventPlan}; use tracedecay_mcp::server::attach_stateless_request_context; +use tracedecay_runtime_core::logging::log_daemon_event; use super::{JsonRpcRequest, write_daemon_preamble}; @@ -145,7 +147,7 @@ async fn admit_registered_file_edit( "file edit could not be encoded for durable admission", ) })?; - broker + let admitted = broker .admit(&event.admission_source(), &payload) .await .map_err(|outcome| { @@ -155,6 +157,21 @@ async fn admit_registered_file_edit( "file edit was not durably admitted", ) })?; + // The host may have exhausted its acknowledgement budget by now. Retain + // operator evidence of the exact durable receipt independently of the + // response write and the later project/binding publication. Match the path + // set by digest without putting edited filenames in persistent logs. + log_daemon_event( + "hook_file_edit_admitted", + &[ + ("project", root.display().to_string()), + ( + "paths_digest", + sha256_hex(format!("{:?}", event.rel_paths).as_bytes()), + ), + ("seq", admitted.seq.to_string()), + ], + ); routed.project_path = Some(root); Ok(routed) } diff --git a/crates/tracedecay/tests/hooks_lsp_suite/hook_spool_drain_test.rs b/crates/tracedecay/tests/hooks_lsp_suite/hook_spool_drain_test.rs index 8fb58a6adb..668cf7fe58 100644 --- a/crates/tracedecay/tests/hooks_lsp_suite/hook_spool_drain_test.rs +++ b/crates/tracedecay/tests/hooks_lsp_suite/hook_spool_drain_test.rs @@ -11,10 +11,13 @@ use std::time::{Duration, Instant}; use serde_json::json; use tracedecay_domain::NativeHostIdentityV1; +use tracedecay_domain::canonical_text::sha256_hex; use tracedecay_hooks::{ HOOK_SYNCHRONOUS_BUDGET, HookDeliveryReceiptWriterV1, HookDeliveryRetentionV1, HookDeliverySourceReceiptV1, hook_delivery_receipt_spool_root, }; +use tracedecay_runtime_core::logging::format_daemon_log_line; +use tracedecay_runtime_core::path_safety::canonical_existing_identity; use tracedecay_runtime_core::storage::default_profile_sharded_layout; use crate::common::{ @@ -508,6 +511,23 @@ fn bound_worktrees(data_root: &Path) -> usize { std::fs::read_dir(data_root.join("hook-configurations")).map_or(0, Iterator::count) } +fn admitted_edit_sequence(log: &str, root: &Path) -> Option { + let marker = format_daemon_log_line( + "hook_file_edit_admitted", + &[ + ("project", root.display().to_string()), + ( + "paths_digest", + sha256_hex(format!("{:?}", ["src/lib.rs"]).as_bytes()), + ), + ], + ); + let prefix = format!("{marker} seq="); + log.lines() + .filter_map(|line| line.strip_prefix(prefix.as_str())?.parse::().ok()) + .find(|sequence| *sequence > 0) +} + #[test] fn an_edit_in_a_never_opened_linked_worktree_binds_it_and_reaches_the_daemon() { let home = tempfile::TempDir::new().unwrap(); @@ -532,6 +552,8 @@ fn an_edit_in_a_never_opened_linked_worktree_binds_it_and_reaches_the_daemon() { ); std::fs::write(root.join("src/lib.rs"), "pub fn edited_in_worktree() {}\n").unwrap(); } + let canonical_roots = + [&worktree, &other_worktree].map(|root| canonical_existing_identity(root).unwrap()); assert_eq!( bound_worktrees(&data_root), 1, @@ -546,10 +568,13 @@ fn an_edit_in_a_never_opened_linked_worktree_binds_it_and_reaches_the_daemon() { let _daemon = spawn_tracedecay_daemon_logged(&home, &log, |command| { command.env("TRACEDECAY_TEST_HOLD_AFTER_CORE_PUBLISH", &hold); }); - assert_eq!( - claude_edit(&home, &worktree, &data_root, "toolu_unbound_first"), - "hook_v2_accepted", - "the first edit in an unbound worktree must reach the daemon: {}\n{}", + let first = claude_edit(&home, &worktree, &data_root, "toolu_unbound_first"); + assert!( + matches!( + first.as_str(), + "hook_v2_accepted" | "hook_v2_catchup_required" + ), + "the unbound callback must report its acknowledgement outcome, got {first}: {}\n{}", std::fs::read_to_string(&log).unwrap(), std::fs::read_to_string(data_root.join("hook_analytics.jsonl")).unwrap() ); @@ -560,22 +585,74 @@ fn an_edit_in_a_never_opened_linked_worktree_binds_it_and_reaches_the_daemon() { ); assert!( hold.is_file(), - "the first edit was acknowledged while full publication is held" + "the first edit reached core publication while full publication is held" ); - assert_eq!( - claude_edit( - &home, - &other_worktree, - &data_root, - "toolu_other_unbound_first" + let completion_deadline = Instant::now() + PROMPT_DRAIN; + let second = claude_edit( + &home, + &other_worktree, + &data_root, + "toolu_other_unbound_first", + ); + assert!( + matches!( + second.as_str(), + "hook_v2_accepted" | "hook_v2_catchup_required" ), - "hook_v2_accepted", - "another linked checkout's same relative file must commit while full publication is held" + "the second unbound callback must report its acknowledgement outcome, got {second}" + ); + // A callback's fixed acknowledgement budget can expire while the daemon + // still commits its already-received event. Do not resubmit either edit: + // require its exact root/path receipt from the canonical broker before + // releasing full publication. Bindings alone cannot prove admission. + assert!( + latency_until( + completion_deadline.saturating_duration_since(Instant::now()), + || { + let admitted = std::fs::read_to_string(&log).unwrap(); + canonical_roots + .iter() + .all(|root| admitted_edit_sequence(&admitted, root).is_some()) + }, + ) + .is_some(), + "both first edits must have exact durable receipts before releasing publication (first={first}, second={second}): {}", + std::fs::read_to_string(&log).unwrap(), ); - // Graph admission publishes hook bindings before the core hold. Full - // publication is the capability boundary this fixture actually holds. - let full_markers = [&worktree, &other_worktree] - .map(|root| format!("project={} phase=full_published", root.display())); + let admitted = std::fs::read_to_string(&log).unwrap(); + for line in admitted + .lines() + .filter(|line| line.contains("event=hook_file_edit_admitted")) + { + assert!( + !line.contains("src/lib.rs"), + "admission evidence must not expose edited filenames: {line}" + ); + } + let sequences = canonical_roots + .each_ref() + .map(|root| admitted_edit_sequence(&admitted, root).unwrap()); + assert_ne!( + sequences[0], sequences[1], + "each linked-worktree edit must have its own committed broker record" + ); + // The first open retains process-wide capacity admission through full + // publication. The other edit must commit its durable receipt above while + // waiting for that admission; its binding follows once the hold releases. + assert_eq!( + bound_worktrees(&data_root), + 2, + "only the main checkout and the first core-published worktree are bound while publication is held" + ); + let full_markers = canonical_roots.each_ref().map(|root| { + format_daemon_log_line( + "project_open_phase", + &[ + ("project", root.display().to_string()), + ("phase", "full_published".to_owned()), + ], + ) + }); let held_log = std::fs::read_to_string(&log).unwrap(); for marker in &full_markers { assert!( @@ -585,11 +662,14 @@ fn an_edit_in_a_never_opened_linked_worktree_binds_it_and_reaches_the_daemon() { } std::fs::remove_file(&hold).unwrap(); assert!( - latency_until(PROMPT_DRAIN, || { - let published = std::fs::read_to_string(&log).unwrap(); - bound_worktrees(&data_root) == 3 - && full_markers.iter().all(|marker| published.contains(marker)) - }) + latency_until( + completion_deadline.saturating_duration_since(Instant::now()), + || { + let published = std::fs::read_to_string(&log).unwrap(); + bound_worktrees(&data_root) == 3 + && full_markers.iter().all(|marker| published.contains(marker)) + } + ) .is_some(), "delivering that edit opens the worktree and publishes its binding: {}", std::fs::read_to_string(&log).unwrap() diff --git a/evals/agent_adoption/BUILD.bazel b/evals/agent_adoption/BUILD.bazel index d223ca97a4..f9f95ae7e9 100644 --- a/evals/agent_adoption/BUILD.bazel +++ b/evals/agent_adoption/BUILD.bazel @@ -6,9 +6,11 @@ py_test( srcs = [ "carry_claude_profile.py", "grade.py", + "select_scenarios.py", "selftest.py", + "stock_check.py", ], - data = glob([ + data = ["claude_tools.sh"] + glob([ "learned_skills/**", "scenarios/*.json", ]), diff --git a/evals/agent_adoption/README.md b/evals/agent_adoption/README.md index cd1697c3fa..f37d7998c1 100644 --- a/evals/agent_adoption/README.md +++ b/evals/agent_adoption/README.md @@ -77,23 +77,127 @@ paths, so do **not** commit run artifacts. |-----|---------|---------| | `TRACEDECAY_AGENT_EVALS` | unset | must be `1` to launch agents (else dry run) | | `HOSTS` | `claude` | `claude`, `codex`, or `claude codex` | -| `CHANNELS` | `full` | ablation conditions: any of `full no-hints no-skills bare cli-only` | +| `CHANNELS` | `full` | conditions: `native bare full no-hints no-skills cli-only` | | `SCENARIOS` | all active | space-separated scenario ids to run | | `EVAL_INCLUDE_DEFERRED` | `0` | also run `status:"deferred"` scenarios | +| `EVAL_INCLUDE_BASELINE` | `0` | include held-out product-effectiveness scenarios | | `CLAUDE_MODELS` | `opus sonnet` | space-separated `claude --model` matrix | | `CODEX_MODELS` | `gpt-5.6-sol` | space-separated `codex exec -m` matrix | | `CODEX_REASONING_EFFORT` | `medium` | Codex reasoning effort passed through `model_reasoning_effort` | | `SCENARIO_TIMEOUT` | `240` | per scenario wall-clock seconds | -| `REPS` | `1` | repetitions per scenario x host x model x condition cell; `>1` suffixes transcripts with `__r` and the grader pools the repetitions, so a cell of n=15 gives a usable exact-binomial interval instead of a single draw | +| `REPS` | `1` | repetitions per scenario x host x model x condition cell; repeated outcomes are reported with Wilson intervals and observed cost/latency ranges | | `PARALLEL` | `1` | concurrent live agent runs; scenarios are read-only against shared fixtures and each `claude` process spawns its own MCP server, so `4` is a safe default for high-rep cells | | `REPEATED_QUERY_HINTS` | `off` | `on` turns on the profile setting `user.repeated_query_hints.v1` for the eval's isolated profile before any agent launches, and suffixes transcripts with `__qhints`; see "Repeated-query hints on vs off" | | `REAL_CLAUDE_DIR` | `$CLAUDE_CONFIG_DIR` or `~/.claude` | where the Claude credential comes from: its `.claude_code_oauth_token` and `settings.json` `env`/`model` keys; see "Store isolation & auth" | -| `TRACEDECAY_BIN` | Bazel build of this checkout | explicit prebuilt tracedecay binary; skips the build and must exist | +| `TRACEDECAY_BIN` | Bazel build of this checkout | explicit prebuilt tracedecay binary; skips the build, must exist, and is required for the product-effectiveness cohort | +| `EFFICIENCY_SCORECARD` | unset | optional no-model scorecard path, joined only when its binary digest matches | | `TRACEDECAY_EVAL_BAZEL_CONFIG` | `release` | Bazel config for the build; `dist` builds the exact release bytes | | `EVAL_OUT` | unset | dir to copy scoreboard + report into | | `LEARNED_SKILL_WORKFLOW` | unset | run only that workflow's held-out scenarios, once per variant, on `full` | | `LEARNED_SKILL_VARIANTS` | `none generated manual` | learned-skill variants to compare | +## Product-effectiveness baseline + +The opt-in `baseline_*` cohort adds three neutral, independently graded tasks: +code-path explanation, a behavior-preserving source edit (including a check +against a harmful boundary change), and recall of a seeded prior decision. +They are excluded from ordinary adoption runs. Run them with one host/model, +one artifact, identical prompts, and repeated trials: + +```sh +bazel build --config=release //crates/tracedecay-cli:tracedecay +TRACEDECAY_BIN="$(bazel info --config=release bazel-bin)/crates/tracedecay-cli/tracedecay" +TRACEDECAY_AGENT_EVALS=1 EVAL_INCLUDE_BASELINE=1 HOSTS=claude \ + TRACEDECAY_BIN="$TRACEDECAY_BIN" \ + CLAUDE_MODELS=sonnet \ + SCENARIOS="baseline_order_stock_failure baseline_stock_check_rename baseline_currency_decision" \ + CHANNELS="native bare no-hints no-skills full" REPS=3 \ + evals/agent_adoption/run.sh +``` + +This is 45 live agent turns; the existing explicit `TRACEDECAY_AGENT_EVALS=1` +opt-in remains required. Native runs receive only Claude's standard file tools +and no TraceDecay MCP/plugin/CLI surface; the native condition is Claude-only. +Every condition retains the same native `Read`, `Write`, `Edit`, `Glob`, `Grep`, +`Task`, `Agent`, and `Skill` permissions. Product conditions add their product +tool permissions to that common set. +`bare` exposes MCP tool descriptions without hooks or skills. `no-hints` and +`no-skills` are the existing channel ablations; `full` is the ordinary bundled +experience. Read per-task results rather than adoption averages: the same +successful answer or edit counts regardless of whether a product surface was +invoked. Channel ablations retain their documented steering differences, so +they are descriptive unless prompt steering is also controlled. + +The report includes per-task pass rates with Wilson intervals, observed +run-to-run cost/latency ranges, tool attempts/failures, and deltas against the +same task/model's native run. Token and USD values remain unknown when the host +does not report usage. One-time fixture initialization/indexing time is shown +separately and is not silently spread across tasks. For cold startup, indexing, +restart, incremental-sync, storage, and RSS measurements, run the existing +no-model scorecard against the same binary and join it when grading: + +```sh +TRACEDECAY_BIN="$(python3 - "$RUN_DIR/meta.json" <<'PY' +import json, sys +print(json.load(open(sys.argv[1]))["binary_path"]) +PY +)" +scripts/efficiency-scorecard.py --binary "$TRACEDECAY_BIN" --runs 3 \ + --output target/efficiency-scorecard +evals/agent_adoption/grade.py --run-dir "$RUN_DIR" \ + --efficiency-scorecard target/efficiency-scorecard/scorecard.json +``` + +The scorecard is rejected unless its binary SHA-256 equals the agent run's +recorded artifact digest. Its wall times and resource measurements remain a +separate cost dimension rather than being converted to model tokens or dollars. +No live-model task outcomes or usage values are checked into the repository. + +The currency task requests one JSON object and compares all three facts to the +seeded decision. Extra claims, duplicate keys, missing facts, and contradictory +prose fail the check. The stock-rename task compiles its captured source with +Bazel and executes trusted assertions for the renamed API, stock boundaries, +reservations, and order totals. The assertions live outside the agent fixture. +Compilation or toolchain failures remain `unverified`; only successful test +execution can pass. Temporary Bazel packages are removed after verification. +This requires a writable repository checkout and Bazel when grading the edit. + +The no-model native regressions run in the Hermetic evaluation regressions CI +job. To exercise valid and destructive edits locally, run +`python3 evals/agent_adoption/stock_check_selftest.py` from the checkout; its +compiler/test invocations all go through Bazel. The ordinary grader self-test +also checks structured-answer rejection and permission parity without a build. + +### Related product-surface dispositions + +The task cohort and runtime scorecard do not measure dashboard navigation +interactions, Work/Remote Brain resident-state footprint, or language-specific +quality and demand. Therefore they do not authorize removing those surfaces. + +| proposal | disposition from available evidence | +|---|---| +| #4220 dashboard navigation and Work projections | Defer removals. #3653 preserves distinct discovery semantics and shared Work query plumbing; this baseline has no dashboard interaction evidence. Keep deep links and all current projections. | +| #4223 Work/Workflow default footprint | Defer default or ownership changes. #3656 qualifies Work and Remote Brain jobs; this baseline does not measure their initialization, resident state, or operator steps. Preserve histories and mounted journeys. | +| #4229 additional grammar defaults | Defer profile changes. This baseline has no language-quality, demand, or matched core-versus-extras build measurement. Preserve advertised language support rather than inferring non-use. | + +These are explicit deferrals, not keep/remove verdicts. The related empirical +acceptance remains open until the corresponding task, resource, and quality +measurements exist. + +No paid-model runs were launched for this change, so there are no new observed +effectiveness or usage values and no capability keep/remove verdict. The +executable cohort can compare native tools with the MCP core and existing +channel variants. The prior learned-skill and optional repeated-query/Scout +evaluations (#3654/#3655) remain prior work; they were not rerun or replaced. +Hook/skill ablations retain the runner's existing steering differences and +must not be read as isolated causal effects until those are controlled. + +| capability | current disposition | +|---|---| +| MCP core versus native tools | Defer a product-value verdict until the held-out tasks are run; this checkout contains no model outcomes. | +| Hook guidance and skills | Defer individual keep/simplify/remove decisions; existing comparisons are executable, but this change has no run evidence and their steering conditions differ. | +| Learned skills and repeated-query/Scout guidance | Preserve the prior #3654/#3655 qualification; no new outcome result or default change is claimed here. | + ### Cost expectations One scenario x host x model is a single short headless agent turn. There are 44 @@ -236,8 +340,8 @@ answer pass. | `kind` | what is checked | evidence | |--------|-----------------|----------| -| `answer` | the final answer states every `required` conclusion and none of the `forbidden` ones | regex per conclusion (case-insensitive) | -| `effect` | the resulting state. `files` maps a glob over the post-run `src/**/*.rs` snapshot to required/forbidden checks; `tool_effects` requires a named tool to have executed successfully with matching input | file text; tool result | +| `answer` | the final answer states every `required` conclusion and none of the `forbidden` ones, or is one JSON object exactly matching `json_equals` | regex per conclusion (case-insensitive), or parsed structured facts | +| `effect` | the resulting state. `behavior: "stock_check_rename"` executes the trusted stock/order suite against captured sources. Other scenarios use `files` for file checks or `tool_effects` for successfully executed calls | Bazel test result; file text; tool result | | `ungraded` | nothing: no verifiable ground truth (run-dependent output, deferred fixture gap, or a routing-only specialist scenario). It carries a `reason` | none | A check is `{"id": "", "pattern": ""}`. The score is the diff --git a/evals/agent_adoption/claude_tools.sh b/evals/agent_adoption/claude_tools.sh new file mode 100644 index 0000000000..ab11447af7 --- /dev/null +++ b/evals/agent_adoption/claude_tools.sh @@ -0,0 +1,12 @@ +#!/usr/bin/env bash + +# Keep the native capabilities fixed across discovery conditions. Product +# conditions add only their MCP and supported CLI surface. +claude_allowed_tools() { + local condition="$1" binary="$2" + local allowed='Read,Write,Edit,Glob,Grep,Task,Agent,Skill' + if [[ "$condition" != native ]]; then + allowed+=",mcp__tracedecay__*,mcp__plugin_tracedecay_graph__*,Bash(tracedecay tool *),Bash($binary tool *)" + fi + printf '%s\n' "$allowed" +} diff --git a/evals/agent_adoption/grade.py b/evals/agent_adoption/grade.py index 0a1abb879a..ba222e68e2 100755 --- a/evals/agent_adoption/grade.py +++ b/evals/agent_adoption/grade.py @@ -28,12 +28,16 @@ import argparse import fnmatch import json +import math import os import re +import statistics import sys from dataclasses import dataclass, field from typing import Any, Optional +import stock_check + # Tools that do not count as a "meaningful" action for first-choice/efficiency. # Loading a skill is routing, which is scored separately. IGNORED_TOOLS = {"TodoWrite", "Skill"} @@ -53,7 +57,7 @@ # Discovery-channel attribution # --------------------------------------------------------------------------- # # Ablation conditions (see run.sh CHANNELS). "full" = every discovery channel on. -CONDITIONS = ("full", "no-hints", "no-skills", "bare", "cli-only") +CONDITIONS = ("native", "bare", "full", "no-hints", "no-skills", "cli-only") KNOWN_HOSTS = ("claude", "codex") # Distinctive fragments of the hook-injected tool hints. These MIRROR the @@ -769,8 +773,10 @@ def repeated_query_hint_next_steps(tr: Transcript) -> list[str]: # --------------------------------------------------------------------------- # # A scenario's `outcome` block declares what a correct run must establish. # {"kind": "answer", "answer": CHECKS} final answer states conclusions +# {"kind": "answer", "answer": {"json_equals": {...}}} exact structured facts # {"kind": "effect", "files": {glob: CHECKS}, "tool_effects": [...]} # resulting state / successful call +# {"kind": "effect", "behavior": "stock_check_rename"} compiled behavior of edited source # {"kind": "ungraded", "reason": "..."} no verifiable ground truth # CHECKS = {"required": [CHECK...], "forbidden": [CHECK...]} with # CHECK = {"id": "", "pattern": ""}. @@ -796,6 +802,25 @@ def _evaluate_checks(text: str, checks: dict) -> dict: } +def _structured_answer(text: str, expected: dict) -> dict: + """Compare complete facts, rejecting contradictory prose and duplicate keys.""" + def unique_keys(pairs): + result = {} + for key, value in pairs: + if key in result: + raise ValueError(f"duplicate answer key: {key}") + result[key] = value + return result + + try: + actual = json.loads(text, object_pairs_hook=unique_keys) + except ValueError as error: + return {"status": "fail", "score": 0.0, "reason": f"answer is not one unambiguous JSON object: {error}"} + matched = actual == expected + return {"status": "pass" if matched else "fail", "score": float(matched), + "evidence": {"answer": {"expected": expected, "actual": actual}}} + + def _glob_files(effects: dict, pattern: str) -> str: """Concatenate the snapshotted files a glob selects (`*` crosses `/`).""" return "\n".join( @@ -816,12 +841,26 @@ def grade_outcome(scn: dict, tr: Transcript, effects: Optional[dict] = None) -> """Task correctness, independent of which tools the agent chose. status: pass | fail | ungraded (no ground truth declared) | unverified - (an effect scenario whose post-run snapshot was not captured). + (the post-run snapshot or its behavioral verification is unavailable). """ outcome = scn.get("outcome") or {"kind": "ungraded", "reason": "scenario declares no outcome"} kind = outcome["kind"] if kind == "ungraded": return {"kind": kind, "status": "ungraded", "score": None, "reason": outcome.get("reason")} + if kind == "answer" and "json_equals" in outcome["answer"]: + return {"kind": kind, **_structured_answer(tr.final_text, outcome["answer"]["json_equals"])} + if kind == "effect" and outcome.get("behavior"): + if effects is None: + return {"kind": kind, "status": "unverified", "score": None, + "reason": "no post-run file snapshot for the behavior oracle"} + if outcome["behavior"] != "stock_check_rename": + return {"kind": kind, "status": "unverified", "score": None, + "reason": "unknown behavior oracle"} + result = stock_check.verify(effects) + status = result["status"] + return {"kind": kind, "status": status, + "score": None if status == "unverified" else float(status == "pass"), + "reason": result["reason"], "evidence": {"behavior": result}} missing: list[str] = [] forbidden: list[str] = [] @@ -895,8 +934,19 @@ def check_checks(sid, where, checks, need_required=True): if not outcome.get("reason"): problems.append(f"{sid}: ungraded outcome needs a reason") elif kind == "answer": - check_checks(sid, "outcome.answer", outcome.get("answer")) + answer = outcome.get("answer") + if isinstance(answer, dict) and "json_equals" in answer: + expected = answer["json_equals"] + if (set(answer) != {"json_equals"} or not isinstance(expected, dict) + or not expected or any(not isinstance(v, str) for v in expected.values())): + problems.append(f"{sid}: json_equals must be the sole answer check and contain nonempty string facts") + else: + check_checks(sid, "outcome.answer", answer) else: + if "behavior" in outcome: + if set(outcome) != {"kind", "behavior"} or outcome["behavior"] != "stock_check_rename": + problems.append(f"{sid}: behavior must be stock_check_rename and the sole effect oracle") + continue files, tool_effects = outcome.get("files", {}), outcome.get("tool_effects", []) if not files and not tool_effects: problems.append(f"{sid}: effect outcome needs files or tool_effects") @@ -1016,7 +1066,7 @@ def score_scenario(scn: dict, tr: Transcript, seeded_facts: dict, run_meta: dict if "expected_skill" in scn or "allowed_skills" in scn: expected = scn.get("expected_skill") allowed = scn.get("allowed_skills", [expected] if expected else []) - measured = run_meta.get("channel_condition", "full") not in {"no-skills", "bare"} + measured = run_meta.get("channel_condition", "full") not in {"native", "no-skills", "bare"} unmeasured_reason = None if measured else "skills_unavailable_in_condition" selected = invoked_skills(tr) if measured else None details["skill_routing"] = { @@ -1071,6 +1121,7 @@ def score_scenario(scn: dict, tr: Transcript, seeded_facts: dict, run_meta: dict "repeated_query_hint_next_steps": repeated_query_hint_next_steps(tr), "final_answer_chars": len(tr.final_text), "cost": tr.cost, + "runner_elapsed_ms": run_meta.get("duration_ms"), "parse_note": tr.parse_note, "run_meta": run_meta, } @@ -1197,9 +1248,13 @@ def rate(key): return agg -def _prompt_tokens(cost: dict) -> Optional[int]: +def _prompt_tokens(cost: dict, host: Optional[str] = None) -> Optional[int]: + if host == "codex": + return cost.get("input_tokens") parts = [cost.get(k) for k in ("input_tokens", "cache_creation_input_tokens", "cache_read_input_tokens")] - return None if parts[0] is None else sum(p or 0 for p in parts) + if any(not isinstance(part, (int, float)) for part in parts): + return None + return sum(parts) def aggregate_cost_and_hints(results: list[dict], by_scenario: bool = False) -> dict: @@ -1231,7 +1286,7 @@ def aggregate_cost_and_hints(results: list[dict], by_scenario: bool = False) -> "mean_duration_s": _mean([None if u.get("duration_ms") is None else u["duration_ms"] / 1000 for u in costs]), "usd_per_pass": usd_per_pass(rs), - "mean_prompt_tokens": _mean([_prompt_tokens(u) for u in costs]), + "mean_prompt_tokens": _mean([_prompt_tokens(r["cost"], r["host"]) for r in rs]), "mean_output_tokens": _mean([u.get("output_tokens") for u in costs]), "hints_total": sum(next_steps.values()), "runs_with_hint": sum(1 for r in rs if r.get("repeated_query_hint_next_steps")), @@ -1240,6 +1295,242 @@ def aggregate_cost_and_hints(results: list[dict], by_scenario: bool = False) -> return out +def _sample_summary(values: list[Optional[float]]) -> dict: + known = sorted(v for v in values if isinstance(v, (int, float)) and math.isfinite(v)) + if not known: + return {"n": len(values), "observed": 0, "unknown": len(values), "mean": None, + "median": None, "range": None} + + def quantile(q: float) -> float: + position = (len(known) - 1) * q + lower = math.floor(position) + upper = math.ceil(position) + return known[lower] + (known[upper] - known[lower]) * (position - lower) + + return { + "n": len(values), + "observed": len(known), + "unknown": len(values) - len(known), + "mean": round(statistics.mean(known), 4), + "median": round(statistics.median(known), 4), + "q1": round(quantile(0.25), 4), + "q3": round(quantile(0.75), 4), + "range": [round(known[0], 4), round(known[-1], 4)], + } + + +def _wilson_interval(passed: int, total: int) -> Optional[list[float]]: + if total == 0: + return None + z = 1.96 + rate = passed / total + denominator = 1 + z * z / total + center = (rate + z * z / (2 * total)) / denominator + margin = z * math.sqrt(rate * (1 - rate) / total + z * z / (4 * total * total)) / denominator + return [round(max(0.0, center - margin), 4), round(min(1.0, center + margin), 4)] + + +def effectiveness_comparisons( + results: list[dict], invalid_runs: Optional[list[dict]] = None +) -> dict: + """Compare verified task outcomes and observed costs without blending adoption. + + Each condition remains grouped by task, host/model, and hint setting. A + condition delta is only reported against that exact cell's native-tools run. + """ + groups: dict[tuple[str, str, str, str], list[dict]] = {} + invalid_groups: dict[tuple[str, str, str, str], list[dict]] = {} + for result in results: + host_model = f"{result['host']}/{result['model']}" if result.get("model") else result["host"] + key = ( + result["id"], + host_model, + result.get("condition", "full"), + result.get("repeated_query_hints", "off"), + ) + groups.setdefault(key, []).append(result) + for result in invalid_runs or []: + model = result.get("model") + host_model = f"{result['host']}/{model}" if model else result["host"] + key = ( + result["id"], + host_model, + result.get("condition", "full"), + result.get("repeated_query_hints", "off"), + ) + invalid_groups.setdefault(key, []).append(result) + + cells: dict[str, dict] = {} + for scenario, host_model, condition, hints in sorted(set(groups) | set(invalid_groups)): + rows = groups.get((scenario, host_model, condition, hints), []) + invalid = invalid_groups.get((scenario, host_model, condition, hints), []) + graded = [r["correctness"]["status"] for r in rows + if r["correctness"]["status"] in ("pass", "fail")] + passed = graded.count("pass") + attempts = len(rows) + len(invalid) + unknown_outcomes = sum(r["correctness"]["status"] == "ungraded" for r in rows) + unknown_outcomes += sum(r.get("outcome_kind") == "ungraded" for r in invalid) + scored_attempts = attempts - unknown_outcomes + costs = [r["cost"] for r in rows] + tool_attempts = [r["tool_success"]["attempted"] for r in rows] + tool_failures = [r["tool_success"]["failed"] for r in rows] + cell = { + "scenario": scenario, + "host_model": host_model, + "condition": condition, + "repeated_query_hints": hints, + "correctness": { + "n": len(graded), + "passed": passed, + "failed": graded.count("fail"), + "pass_rate": round(passed / len(graded), 4) if graded else None, + "wilson_95": _wilson_interval(passed, len(graded)), + "ungraded": sum(r["correctness"]["status"] == "ungraded" for r in rows), + "unverified": sum(r["correctness"]["status"] == "unverified" for r in rows), + }, + "task_success": { + "attempts": attempts, + "scored_attempts": scored_attempts, + "passed": passed, + "unsuccessful_or_unverified": scored_attempts - passed, + "unknown_outcomes": unknown_outcomes, + "pass_rate": round(passed / scored_attempts, 4) if scored_attempts else None, + "wilson_95": _wilson_interval(passed, scored_attempts), + "invalid_runs": len(invalid), + }, + "observed": { + "input_tokens": _sample_summary([c.get("input_tokens") for c in costs] + [None] * len(invalid)), + "cache_read_tokens": _sample_summary( + [c.get("cache_read_input_tokens") for c in costs] + [None] * len(invalid) + ), + "cache_creation_tokens": _sample_summary( + [c.get("cache_creation_input_tokens") for c in costs] + [None] * len(invalid) + ), + "prompt_tokens": _sample_summary( + [_prompt_tokens(r["cost"], r["host"]) for r in rows] + [None] * len(invalid) + ), + "output_tokens": _sample_summary( + [c.get("output_tokens") for c in costs] + [None] * len(invalid) + ), + "total_tokens": _sample_summary( + [c.get("total_tokens") for c in costs] + [None] * len(invalid) + ), + "cost_usd": _sample_summary( + [c.get("total_cost_usd") for c in costs] + [None] * len(invalid) + ), + "elapsed_ms": _sample_summary( + [r.get("runner_elapsed_ms") for r in rows] + + [r.get("runner_elapsed_ms") for r in invalid] + ), + "tool_attempts": _sample_summary(tool_attempts + [None] * len(invalid)), + "tool_failures": _sample_summary(tool_failures + [None] * len(invalid)), + }, + } + cells[" / ".join((scenario, host_model, condition, f"hints-{hints}"))] = cell + + for cell in cells.values(): + if cell["condition"] == "native": + cell["comparison_to_native"] = {"status": "reference"} + continue + key = ( + cell["scenario"], + cell["host_model"], + "native", + cell["repeated_query_hints"], + ) + baseline = groups.get(key, []) + baseline_invalid = invalid_groups.get(key, []) + if key not in groups and key not in invalid_groups: + cell["comparison_to_native"] = { + "status": "missing_native_baseline", + "pass_rate_delta": None, + } + continue + baseline_passes = [r["correctness"]["status"] for r in baseline + if r["correctness"]["status"] in ("pass", "fail")] + native_attempts = len(baseline) + len(baseline_invalid) + native_unknown = sum(r["correctness"]["status"] == "ungraded" for r in baseline) + native_unknown += sum(r.get("outcome_kind") == "ungraded" for r in baseline_invalid) + native_scored_attempts = native_attempts - native_unknown + product_rate = cell["task_success"]["pass_rate"] + native_rate = ( + baseline_passes.count("pass") / native_scored_attempts if native_scored_attempts else None + ) + native_costs = [r["cost"] for r in baseline] + native_metrics = { + "prompt_tokens": _sample_summary( + [_prompt_tokens(r["cost"], r["host"]) for r in baseline] + )["median"], + "total_tokens": _sample_summary( + [c.get("total_tokens") for c in native_costs] + [None] * len(baseline_invalid) + )["median"], + "cost_usd": _sample_summary( + [c.get("total_cost_usd") for c in native_costs] + [None] * len(baseline_invalid) + )["median"], + "elapsed_ms": _sample_summary([r.get("runner_elapsed_ms") for r in baseline])["median"], + "tool_attempts": _sample_summary([r["tool_success"]["attempted"] for r in baseline])["median"], + } + if baseline_invalid: + native_metrics["prompt_tokens"] = _sample_summary( + [_prompt_tokens(r["cost"], r["host"]) for r in baseline] + [None] * len(baseline_invalid) + )["median"] + native_metrics["tool_attempts"] = _sample_summary( + [r["tool_success"]["attempted"] for r in baseline] + [None] * len(baseline_invalid) + )["median"] + if baseline_invalid: + native_metrics["elapsed_ms"] = _sample_summary( + [r.get("runner_elapsed_ms") for r in baseline] + + [r.get("runner_elapsed_ms") for r in baseline_invalid] + )["median"] + observed_deltas = {} + for metric, baseline_median in native_metrics.items(): + current_median = cell["observed"][metric]["median"] + observed_deltas[metric] = ( + round(current_median - baseline_median, 4) + if current_median is not None and baseline_median is not None else None + ) + cell["comparison_to_native"] = { + "status": "compared" if product_rate is not None and native_rate is not None else "ungraded", + "native_success_rate": round(native_rate, 4) if native_rate is not None else None, + "success_rate_delta": round(product_rate - native_rate, 4) + if product_rate is not None and native_rate is not None else None, + "observed_median_deltas": observed_deltas, + } + return cells + + +def load_efficiency_scorecard(path: str, run_meta: dict) -> dict: + with open(path) as stream: + scorecard = json.load(stream) + if not scorecard.get("schema", "").startswith("tracedecay.efficiency-scorecard/"): + raise ValueError("unsupported efficiency scorecard schema") + runs = scorecard.get("runs") or [] + if scorecard.get("verdict") != "ok" or not runs or any( + run.get("status") != "ok" for run in runs + ): + raise ValueError("efficiency scorecard contains a failed or incomplete run") + expected_sha = run_meta.get("binary_sha256") + actual_sha = (scorecard.get("binary") or {}).get("sha256") + if not expected_sha or expected_sha != actual_sha: + raise ValueError("efficiency scorecard binary SHA-256 does not match the agent eval") + median = scorecard.get("median") or {} + if not isinstance(median, dict) or any( + not isinstance(name, str) or isinstance(value, bool) + or not isinstance(value, (int, float)) or not math.isfinite(value) + for name, value in median.items() + ): + raise ValueError("efficiency scorecard medians must be finite scalar measurements") + return { + "binary_sha256": actual_sha, + "binary_version": (scorecard.get("binary") or {}).get("version"), + "source_git_commit": (scorecard.get("source") or {}).get("git_commit"), + "fixture_sha256": (scorecard.get("fixture") or {}).get("tree_sha256"), + "runs": len(scorecard.get("runs", [])), + "median": median, + "attribution": "separate no-model runtime journey; not added to agent token or USD totals", + } + + def usd_per_pass(rs: list[dict]) -> Optional[float]: """Total spend divided by verified passes; None when spend is unknown or nothing passed.""" spend = [r["cost"].get("total_cost_usd") for r in rs] @@ -1333,8 +1624,19 @@ def render_report(scoreboard: dict) -> str: meta = scoreboard.get("meta", {}) lines.append(f"- run: `{meta.get('run_id','?')}`") lines.append(f"- git: `{meta.get('git_sha','?')}`") + lines.append(f"- binary: `{meta.get('binary_version') or 'unknown'}` " + f"(sha256 `{meta.get('binary_sha256') or 'unknown'}`)") + lines.append(f"- fixture sha256: `{meta.get('fixture_sha256') or 'unknown'}`") lines.append(f"- graded: {len(scoreboard['results'])} transcript(s)") lines.append(f"- invalid launches excluded: {len(meta.get('invalid_runs', []))}") + setup_ms = meta.get("fixture_setup_elapsed_ms") + lines.append(f"- one-time fixture initialization/indexing: " + f"{setup_ms} ms" if setup_ms is not None + else "- one-time fixture initialization/indexing: unknown") + scorecard = meta.get("efficiency_scorecard") + if scorecard: + lines.append(f"- no-model efficiency scorecard: binary SHA matched; " + f"{scorecard['runs']} run(s); fixture `{scorecard.get('fixture_sha256') or 'unknown'}`") lines.append("") lines.append("## Skill routing (separate from task outcomes)") lines.append("") @@ -1355,7 +1657,8 @@ def fmt(x): lines.append("") lines.append("Pass means every required conclusion or effect was established and no forbidden " "one appeared, whichever tools were used. `ungraded` scenarios declare no " - "verifiable ground truth; `unverified` effect scenarios lack a post-run snapshot. " + "verifiable ground truth; `unverified` effect scenarios lack a post-run snapshot " + "or completed behavioral verification. " "Both are excluded from the pass rate.") lines.append("") lines.append("| host/model | n | pass | fail | unverified | ungraded | pass rate |") @@ -1445,7 +1748,9 @@ def cell(name): lines.append("## Correctness, cost, latency, and repeated-query hints") lines.append("") - lines.append("Prompt tokens include cache reads and writes. `-` means the host did not report it.") + lines.append("Prompt tokens include cache reads and writes when the host reports them; schema/context " + "tokens are included in host-reported input usage, but hosts do not separately " + "attribute tool-schema overhead. `-` means unknown, not zero.") lines.append("") lines += render_cost_and_hints(scoreboard.get("cost_and_hints", {})) if scoreboard.get("repeated_query_scenarios"): @@ -1453,6 +1758,90 @@ def cell(name): lines.append("") lines += render_cost_and_hints(scoreboard["repeated_query_scenarios"]) + lines.append("## Matched task effectiveness and observed cost") + lines.append("") + lines.append("Task outcome is independently graded and native success counts equally. " + "Intervals are 95% Wilson intervals for the per-cell pass rate; cost ranges " + "describe observed run-to-run variability. Timeouts, failed launches, and " + "unverified outcomes count as unsuccessful attempts, not as successful task " + "completions. Prompt tokens include cache reads " + "and writes where reported; tool schema/context overhead is part of host-reported " + "input usage and cannot be separated by these hosts. Missing usage is unknown, " + "never zero. Cold startup/indexing is reported separately and is not converted " + "to dollars or silently amortized.") + lines.append("") + lines.append("| task / host / condition | successful / scored attempts | 95% interval | verified outcome pass rate | success Δ vs native | median cost Δ (prompt tok / USD / ms) | input tokens | cache read | cache create | output tokens | prompt total | total tokens | USD | elapsed ms | tool attempts / failures |") + lines.append("|---|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|") + + def sample_cell(sample: dict, digits=0): + if not sample or sample["median"] is None: + return "-" + lo, hi = sample["range"] + return f"{sample['median']:.{digits}f} [{lo:.{digits}f}, {hi:.{digits}f}] (unknown {sample['unknown']})" + + for key, cell in sorted(scoreboard.get("effectiveness", {}).items()): + correctness = cell["correctness"] + task_success = cell["task_success"] + task_success_text = f"{task_success['passed']}/{task_success['scored_attempts']}" + if task_success["unknown_outcomes"]: + task_success_text += f" ({task_success['unknown_outcomes']} ungraded)" + if task_success["invalid_runs"]: + task_success_text += f" ({task_success['invalid_runs']} invalid)" + interval = task_success["wilson_95"] + interval_text = "-" if interval is None else f"[{interval[0]:.2f}, {interval[1]:.2f}]" + comparison = cell.get("comparison_to_native", {}) + delta = comparison.get("success_rate_delta") + delta_text = "-" if delta is None else f"{delta:+.2f}" + cost_deltas = comparison.get("observed_median_deltas", {}) + def delta_value(metric, digits): + value = cost_deltas.get(metric) + return "-" if value is None else f"{value:+.{digits}f}" + delta_cost_text = ( + f"{delta_value('prompt_tokens', 0)} / {delta_value('cost_usd', 4)} / " + f"{delta_value('elapsed_ms', 0)}" + ) + observed = cell["observed"] + attempts = observed["tool_attempts"]["mean"] + failures = observed["tool_failures"]["mean"] + tool_unknown = observed["tool_attempts"]["unknown"] + tool_text = "-" if attempts is None else f"{attempts:.1f} / {failures:.1f}" + if tool_unknown: + tool_text += f" ({tool_unknown} unknown)" + lines.append( + f"| {key} | {task_success_text} | {interval_text} | " + f"{fmt(correctness['pass_rate'])} | {delta_text} | " + f"{delta_cost_text} | " + f"{sample_cell(observed['input_tokens'])} | {sample_cell(observed['cache_read_tokens'])} | " + f"{sample_cell(observed['cache_creation_tokens'])} | {sample_cell(observed['output_tokens'])} | " + f"{sample_cell(observed['prompt_tokens'])} | {sample_cell(observed['total_tokens'])} | " + f"{sample_cell(observed['cost_usd'], 4)} | " + f"{sample_cell(observed['elapsed_ms'])} | {tool_text} |" + ) + lines.append("") + if scorecard: + lines.append("### Startup, indexing, and repeated-use resource baseline") + lines.append("") + lines.append("The efficiency scorecard is a separate no-model runtime journey. Its artifact " + "SHA-256 must match this agent evaluation. These wall times and resource " + "measurements are not model-token or monetary costs.") + lines.append("") + lines.append("| metric | median | unit |") + lines.append("|---|---:|---|") + for metric, value in sorted(scorecard["median"].items()): + if metric.endswith("_seconds"): + unit = "s" + elif metric.endswith("_ms"): + unit = "ms" + elif any(part == "bytes" or part.endswith("_bytes") for part in metric.split(".")): + unit = "B" + else: + unit = "—" + rendered = f"{value:,.3f}" + if unit not in ("s", "ms"): + rendered = rendered.rstrip("0").rstrip(".") + lines.append(f"| `{metric}` | {rendered} | {unit} |") + lines.append("") + if scoreboard.get("learned_skill"): lines += render_learned_skill(scoreboard["learned_skill"]) @@ -1546,6 +1935,10 @@ def main() -> int: here = os.path.dirname(os.path.abspath(__file__)) ap.add_argument("--run-dir") ap.add_argument("--scenarios", default=os.path.join(here, "scenarios")) + ap.add_argument( + "--efficiency-scorecard", + help="no-model efficiency-scorecard JSON from the exact same binary artifact", + ) ap.add_argument( "--lint-only", action="store_true", @@ -1627,6 +2020,15 @@ def main() -> int: if os.path.exists(meta_path): with open(meta_path) as f: run_meta_all = json.load(f) + efficiency_scorecard = None + if args.efficiency_scorecard: + try: + efficiency_scorecard = load_efficiency_scorecard( + args.efficiency_scorecard, run_meta_all + ) + except (OSError, ValueError) as error: + print(f"efficiency scorecard rejected: {error}", file=sys.stderr) + return 1 results = [] invalid_runs = [] @@ -1669,6 +2071,9 @@ def main() -> int: "host": host, "model": per_meta.get("model"), "condition": condition, + "repeated_query_hints": per_meta.get("repeated_query_hints", "off"), + "runner_elapsed_ms": per_meta.get("duration_ms"), + "outcome_kind": (scn.get("outcome") or {}).get("kind", "ungraded"), "exit_code": exit_code, "timed_out": bool(per_meta.get("timed_out")), "reason": "timed_out" if per_meta.get("timed_out") else "launch_failed", @@ -1684,6 +2089,9 @@ def main() -> int: "host": host, "model": per_meta.get("model"), "condition": condition, + "repeated_query_hints": per_meta.get("repeated_query_hints", "off"), + "runner_elapsed_ms": per_meta.get("duration_ms"), + "outcome_kind": (scn.get("outcome") or {}).get("kind", "ungraded"), "exit_code": exit_code, "timed_out": False, "reason": "unparseable_empty", @@ -1703,11 +2111,18 @@ def main() -> int: "meta": { "run_id": os.path.basename(os.path.normpath(run_dir)), "git_sha": run_meta_all.get("git_sha", "?"), + "binary_path": run_meta_all.get("binary_path"), + "binary_sha256": run_meta_all.get("binary_sha256"), + "binary_version": run_meta_all.get("binary_version"), + "fixture_sha256": run_meta_all.get("fixture_sha256"), + "fixture_setup_elapsed_ms": run_meta_all.get("fixture_setup_elapsed_ms"), + "efficiency_scorecard": efficiency_scorecard, "hosts": run_meta_all.get("hosts", {}), "invalid_runs": invalid_runs, }, "aggregate": aggregate(results), "cost_and_hints": aggregate_cost_and_hints(results), + "effectiveness": effectiveness_comparisons(results, invalid_runs), "repeated_query_scenarios": aggregate_cost_and_hints( [r for r in results if r.get("repeated_query_class")], by_scenario=True), "results": results, diff --git a/evals/agent_adoption/run.sh b/evals/agent_adoption/run.sh index 97a42093fd..1cf2052b94 100755 --- a/evals/agent_adoption/run.sh +++ b/evals/agent_adoption/run.sh @@ -27,6 +27,7 @@ # HOSTS space-separated: "claude", "codex", or "claude codex" (default: claude) # SCENARIOS space-separated scenario ids to run (default: all active) # EVAL_INCLUDE_DEFERRED=1 also run scenarios with status="deferred" +# EVAL_INCLUDE_BASELINE=1 also run the held-out product-effectiveness cohort # CLAUDE_MODELS space-separated Claude matrix (default: opus sonnet) # CODEX_MODELS space-separated Codex matrix (default: gpt-5.6-sol) # CODEX_REASONING_EFFORT Codex reasoning effort (default: medium) @@ -57,6 +58,7 @@ umask 077 here="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" repo_root="$(cd "$here/../.." && pwd)" +source "$here/claude_tools.sh" EVAL_SCENARIOS_DIR="$(cd "${EVAL_SCENARIOS_DIR:-$here/scenarios}" && pwd)" export EVAL_SCENARIOS_DIR @@ -80,9 +82,9 @@ REPEATED_QUERY_HINTS="${REPEATED_QUERY_HINTS:-off}" # Ablation matrix. Default is "full" only (all discovery channels on) to keep # cost bounded, each extra condition multiplies the number of live agent runs. -# Opt in with e.g. CHANNELS="full no-hints no-skills bare cli-only". See the README. +# Opt in with e.g. CHANNELS="native bare" or "full no-hints". See the README. CHANNELS="${CHANNELS:-full}" -KNOWN_CONDITIONS="full no-hints no-skills bare cli-only" +KNOWN_CONDITIONS="native bare full no-hints no-skills cli-only" for c in $CHANNELS; do case " $KNOWN_CONDITIONS " in *" $c "*) : ;; @@ -114,6 +116,10 @@ STEER_TEXT="This repository has indexed code-relationship evidence available. Ch live=0 if [[ "${TRACEDECAY_AGENT_EVALS:-}" == "1" ]]; then live=1; fi +if [[ "${EVAL_INCLUDE_BASELINE:-0}" == "1" && -z "${TRACEDECAY_BIN:-}" ]]; then + echo "error: product-effectiveness evals require an explicit shipping TRACEDECAY_BIN" >&2 + exit 2 +fi # Live and dry runs both use the shipping Bazel binary built from this checkout # (scripts/build-eval-binary.sh). TRACEDECAY_BIN selects an already-built @@ -377,6 +383,7 @@ build_fixture() { fixture_main="$work/fixture-main" fixture_broken="$work/fixture-broken" +setup_started_ns="$(python3 -c 'import time; print(time.monotonic_ns())')" echo "indexing fixture-main (with seeded git history)..." build_fixture "$fixture_main" clean history if [[ -n "${LEARNED_SKILL_GENERATE:-}" ]]; then @@ -454,11 +461,53 @@ if [[ "$REPEATED_QUERY_HINTS" == on ]]; then fi fi -git_sha="$(git -C "$repo_root" rev-parse --short HEAD 2>/dev/null || echo unknown)" +setup_elapsed_ms=$(( ($(python3 -c 'import time; print(time.monotonic_ns())') - setup_started_ns) / 1000000 )) +git_sha="$(git -C "$repo_root" rev-parse HEAD 2>/dev/null || echo unknown)" +binary_path_json="$(python3 -c 'import json,sys; print(json.dumps(sys.argv[1]))' "$TD")" +binary_sha256="$(python3 - "$TD" <<'PY' +import hashlib, sys +digest = hashlib.sha256() +with open(sys.argv[1], "rb") as binary: + for chunk in iter(lambda: binary.read(1 << 20), b""): + digest.update(chunk) +print(digest.hexdigest()) +PY +)" +binary_version="$("$TD" --version 2>/dev/null | head -1 | tr -d '\n')" +fixture_digest() { + python3 - "$1" <<'PY' +import hashlib, os, sys +root = sys.argv[1] +digest = hashlib.sha256() +for base, dirs, files in os.walk(root): + dirs[:] = sorted(d for d in dirs if d != ".git") + for name in sorted(files): + path = os.path.join(base, name) + relative = os.path.relpath(path, root).replace(os.sep, "/") + digest.update(relative.encode("utf-8")) + digest.update(b"\0") + with open(path, "rb") as contents: + for chunk in iter(lambda: contents.read(1 << 20), b""): + digest.update(chunk) +print(digest.hexdigest()) +PY +} +fixture_main_sha256="$(fixture_digest "$fixture_main")" +fixture_broken_sha256="$(fixture_digest "$fixture_broken")" cat > "$run_dir/meta.json" < channels: +# native no MCP, plugin, hooks, skills, or TraceDecay CLI access +# bare MCP descriptions only; no hooks, skills, or steering # full hooks ON | skills ON | steering=plugin (candidate plugin, hermetic profile) # no-hints hooks OFF | skills ON | steering=fixed (isolates skill/description efficacy) # no-skills hooks ON | skills OFF | steering=fixed (isolates hint/description efficacy) @@ -493,6 +544,7 @@ fi provision_variant() { # $1 = condition; provisions $work/plugins/ (hermetic plugin copy). local cond="$1" d="$work/plugins/$1" + [[ "$cond" == "native" ]] && return 0 [[ -d "$d" ]] && return 0 mkdir -p "$work/plugins" cp -R "$plugin_src" "$d" @@ -517,6 +569,12 @@ CLAUDE_EXTRA=() claude_extra_for() { CLAUDE_EXTRA=() local cond="$1" fdir="$2" + if [[ "$cond" == "native" ]]; then + CLAUDE_EXTRA=(--setting-sources user,project,local + --strict-mcp-config --mcp-config "$empty_mcp_cfg" + --add-dir "$fdir") + return 0 + fi if [[ "$have_plugin" != "1" ]]; then echo "error: candidate plugin dir missing; cannot run '$cond' hermetically" >&2 return 2 @@ -538,31 +596,16 @@ claude_extra_for() { } # ---- select scenarios ------------------------------------------------------ # -python3 - "$EVAL_SCENARIOS_DIR" "${SCENARIOS:-}" "${EVAL_INCLUDE_DEFERRED:-0}" "$LEARNED_SKILL_WORKFLOW" > "$work/selected.tsv" <<'PY' -import json, os, sys -sdir, filt, incl_def, workflow = sys.argv[1], sys.argv[2].split(), sys.argv[3] == "1", sys.argv[4] -for fn in sorted(os.listdir(sdir)): - if not fn.endswith(".json"): - continue - s = json.load(open(os.path.join(sdir, fn))) - if workflow: - if (s.get("learned_skill") or {}).get("workflow") != workflow: - continue - elif s.get("status") == "learned" or (s.get("status") == "deferred" and not incl_def): - continue - if filt and s["id"] not in filt: - continue - for host in s.get("hosts", []): - # emit: id \t host \t fixture \t effect(0|1) \t prompt - effect = "1" if s.get("outcome", {}).get("files") else "0" - print("\t".join([s["id"], host, s.get("fixture", "main"), effect, s["prompt"].replace("\t", " ").replace("\n", " ").replace("\r", " ")])) -PY +python3 "$here/select_scenarios.py" "$EVAL_SCENARIOS_DIR" "${SCENARIOS:-}" \ + "${EVAL_INCLUDE_DEFERRED:-0}" "$LEARNED_SKILL_WORKFLOW" "${EVAL_INCLUDE_BASELINE:-0}" \ + > "$work/selected.tsv" # ---- run each scenario x host x condition ---------------------------------- # run_claude() { # Uses the global CLAUDE_EXTRA array set by claude_extra_for. - local prompt="$1" fixture="$2" out="$3" err="$4" model="$5" home="$6" - local allowed="Read,Glob,Grep,Task,Agent,Skill,mcp__tracedecay__*,mcp__plugin_tracedecay_graph__*,Bash(tracedecay tool *),Bash($TD tool *)" + local prompt="$1" fixture="$2" out="$3" err="$4" model="$5" home="$6" condition="$7" + local allowed + allowed="$(claude_allowed_tools "$condition" "$TD")" ( cd "$fixture" && HOME="$home" CLAUDE_CONFIG_DIR="$home/.claude" PATH="$EVAL_PATH" \ timeout "$SCENARIO_TIMEOUT" claude -p "$prompt" \ --output-format stream-json --verbose --include-hook-events \ @@ -613,36 +656,40 @@ PY # One live scenario x host x model x condition x repetition. Runs in a # background subshell when PARALLEL > 1, so it must not mutate shared state. run_one() { - local cond="$1" sid="$2" host="$3" fixture="$4" prompt="$5" model="$6" base="$7" variant="$8" effect="${9:-0}" - local fdir out err start end rc=0 timed_out=false home="$CLAUDE_EVAL_HOME" learned="" + local cond="$1" sid="$2" host="$3" fixture="$4" prompt="$5" model="$6" base="$7" variant="$8" effect="${9:-0}" rep="${10:-1}" + local scenario_sha256="${11}" fdir out err start end duration_ms fixture_sha model_settings_json="{}" rc=0 timed_out=false home="$CLAUDE_EVAL_HOME" learned="" if [[ -n "$variant" ]]; then home="${VARIANT_HOME[$variant]}" learned=",\"learned_skill_variant\":\"$variant\",\"learned_skill_id\":\"${VARIANT_SKILL[$variant]}\"" fi + [[ "$host" == "codex" ]] && model_settings_json="{\"reasoning_effort\":\"$CODEX_REASONING_EFFORT\"}" fdir="$(fixture_dir_for "$fixture")" + fixture_sha="$fixture_main_sha256" + [[ "$fixture" == "broken" ]] && fixture_sha="$fixture_broken_sha256" out="$run_dir/${base}.stdout.jsonl" err="$run_dir/${base}.stderr.log" echo "RUN $sid [$host/$model/$cond${variant:+/v-$variant}] ..." - start=$(date +%s) + start="$(python3 -c 'import time; print(time.monotonic_ns())')" if [[ "$host" == "claude" ]]; then claude_extra_for "$cond" "$fdir" - run_claude "$prompt" "$fdir" "$out" "$err" "$model" "$home" || rc=$? + run_claude "$prompt" "$fdir" "$out" "$err" "$model" "$home" "$cond" || rc=$? else run_codex "$prompt" "$fdir" "$out" "$err" "$model" || rc=$? fi - end=$(date +%s) + end="$(python3 -c 'import time; print(time.monotonic_ns())')" + duration_ms=$(( (end - start) / 1000000 )) [[ $rc -eq 124 ]] && timed_out=true [[ "$effect" == 1 ]] && snapshot_effects "$fdir" "$run_dir/${base}.effects.json" cat > "$run_dir/${base}.meta.json" <= PARALLEL )); do wait -n || true; done - run_one "$cond" "$sid" "$host" "$fixture" "$prompt" "$model" "$base" "$variant" 0 & + run_one "$cond" "$sid" "$host" "$fixture" "$prompt" "$model" "$base" "$variant" 0 "$rep" "$scenario_sha256" & else - run_one "$cond" "$sid" "$host" "$fixture" "$prompt" "$model" "$base" "$variant" 0 + run_one "$cond" "$sid" "$host" "$fixture" "$prompt" "$model" "$base" "$variant" 0 "$rep" "$scenario_sha256" fi done done @@ -693,7 +740,13 @@ wait # ---- grade ----------------------------------------------------------------- # if [[ "$live" == "1" ]]; then echo "grading..." - python3 "$here/grade.py" --run-dir "$run_dir" --scenarios "$EVAL_SCENARIOS_DIR" || true + grade_args=(--run-dir "$run_dir" --scenarios "$EVAL_SCENARIOS_DIR") + [[ -n "${EFFICIENCY_SCORECARD:-}" ]] && grade_args+=(--efficiency-scorecard "$EFFICIENCY_SCORECARD") + if [[ -n "${EFFICIENCY_SCORECARD:-}" ]]; then + python3 "$here/grade.py" "${grade_args[@]}" || exit 1 + else + python3 "$here/grade.py" "${grade_args[@]}" || true + fi if [[ -n "${EVAL_OUT:-}" ]]; then mkdir -p "$EVAL_OUT" cp "$run_dir/scoreboard.json" "$run_dir/report.md" "$EVAL_OUT/" 2>/dev/null || true diff --git a/evals/agent_adoption/scenarios/baseline_currency_decision.json b/evals/agent_adoption/scenarios/baseline_currency_decision.json new file mode 100644 index 0000000000..db6e8b8a2a --- /dev/null +++ b/evals/agent_adoption/scenarios/baseline_currency_decision.json @@ -0,0 +1,22 @@ +{ + "id": "baseline_currency_decision", + "category": "product_effectiveness", + "hosts": [ + "claude" + ], + "fixture": "main", + "status": "baseline", + "prompt": "Was multi-currency support accepted or rejected, in which architecture-review month, and what units are order totals stored in? Return only a JSON object with the keys multi_currency_decision, review_month (YYYY-MM), and order_total_units. Use null for any fact you cannot establish.", + "outcome": { + "kind": "answer", + "answer": { + "json_equals": { + "multi_currency_decision": "rejected", + "review_month": "2026-05", + "order_total_units": "USD cents" + } + } + }, + "seeded_fact": "currency_fact_id", + "notes": "Held-out session-memory task. The complete structured answer must match the seeded decision; contradictory prose, duplicate keys, and extra claims cannot pass." +} diff --git a/evals/agent_adoption/scenarios/baseline_order_stock_failure.json b/evals/agent_adoption/scenarios/baseline_order_stock_failure.json new file mode 100644 index 0000000000..4e3d59c05f --- /dev/null +++ b/evals/agent_adoption/scenarios/baseline_order_stock_failure.json @@ -0,0 +1,30 @@ +{ + "id": "baseline_order_stock_failure", + "category": "product_effectiveness", + "hosts": [ + "claude" + ], + "fixture": "main", + "status": "baseline", + "prompt": "If an order requests more units than a warehouse has, how does order placement fail, and which checks are involved?", + "outcome": { + "kind": "answer", + "answer": { + "required": [ + { + "id": "names reserve_stock", + "pattern": "\\breserve_stock\\b" + }, + { + "id": "names check_availability", + "pattern": "\\bcheck_availability\\b" + }, + { + "id": "explains failure", + "pattern": "\\b(?:Err|error|fails?|insufficient)\\b" + } + ] + } + }, + "notes": "New held-out answer task with an independently checked failure path; included only by explicit baseline-cohort opt-in." +} diff --git a/evals/agent_adoption/scenarios/baseline_stock_check_rename.json b/evals/agent_adoption/scenarios/baseline_stock_check_rename.json new file mode 100644 index 0000000000..449f06ef7e --- /dev/null +++ b/evals/agent_adoption/scenarios/baseline_stock_check_rename.json @@ -0,0 +1,15 @@ +{ + "id": "baseline_stock_check_rename", + "category": "product_effectiveness", + "hosts": [ + "claude" + ], + "fixture": "main", + "status": "baseline", + "prompt": "Rename the warehouse availability check to has_sufficient_stock and update its callers. Keep the existing stock boundary behavior unchanged.", + "outcome": { + "kind": "effect", + "behavior": "stock_check_rename" + }, + "notes": "Held-out mutation task. Trusted Bazel tests compile the captured sources and exercise the renamed API, stock boundaries, reservations, and order totals. Missing execution is unverified, never a pass." +} diff --git a/evals/agent_adoption/select_scenarios.py b/evals/agent_adoption/select_scenarios.py new file mode 100644 index 0000000000..538f798ba3 --- /dev/null +++ b/evals/agent_adoption/select_scenarios.py @@ -0,0 +1,36 @@ +#!/usr/bin/env python3 +"""Select runner inputs and hash the same scenario bytes used for each prompt.""" +from __future__ import annotations + +import hashlib +import json +import sys +from pathlib import Path + + +def main() -> None: + directory, selected, include_deferred, workflow, include_baseline = sys.argv[1:] + selected_ids = selected.split() + for path in sorted(Path(directory).glob("*.json")): + source = path.read_bytes() + scenario = json.loads(source) + if workflow: + if (scenario.get("learned_skill") or {}).get("workflow") != workflow: + continue + elif (scenario.get("status") == "learned" + or (scenario.get("status") == "deferred" and include_deferred != "1") + or (scenario.get("status") == "baseline" and include_baseline != "1")): + continue + if selected_ids and scenario["id"] not in selected_ids: + continue + outcome = scenario.get("outcome", {}) + effect = "1" if outcome.get("files") or outcome.get("behavior") else "0" + digest = hashlib.sha256(source).hexdigest() + prompt = scenario["prompt"].replace("\t", " ").replace("\n", " ").replace("\r", " ") + for host in scenario.get("hosts", []): + print("\t".join([scenario["id"], host, scenario.get("fixture", "main"), + effect, digest, prompt])) + + +if __name__ == "__main__": + main() diff --git a/evals/agent_adoption/selftest.py b/evals/agent_adoption/selftest.py index 1def1e7adf..319dfed126 100644 --- a/evals/agent_adoption/selftest.py +++ b/evals/agent_adoption/selftest.py @@ -12,6 +12,7 @@ """ from __future__ import annotations +import hashlib import json import os import re @@ -19,6 +20,7 @@ import sys import tempfile from pathlib import Path +from unittest.mock import patch HERE = os.path.dirname(os.path.abspath(__file__)) sys.path.insert(0, HERE) @@ -125,6 +127,8 @@ def test_channels(): # unprompted: bare condition, adoption with nothing to credit. tr = transcript([claude_tool(TD, {}), claude_result("done")]) check("channel: unprompted under bare", grade.attribute_channel(tr, "bare") == grade.CH_UNPROMPTED) + check("channel: native run has no TraceDecay adoption", + grade.attribute_channel(transcript([claude_result("done")]), "native") == grade.CH_NONE) # CLI-only: the plugin has agents/skills but no MCP registration. A Bash # call to the TraceDecay CLI is still adoption and must retain tool identity. @@ -383,6 +387,14 @@ def test_end_to_end(): os.makedirs(rdir) with open(os.path.join(sdir, "explore_reserve_stock.json"), "w") as f: json.dump(scn, f) + with open(os.path.join(rdir, "meta.json"), "w") as f: + json.dump({ + "git_sha": "source-revision", + "binary_sha256": "artifact-digest", + "binary_version": "tracedecay test", + "fixture_sha256": "fixture-digest", + "fixture_setup_elapsed_ms": 1250, + }, f) # full condition, steering-driven, correct answer. answer = "The reserve_stock function decrements on-hand stock." @@ -430,8 +442,17 @@ def test_end_to_end(): agg = sb["aggregate"]["claude"] check("e2e: adoption_rate 1.0", agg["adoption_rate"] == 1.0) check("e2e: per-condition present", set(agg["conditions"]) == {"full", "bare"}) + check("e2e: no native baseline is reported as missing", + all(cell["comparison_to_native"]["status"] == "missing_native_baseline" + for cell in sb["effectiveness"].values())) + check("e2e: artifact and one-time setup identity preserved", + sb["meta"]["binary_sha256"] == "artifact-digest" + and sb["meta"]["fixture_setup_elapsed_ms"] == 1250) report = open(os.path.join(rdir, "report.md")).read() check("e2e: report has channel table", "Channel efficacy" in report) + check("e2e: report includes effectiveness and unknown setup attribution", + "Matched task effectiveness and observed cost" in report + and "1250 ms" in report) check("e2e: run meta hint setting reaches the scoreboard", by_cond["bare"].get("repeated_query_hints") == "on" and by_cond["full"].get("repeated_query_hints") == "off") @@ -669,6 +690,262 @@ def test_cost_and_corrections(): check("cost: scored result carries cost", scored["cost"] == tr.cost) +def test_product_effectiveness_comparisons(): + scenario = { + "id": "baseline_sample", + "category": "product_effectiveness", + "outcome": answer_outcome("correct"), + } + + def scored(condition, answer, usage, elapsed): + event = {"type": "result", "result": answer} + if usage is not None: + event["usage"] = usage + return grade.score_scenario( + scenario, + transcript([json.dumps(event)]), + {}, + { + "model": "sonnet", + "channel_condition": condition, + "duration_ms": elapsed, + "repeated_query_hints": "off", + }, + ) + + rows = [ + scored("native", "correct answer", { + "input_tokens": 100, "output_tokens": 20, + "cache_creation_input_tokens": 0, "cache_read_input_tokens": 0, + }, 1000), + scored("native", "wrong answer", { + "input_tokens": 120, "output_tokens": 30, + "cache_creation_input_tokens": 0, "cache_read_input_tokens": 0, + }, 1100), + scored("bare", "correct answer", None, 900), + scored("full", "correct answer", {"input_tokens": 80, "output_tokens": 10}, 800), + ] + for row in rows: + row["tool_success"] = {"attempted": 0, "failed": 0} + invalid = [{ + "id": "baseline_sample", + "host": "claude", + "model": "sonnet", + "condition": "native", + "repeated_query_hints": "off", + "runner_elapsed_ms": 1200, + "outcome_kind": "answer", + }] + cells = grade.effectiveness_comparisons(rows, invalid) + native = cells["baseline_sample / claude/sonnet / native / hints-off"] + bare = cells["baseline_sample / claude/sonnet / bare / hints-off"] + full = cells["baseline_sample / claude/sonnet / full / hints-off"] + check("effectiveness: task success is grouped independently of adoption", + native["correctness"]["pass_rate"] == 0.5 and native["correctness"]["wilson_95"] is not None) + check("effectiveness: condition effect is compared on the same task/model", + full["comparison_to_native"]["success_rate_delta"] == round(2 / 3, 4) + and bare["comparison_to_native"]["success_rate_delta"] == round(2 / 3, 4)) + check("effectiveness: invalid launches count as unsuccessful attempts", + native["task_success"]["attempts"] == 3 + and native["task_success"]["invalid_runs"] == 1 + and native["task_success"]["pass_rate"] == round(1 / 3, 4)) + check("effectiveness: missing usage remains unknown", + bare["observed"]["total_tokens"]["unknown"] == 1 + and bare["observed"]["total_tokens"]["mean"] is None) + check("effectiveness: repeated-run variability is reported", + native["observed"]["elapsed_ms"]["range"] == [1000, 1200] + and native["observed"]["prompt_tokens"]["range"] == [100, 120]) + check("effectiveness: cache usage remains separately observable", + native["observed"]["cache_read_tokens"]["unknown"] == 1 + and native["observed"]["cache_creation_tokens"]["unknown"] == 1 + and grade._prompt_tokens({"input_tokens": 100}, "claude") is None + and grade._prompt_tokens( + {"input_tokens": 1000, "cache_read_input_tokens": 800}, "codex" + ) == 1000) + ungraded_scenario = { + "id": "baseline_ungraded", + "category": "product_effectiveness", + "outcome": {"kind": "ungraded", "reason": "no independent checker"}, + } + ungraded = grade.score_scenario( + ungraded_scenario, + transcript([claude_result("an answer")]), + {}, + {"channel_condition": "native"}, + ) + ungraded["tool_success"] = {"attempted": 0, "failed": 0} + ungraded_cell = grade.effectiveness_comparisons([ungraded])[ + "baseline_ungraded / claude / native / hints-off" + ] + check("effectiveness: ungraded outcomes remain unknown, not failures", + ungraded_cell["task_success"]["pass_rate"] is None + and ungraded_cell["task_success"]["unknown_outcomes"] == 1) + + with tempfile.TemporaryDirectory() as work: + path = Path(work) / "scorecard.json" + path.write_text(json.dumps({ + "schema": "tracedecay.efficiency-scorecard/v1", + "binary": {"sha256": "abc", "version": "tracedecay 1"}, + "source": {"git_commit": "commit"}, + "fixture": {"tree_sha256": "fixture"}, + "verdict": "ok", + "runs": [{"status": "ok"}, {"status": "ok"}], + "median": { + "cold_index.wall_seconds": 12.5, + "baseline_restart.spawn_to_current_seconds": 1.25, + "daemon_restart.spawn_to_current_seconds": 2.5, + "incremental_sync.store.generation_physical_growth_bytes": 4096, + "rss.peak_bytes.cold_index": 1048576, + "tool_calls.search.p95_ms": 42.75, + }, + })) + attached = grade.load_efficiency_scorecard(str(path), {"binary_sha256": "abc"}) + check("effectiveness: cold-index baseline requires matching artifact identity", + attached["median"]["cold_index.wall_seconds"] == 12.5 + and attached["runs"] == 2) + check("effectiveness: scorecard join retains restart, storage, RSS, and tool costs", + attached["median"] == json.loads(path.read_text())["median"]) + resource_report = grade.render_report({ + "results": [], "aggregate": {}, + "meta": {"efficiency_scorecard": attached}, + }) + check("effectiveness: resource report includes measured units", + "baseline_restart.spawn_to_current_seconds` | 1.250 | s" in resource_report + and "generation_physical_growth_bytes` | 4,096 | B" in resource_report + and "rss.peak_bytes.cold_index` | 1,048,576 | B" in resource_report + and "tool_calls.search.p95_ms` | 42.750 | ms" in resource_report) + try: + grade.load_efficiency_scorecard(str(path), {"binary_sha256": "other"}) + matched_only = False + except ValueError: + matched_only = True + check("effectiveness: scorecard from another artifact is rejected", matched_only) + value = json.loads(path.read_text()) + value["runs"][0]["status"] = "failed" + path.write_text(json.dumps(value)) + failed_scorecard_rejected = False + try: + grade.load_efficiency_scorecard(str(path), {"binary_sha256": "abc"}) + except ValueError: + failed_scorecard_rejected = True + check("effectiveness: incomplete efficiency scorecard is rejected", + failed_scorecard_rejected) + + baseline = {} + for name in ( + "baseline_order_stock_failure.json", + "baseline_stock_check_rename.json", + "baseline_currency_decision.json", + ): + scenario = json.loads((Path(HERE) / "scenarios" / name).read_text()) + baseline[scenario["id"]] = scenario + check("effectiveness: baseline tasks are neutral and independently graded", + not grade.lint_scenarios(baseline) and not grade.lint_outcomes(baseline)) + + +def test_scenario_selection(): + with tempfile.TemporaryDirectory() as work: + # The declared id is an artifact identity, not a source filename. + scenario = {"id": "custom_case", "hosts": ["claude", "codex"], + "prompt": "Rename the stock check.\nPreserve behavior.", + "status": "baseline", "outcome": {"kind": "effect", "behavior": "stock_check_rename"}} + source = json.dumps(scenario, indent=2).encode() + path = Path(work) / "experiment.json" + path.write_bytes(source) + command = [sys.executable, str(Path(HERE) / "select_scenarios.py"), + work, "custom_case", "0", "", "1"] + selected = subprocess.run(command, capture_output=True, text=True) + rows = [line.split("\t") for line in selected.stdout.splitlines()] + check("runner: custom filename selects the declared id for both hosts", + selected.returncode == 0 and len(rows) == 2 + and {row[1] for row in rows} == {"claude", "codex"}, selected.stderr) + check("runner: selected prompt and digest use the same source bytes", + len(rows) == 2 and all(row == [ + "custom_case", host, "main", "1", hashlib.sha256(source).hexdigest(), + "Rename the stock check. Preserve behavior." + ] for row, host in zip(rows, ["claude", "codex"]))) + path.write_bytes(source + b"\n") + changed = subprocess.run(command, capture_output=True, text=True) + check("runner: scenario byte changes update the selected identity", + changed.returncode == 0 and bool(changed.stdout) + and changed.stdout.split("\t")[4] == hashlib.sha256(source + b"\n").hexdigest()) + excluded = subprocess.run(command[:-1] + ["0"], capture_output=True, text=True) + check("runner: baseline opt-in remains required", + excluded.returncode == 0 and not excluded.stdout) + + +def test_baseline_oracles(): + currency = json.loads((Path(HERE) / "scenarios/baseline_currency_decision.json").read_text()) + expected = {"multi_currency_decision": "rejected", "review_month": "2026-05", + "order_total_units": "USD cents"} + answers = { + "valid structured facts": (json.dumps(expected), "pass"), + "reordered structured facts": (json.dumps(dict(reversed(list(expected.items())))), "pass"), + "contradictory prose containing all keywords": ( + "Multi-currency was not rejected in the 2026-05 review; totals are not USD cents.", "fail"), + "wrong decision": (json.dumps({**expected, "multi_currency_decision": "accepted"}), "fail"), + "wrong units": (json.dumps({**expected, "order_total_units": "EUR cents"}), "fail"), + "missing date": (json.dumps({key: value for key, value in expected.items() + if key != "review_month"}), "fail"), + "extra contradictory explanation": (json.dumps({**expected, "explanation": "Actually accepted"}), "fail"), + "trailing contradiction": (json.dumps(expected) + " But that rejection was reversed.", "fail"), + "duplicate contradictory decision": ( + '{"multi_currency_decision":"accepted","multi_currency_decision":"rejected",' + '"review_month":"2026-05","order_total_units":"USD cents"}', "fail"), + } + for name, (answer, status) in answers.items(): + got = grade.grade_outcome(currency, transcript([claude_result(answer)])) + check("currency oracle: " + name, got["status"] == status, str(got)) + + edit = json.loads((Path(HERE) / "scenarios/baseline_stock_check_rename.json").read_text()) + destructive = { + "src/inventory.rs": "pub fn has_sufficient_stock(&self, sku: &str, qty: u32) -> bool { self.on_hand.get(sku).copied().unwrap_or(0) >= qty }", + "src/orders.rs": "// has_sufficient_stock\npub fn place_order() {}", + } + got = grade.grade_outcome(edit, transcript([claude_result("done")]), destructive) + check("stock oracle: deleting the order implementation cannot pass", got["status"] != "pass", str(got)) + missing = grade.grade_outcome(edit, transcript([claude_result("done")])) + check("stock oracle: missing snapshot is unverified", + missing["status"] == "unverified" and missing["score"] is None) + with patch.object(grade.stock_check.shutil, "which", return_value=None): + unavailable = grade.grade_outcome(edit, transcript([claude_result("done")]), {"src/lib.rs": ""}) + check("stock oracle: unavailable execution is never a pass", + unavailable["status"] == "unverified" and unavailable["score"] is None) + unsafe = grade.grade_outcome(edit, transcript([claude_result("done")]), + {"src/lib.rs": "", "src/../../escape.rs": ""}) + check("stock oracle: captured paths cannot escape temporary staging", + unsafe["status"] == "unverified" and unsafe["score"] is None) + mixed_answer = {"kind": "answer", "answer": { + "json_equals": expected, "required": [{"id": "keyword", "pattern": "rejected"}], + }} + check("schema: structured facts cannot silently bypass other answer checks", + bool(grade.lint_outcomes({"mixed": {"outcome": mixed_answer}}))) + check("schema: unknown behavior oracles are rejected", + bool(grade.lint_outcomes({"unknown": {"outcome": {"kind": "effect", "behavior": "unknown"}}}))) + check("schema: behavioral checks cannot silently bypass other effect checks", + bool(grade.lint_outcomes({"mixed": {"outcome": { + **edit["outcome"], "tool_effects": [{"id": "call", "tools": ["Edit"]}], + }}}))) + + +def test_claude_permissions(): + native = {"Read", "Write", "Edit", "Glob", "Grep", "Task", "Agent", "Skill"} + for condition in grade.CONDITIONS: + result = subprocess.run( + ["bash", "-c", 'source "$1" && claude_allowed_tools "$2" "$3"', + "permissions", str(Path(HERE) / "claude_tools.sh"), condition, "/tmp/eval binary"], + capture_output=True, text=True, + ) + allowed = set(result.stdout.strip().split(",")) + check(f"permissions: {condition} retains identical native editing capabilities", + result.returncode == 0 and allowed.intersection(native) == native, result.stderr) + check(f"permissions: {condition} controls product capabilities only", + allowed == native if condition == "native" else + allowed == native | {"mcp__tracedecay__*", "mcp__plugin_tracedecay_graph__*", + "Bash(tracedecay tool *)", "Bash(/tmp/eval binary tool *)"}, + str(allowed)) + + LEARNED_SKILL = "pricing-change-review" @@ -1023,6 +1300,10 @@ def main() -> int: test_repeated_query_hints_and_usage() test_skill_routing() test_cost_and_corrections() + test_product_effectiveness_comparisons() + test_scenario_selection() + test_baseline_oracles() + test_claude_permissions() test_learned_skill_routing() test_learned_skill_lint() test_learned_skill_end_to_end() diff --git a/evals/agent_adoption/stock_check.py b/evals/agent_adoption/stock_check.py new file mode 100644 index 0000000000..a32fd43094 --- /dev/null +++ b/evals/agent_adoption/stock_check.py @@ -0,0 +1,82 @@ +"""Execute trusted stock/order assertions against an agent's captured Rust sources. + +The agent cannot edit the assertions or BUILD file. Bazel remains the compiler +and test authority; missing tooling or a failed build is unverified, never a +successful edit. Temporary packages live in the workspace only for this call. +""" +from __future__ import annotations + +import shutil +import subprocess +import tempfile +from pathlib import Path, PurePosixPath + +HERE = Path(__file__).resolve().parent +BUILD = '''load("@rules_rust//rust:defs.bzl", "rust_library", "rust_test") + +rust_library( + name = "orders_fixture", + srcs = glob(["src/**/*.rs"]), + crate_root = "src/lib.rs", + edition = "2021", + tags = ["manual"], +) + +rust_test( + name = "stock_check", + srcs = ["stock_check.rs"], + crate_root = "stock_check.rs", + edition = "2021", + deps = [":orders_fixture"], + use_libtest_harness = False, + size = "small", + tags = ["manual"], +) +''' + + +def verify(effects: dict) -> dict: + """Return pass/fail only for observed behavior; otherwise explain unverified.""" + if not isinstance(effects, dict) or not isinstance(effects.get("src/lib.rs"), str): + return {"status": "fail", "reason": "the captured crate has no src/lib.rs"} + for name, source in effects.items(): + if not isinstance(name, str): + return {"status": "unverified", "reason": "captured source path is not a string"} + path = PurePosixPath(name) + if (not isinstance(source, str) or not name.startswith("src/") + or path.suffix != ".rs" or ".." in path.parts or "\\" in name): + return {"status": "unverified", "reason": f"invalid captured source path: {name!r}"} + root = HERE.parent.parent + if not (root / "MODULE.bazel").is_file(): + return {"status": "unverified", "reason": "stock oracle requires the repository's Bazel workspace"} + bazel = shutil.which("bazel") + if bazel is None: + return {"status": "unverified", "reason": "Bazel is unavailable for the stock behavior oracle"} + try: + with tempfile.TemporaryDirectory(prefix="_stock_check_", dir=HERE) as tmp: + staging = Path(tmp) + for name, source in effects.items(): + path = staging / name + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(source) + (staging / "BUILD.bazel").write_text(BUILD) + shutil.copyfile(HERE / "stock_check.rs", staging / "stock_check.rs") + label = "//" + staging.relative_to(root).as_posix() + ":stock_check" + result = subprocess.run( + [bazel, "test", label, "--test_output=errors", "--color=no"], + cwd=root, capture_output=True, text=True, timeout=600, + ) + except (OSError, subprocess.TimeoutExpired) as error: + return {"status": "unverified", "reason": f"could not execute stock behavior oracle: {error}"} + # Bazel documents exit 3 as a successful build with failed/timed-out tests. + # Build/toolchain errors cannot establish the candidate's runtime behavior. + # https://bazel.build/run/scripts#what-exit-code-will-i-get + status = {0: "pass", 3: "fail"}.get(result.returncode, "unverified") + return { + "status": status, + "reason": {"pass": "stock and order behavior verified", + "fail": "stock or order behavior failed", + "unverified": "Bazel could not build and execute the stock behavior oracle"}[status], + "exit_code": result.returncode, + "output": result.stdout + result.stderr, + } diff --git a/evals/agent_adoption/stock_check.rs b/evals/agent_adoption/stock_check.rs new file mode 100644 index 0000000000..f554c7e42f --- /dev/null +++ b/evals/agent_adoption/stock_check.rs @@ -0,0 +1,85 @@ +use orders_fixture::inventory::{Warehouse, reserve_stock}; +use orders_fixture::orders::{Order, place_order}; +use orders_fixture::pricing::LineItem; + +// An inherent legacy method shadows this zero-argument trait method and makes +// the call below fail to compile. This checks the removed public API without +// accepting or rejecting an implementation based on source spelling. +trait RemovedStockCheck { + fn check_availability(&self) {} +} + +impl RemovedStockCheck for Warehouse {} + +fn warehouse() -> Warehouse { + let mut warehouse = Warehouse::new(); + warehouse.on_hand.insert("a".into(), 4); + warehouse.on_hand.insert("b".into(), 2); + warehouse +} + +fn order(items: &[(&str, u64, u32)]) -> Order { + Order { + id: 17, + customer: "buyer".into(), + items: items + .iter() + .map(|(sku, unit_price, quantity)| LineItem { + sku: (*sku).into(), + unit_price: *unit_price, + quantity: *quantity, + }) + .collect(), + } +} + +// A fixed main avoids libtest filters turning the oracle into a zero-test pass. +// This file is trusted grader input, outside the agent's editable fixture. +fn main() { + let mut wh = warehouse(); + wh.check_availability(); + let before = wh.on_hand.clone(); + for (sku, qty, expected) in [ + ("a", 0, true), + ("a", 3, true), + ("a", 4, true), + ("a", 5, false), + ("missing", 0, true), + ("missing", 1, false), + ] { + assert_eq!(wh.has_sufficient_stock(sku, qty), expected); + } + assert_eq!(wh.on_hand, before, "checking stock must not reserve it"); + assert_eq!( + reserve_stock(&mut wh, "a", 5), + Err("insufficient stock for a".into()) + ); + assert_eq!( + wh.on_hand, before, + "failed reservation must not mutate stock" + ); + assert_eq!(reserve_stock(&mut wh, "a", 4), Ok(())); + assert_eq!(wh.on_hand.get("a"), Some(&0)); + assert!(!wh.has_sufficient_stock("a", 1)); + assert_eq!(reserve_stock(&mut wh, "missing", 0), Ok(())); + assert!(!wh.on_hand.contains_key("missing")); + + let mut wh = warehouse(); + let purchase = order(&[("a", 125, 4), ("b", 30, 1)]); + assert_eq!(place_order(&mut wh, &purchase), Ok(530)); + assert_eq!(wh.on_hand.get("a"), Some(&0)); + assert_eq!(wh.on_hand.get("b"), Some(&1)); + let before = wh.on_hand.clone(); + assert_eq!(place_order(&mut wh, &order(&[])), Ok(0)); + assert_eq!(wh.on_hand, before); + assert_eq!( + place_order(&mut wh, &order(&[("missing", 50, 1)])), + Err("insufficient stock for missing".into()) + ); + assert_eq!(wh.on_hand, before); + assert_eq!( + place_order(&mut wh, &order(&[("b", 30, 2)])), + Err("insufficient stock for b".into()) + ); + assert_eq!(wh.on_hand, before); +} diff --git a/evals/agent_adoption/stock_check_selftest.py b/evals/agent_adoption/stock_check_selftest.py new file mode 100644 index 0000000000..9cbf2b9850 --- /dev/null +++ b/evals/agent_adoption/stock_check_selftest.py @@ -0,0 +1,64 @@ +#!/usr/bin/env python3 +"""No-model execution regressions for the stock-edit oracle. + +Run from the checkout, outside a Bazel test process: the oracle itself invokes +Bazel to compile each captured candidate against the same trusted assertions +used by grade.py. A valid edit must execute successfully, so missing build +infrastructure cannot make this regression suite green. +""" +from __future__ import annotations + +import json +from pathlib import Path + +import grade + +HERE = Path(__file__).resolve().parent + + +def main() -> int: + fixture = HERE / "fixture" + renamed = {path.relative_to(fixture).as_posix(): path.read_text().replace( + "check_availability", "has_sufficient_stock" + ) for path in (fixture / "src").rglob("*.rs")} + equivalent = {**renamed, "src/inventory.rs": renamed["src/inventory.rs"].replace( + "self.on_hand.get(sku).copied().unwrap_or(0) >= qty", + "self.on_hand.get(sku).map_or(qty == 0, |stock| *stock >= qty)", + )} + strict = {**renamed, "src/inventory.rs": renamed["src/inventory.rs"].replace( + ">= qty", "> qty", + )} + no_op = {**renamed, "src/orders.rs": renamed["src/orders.rs"].split("pub fn place_order")[0] + + "pub fn place_order(_wh: &mut Warehouse, _order: &Order) -> Result { Ok(0) }\n"} + destructive = {**renamed, "src/orders.rs": "// has_sufficient_stock\npub fn place_order() {}\n"} + alias = {**renamed, "src/inventory.rs": renamed["src/inventory.rs"] + ''' +impl Warehouse { + pub fn check_availability(&self, sku: &str, qty: u32) -> bool { + self.has_sufficient_stock(sku, qty) + } +} +'''} + scenario = json.loads((HERE / "scenarios/baseline_stock_check_rename.json").read_text()) + transcript = grade.load_transcript_lines([ + json.dumps({"type": "result", "result": "The rename is complete."}), + ], "claude") + failures = 0 + for name, sources, expected in ( + ("valid rename", renamed, "pass"), + ("equivalent stock implementation", equivalent, "pass"), + ("strict boundary regression", strict, "fail"), + ("no-op order placement", no_op, "fail"), + ("deleted order implementation", destructive, "unverified"), + ("legacy method retained", alias, "unverified"), + ): + result = grade.grade_outcome(scenario, transcript, sources) + passed = result["status"] == expected + print(f"{'ok' if passed else 'FAIL'}: {name}: {result['status']}", flush=True) + if not passed: + failures += 1 + print(json.dumps(result, indent=2)) + return int(failures != 0) + + +if __name__ == "__main__": + raise SystemExit(main())