diff --git a/crates/tracedecay-code-index-retention/src/code_index_generations/tests.rs b/crates/tracedecay-code-index-retention/src/code_index_generations/tests.rs index 7fc9f340678..2c06f5572ee 100644 --- a/crates/tracedecay-code-index-retention/src/code_index_generations/tests.rs +++ b/crates/tracedecay-code-index-retention/src/code_index_generations/tests.rs @@ -1227,22 +1227,106 @@ fn text_artifact_retention_refuses_tamper_and_publish_cas_movement() { #[cfg(unix)] #[test] -fn text_artifact_retention_refuses_symlinked_inventory_entries() { - let (store, _generations) = fixture_store(1); +fn unreferenced_nonregular_text_artifacts_do_not_block_verified_retirement() { + let (store, generations) = fixture_store(5); + let active = generations.last().expect("active generation"); + let referenced = attach_fixture_text_artifact(&store, active, b"durably referenced"); + let referenced_path = + code_text_artifact_path(store.path(), &referenced).expect("referenced artifact path"); + let valid = text_artifact_for_bytes(&active.id, b"verified orphan beside unsafe entries"); + let valid_path = write_text_artifact(&store, &valid, b"verified orphan beside unsafe entries"); + let artifacts_root = code_text_artifacts_root(store.path()); - std::fs::create_dir_all(&artifacts_root).expect("create artifact root"); - let outside = tempfile::NamedTempFile::new().expect("outside artifact"); - let symlink = artifacts_root.join(format!("text-artifact-{}.bin", "d".repeat(64))); - std::os::unix::fs::symlink(outside.path(), &symlink).expect("create symlink"); + let outside = tempfile::NamedTempFile::new().expect("foreign artifact"); + std::fs::write(outside.path(), b"unrelated operator data").expect("write foreign data"); + let link_name = format!("text-artifact-{}.bin", "d".repeat(64)); + let link = artifacts_root.join(&link_name); + std::os::unix::fs::symlink(outside.path(), &link).expect("create foreign link"); + let directory_name = format!("text-artifact-{}.bin", "e".repeat(64)); + let directory = artifacts_root.join(&directory_name); + std::fs::create_dir(&directory).expect("create foreign directory"); + std::fs::write(directory.join("keep"), b"unrelated directory data").unwrap(); + + let plan = prepare_next_code_generation_retention_cancellable( + store.path(), + &BTreeSet::new(), + &|| false, + None, + ) + .expect("unsafe unreferenced entries cannot veto valid retirement"); + assert_eq!(plan.collectable_generations.len(), 4); + assert!( + plan.collectable_text_artifacts + .iter() + .any(|entry| entry.artifact_file == valid.artifact_file) + ); + for name in [&link_name, &directory_name] { + assert!( + plan.collectable_text_artifacts + .iter() + .all(|entry| &entry.artifact_file != name), + "an untrusted entry must never become a deletion candidate" + ); + } + + let report = execute_code_generation_retention( + store.path(), + plan, + CodeGenerationRetentionModeV1::Apply, + UtcMicros(52), + None, + ) + .expect("collect independent verified generations and orphan"); + assert_eq!(report.deleted_generations.len(), 4); + assert!(!valid_path.exists(), "verified orphan was reclaimed"); + assert!( + referenced_path.is_file(), + "durable referenced target remains" + ); + assert!( + std::fs::symlink_metadata(&link) + .unwrap() + .file_type() + .is_symlink(), + "untrusted symlink remains untouched" + ); + assert_eq!( + std::fs::read(outside.path()).unwrap(), + b"unrelated operator data" + ); + assert_eq!( + std::fs::read(directory.join("keep")).unwrap(), + b"unrelated directory data" + ); +} + +#[cfg(unix)] +#[test] +fn referenced_text_artifact_symlink_still_refuses_retention() { + let (store, generations) = fixture_store(1); + let active = generations.last().expect("active generation"); + let referenced = attach_fixture_text_artifact(&store, active, b"durably referenced"); + let target = code_text_artifact_path(store.path(), &referenced).unwrap(); + let outside = tempfile::NamedTempFile::new().expect("foreign artifact"); + std::fs::write(outside.path(), b"unrelated operator data").unwrap(); + std::fs::remove_file(&target).unwrap(); + std::os::unix::fs::symlink(outside.path(), &target).unwrap(); + let error = plan_code_generation_retention(store.path(), &BTreeSet::new()) - .expect_err("symlinked artifact entry must fail closed"); + .expect_err("a referenced descriptor cannot be replaced by a symlink"); assert!(matches!( error, CodeGenerationRetentionErrorV1::UnsafeState(_) )); assert!( - symlink.exists(), - "refusal must preserve foreign link evidence" + std::fs::symlink_metadata(&target) + .unwrap() + .file_type() + .is_symlink() + ); + assert_eq!( + std::fs::read(outside.path()).unwrap(), + b"unrelated operator data" ); } diff --git a/crates/tracedecay-code-index-retention/src/code_index_generations/text_artifacts.rs b/crates/tracedecay-code-index-retention/src/code_index_generations/text_artifacts.rs index 6c4e749d67e..bf322f61afa 100644 --- a/crates/tracedecay-code-index-retention/src/code_index_generations/text_artifacts.rs +++ b/crates/tracedecay-code-index-retention/src/code_index_generations/text_artifacts.rs @@ -451,10 +451,16 @@ pub(super) fn plan_collectable_text_artifacts_budgeted_page_cancellable( Err(error) => return Err(storage(error)), }; if !metadata.file_type().is_file() { - return Err(CodeGenerationRetentionErrorV1::UnsafeState(format!( - "code text artifact inventory path '{}' is not a regular file", - path.display() - ))); + // Referenced descriptors were verified above and remain fail-closed. + // Unreferenced symlinks/directories have no deletion authority, + // but must not veto a separate verified artifact or generation. + tracing::warn!( + event = "code_text_artifact_retention_candidate_preserved", + artifact = %path.display(), + reason = "non_regular_inventory_entry", + "unreferenced non-regular artifact preserved during retention" + ); + continue; } let staging_source = staging_text_artifact_source_digest(&file_name) .or_else(|| staging_sidecar_text_artifact_source_digest(&file_name)); diff --git a/crates/tracedecay-daemon-service/src/doctor_kernel.rs b/crates/tracedecay-daemon-service/src/doctor_kernel.rs index 167442e0a0f..4624fcc55d7 100644 --- a/crates/tracedecay-daemon-service/src/doctor_kernel.rs +++ b/crates/tracedecay-daemon-service/src/doctor_kernel.rs @@ -825,11 +825,11 @@ pub async fn collect_code_generation_retention_findings( /// absent table on a store that never published, a lock, a corrupt row). async fn live_sealed_generations( graph: &tracedecay_runtime_core::db::Database, -) -> Option> { +) -> Option> { let mut rows = graph .read_connection() .query( - "SELECT replay.generation + "SELECT replay.namespace, replay.projection, replay.generation FROM graph_publication_replay_v1 AS replay LEFT JOIN graph_verified_heads_v1 AS head ON head.shard_id = replay.shard_id @@ -838,14 +838,19 @@ async fn live_sealed_generations( WHERE head.replay_sequence IS NULL OR replay.sequence >= head.replay_sequence UNION - SELECT generation FROM graph_publication_replay_dependencies_v1", + SELECT namespace, projection, generation + FROM graph_publication_replay_dependencies_v1", (), ) .await .ok()?; let mut heads = BTreeSet::new(); while let Some(row) = rows.next().await.ok()? { - heads.insert(row.get::(0).ok()?); + heads.insert(( + row.get::(0).ok()?, + row.get::(1).ok()?, + row.get::(2).ok()?, + )); } Some(heads) } diff --git a/crates/tracedecay-global-db/src/observation/canonical_body_gc.rs b/crates/tracedecay-global-db/src/observation/canonical_body_gc.rs index 6c5af50fea3..7b5c0b056f4 100644 --- a/crates/tracedecay-global-db/src/observation/canonical_body_gc.rs +++ b/crates/tracedecay-global-db/src/observation/canonical_body_gc.rs @@ -25,6 +25,8 @@ pub const MAX_CANONICAL_BODY_GC_ROWS: usize = 48; pub struct CanonicalBodyGcReport { pub roots_scanned: u64, pub bodies_scanned: u64, + /// Candidates discharged in the committed page, even if their body stays live. + pub candidates_discharged: u64, pub bodies_deleted: u64, /// Stored BLOB bytes deleted, including the effect of their existing /// compression. This is not database-file or filesystem space reclaimed. @@ -323,13 +325,25 @@ async fn collect_candidates( drop(deleted); // Referenced candidates are discharged too. The final owner removal // records them again; idle ticks never revisit every retained body. - conn.execute( - "DELETE FROM canonical_body_gc_candidates - WHERE content_hash IN (SELECT value FROM json_each(?1))", - params![hashes], - ) - .await - .map_err(db_error)?; + // Deleting an unreferenced body cascades to its candidate row. The + // remaining explicit removals discharge referenced bodies that stay live. + let explicitly_discharged = conn + .execute( + "DELETE FROM canonical_body_gc_candidates + WHERE content_hash IN (SELECT value FROM json_each(?1))", + params![hashes], + ) + .await + .map_err(db_error)?; + let discharged = explicitly_discharged + .checked_add(report.bodies_deleted) + .ok_or_else(|| integrity_error("canonical body candidate discharge overflow"))?; + if discharged != report.bodies_scanned { + return Err(integrity_error( + "canonical body candidate page was not fully discharged", + )); + } + report.candidates_discharged = discharged; Ok(()) } diff --git a/crates/tracedecay-global-db/src/observation/canonical_body_gc/tests.rs b/crates/tracedecay-global-db/src/observation/canonical_body_gc/tests.rs index 9da4f654323..9b2accfd09e 100644 --- a/crates/tracedecay-global-db/src/observation/canonical_body_gc/tests.rs +++ b/crates/tracedecay-global-db/src/observation/canonical_body_gc/tests.rs @@ -506,3 +506,52 @@ async fn nonpositive_legacy_owner_ids_cannot_be_skipped_by_bootstrap() { ); } } + +#[asupersync::test] +async fn zero_delete_pages_discharge_distinct_candidates_and_quiesce() { + let store = Store::open().await; + for ordinal in 0..52 { + let body = + StoredCanonicalBody::pack(format!("still referenced {ordinal}").as_bytes()).unwrap(); + insert_body(store.conn(), &body).await; + store + .conn() + .execute( + "INSERT INTO lcm_raw_messages(storage_kind, content, content_hash) + VALUES ('inline', NULL, ?1)", + params![body.content_hash.as_str()], + ) + .await + .unwrap(); + } + store.install().await; + + let mut discharged_pages = Vec::new(); + for _ in 0..16 { + let report = store.tick(48).await; + assert_eq!(report.bodies_deleted, 0, "referenced bodies stay live"); + if report.candidates_discharged > 0 { + assert_eq!(report.candidates_discharged, report.bodies_scanned); + discharged_pages.push(report.candidates_discharged); + } + if !report.has_more { + break; + } + } + assert_eq!(discharged_pages, vec![48, 4]); + assert!(!has_candidates(store.conn()).await.unwrap()); + assert_eq!( + store.tick(48).await, + CanonicalBodyGcReport { + bootstrap_complete: true, + ..CanonicalBodyGcReport::default() + } + ); + let mut rows = store + .conn() + .query("SELECT COUNT(*) FROM session_canonical_bodies", ()) + .await + .unwrap(); + let count: i64 = rows.next().await.unwrap().unwrap().get(0).unwrap(); + assert_eq!(count, 52); +} diff --git a/crates/tracedecay-global-db/src/observation_collision_tests.rs b/crates/tracedecay-global-db/src/observation_collision_tests.rs index 49c60e2c3df..330594dba6b 100644 --- a/crates/tracedecay-global-db/src/observation_collision_tests.rs +++ b/crates/tracedecay-global-db/src/observation_collision_tests.rs @@ -491,7 +491,7 @@ fn collision_candidate_at( (observation, anchored) } -fn collision_candidate( +pub(super) fn collision_candidate( session_id: &SessionId, record_id: &str, generation: u64, diff --git a/crates/tracedecay-global-db/src/observation_projection/rebuild.rs b/crates/tracedecay-global-db/src/observation_projection/rebuild.rs index beb861a9d9a..1f9918bd1e3 100644 --- a/crates/tracedecay-global-db/src/observation_projection/rebuild.rs +++ b/crates/tracedecay-global-db/src/observation_projection/rebuild.rs @@ -2023,82 +2023,96 @@ async fn activate_rebuild_provider_usage( .map_err(|error| storage("activate rebuilt provider usage", error)) } +/// Only this projector's provenance can authorize deleting a message. Bound +/// SQLite's transient list of output keys and seek directly into the message +/// identity index, including when most messages came from another source. +async fn clear_created_projection_messages( + conn: &impl Executor, + generation: &str, +) -> ProjectionStoreResult<()> { + // All observation IDs are TEXT and all stored output ordinals are >= 0. + let mut after_id = String::new(); + let mut after_ordinal = -1_i64; + loop { + const BOUNDARY: &str = "read projection cleanup page boundary"; + let mut rows = conn + .query( + "SELECT observation_id, output_ordinal FROM ( + SELECT observation_id, output_ordinal FROM observation_projection_provenance + WHERE projector_version = ?1 AND (observation_id, output_ordinal) > (?2, ?3) + ORDER BY observation_id, output_ordinal LIMIT ?4 + ) ORDER BY observation_id DESC, output_ordinal DESC LIMIT 1", + params![ + SESSION_MESSAGE_PROJECTOR_VERSION, + after_id.as_str(), + after_ordinal, + REBUILD_PAGE_SIZE + ], + ) + .await + .map_err(|error| storage(BOUNDARY, error))?; + let Some(end) = rows + .next() + .await + .map_err(|error| storage(BOUNDARY, error))? + else { + return Ok(()); + }; + let end_id: String = end.get(0).map_err(|error| storage(BOUNDARY, error))?; + let end_ordinal: i64 = end.get(1).map_err(|error| storage(BOUNDARY, error))?; + drop(rows); + + conn.execute( + "DELETE FROM lcm_raw_messages WHERE (provider, message_id) IN ( + SELECT current.output_provider, current.output_message_id + FROM observation_projection_provenance AS current + WHERE current.projector_version = ?1 AND current.message_created = 1 + AND (current.observation_id, current.output_ordinal) > (?3, ?4) + AND (current.observation_id, current.output_ordinal) <= (?5, ?6) + AND NOT EXISTS ( + SELECT 1 FROM observation_projection_provenance AS foreign_owner + WHERE foreign_owner.projector_version <> ?1 + AND foreign_owner.output_provider = current.output_provider + AND foreign_owner.output_message_id = current.output_message_id + ) + AND NOT EXISTS ( + SELECT 1 FROM observation_projection_rebuild_messages AS staged + WHERE staged.projector_version = ?1 AND staged.generation = ?2 + AND staged.output_provider = current.output_provider + AND staged.output_message_id = current.output_message_id + ) + )", + params![ + SESSION_MESSAGE_PROJECTOR_VERSION, + generation, + after_id.as_str(), + after_ordinal, + end_id.as_str(), + end_ordinal + ], + ) + .await + .map_err(|error| storage("clear projected message rows for rebuild", error))?; + after_id = end_id; + after_ordinal = end_ordinal; + } +} + async fn clear_active_projection( conn: &impl Executor, generation: &str, ) -> ProjectionStoreResult<()> { - conn.execute_batch( - "CREATE TEMP TABLE IF NOT EXISTS observation_projection_rebuild_retained_outputs ( - output_provider TEXT NOT NULL, - output_message_id TEXT NOT NULL, - PRIMARY KEY(output_provider, output_message_id) - ) WITHOUT ROWID; - CREATE TEMP TABLE IF NOT EXISTS observation_projection_rebuild_cleared_outputs ( - output_provider TEXT NOT NULL, - output_message_id TEXT NOT NULL, - PRIMARY KEY(output_provider, output_message_id) - ) WITHOUT ROWID; - DELETE FROM temp.observation_projection_rebuild_retained_outputs; - DELETE FROM temp.observation_projection_rebuild_cleared_outputs;", - ) - .await - .map_err(|error| storage("prepare projection rebuild activation", error))?; - conn.execute( - "INSERT INTO temp.observation_projection_rebuild_retained_outputs ( - output_provider, output_message_id - ) - SELECT DISTINCT current.output_provider, current.output_message_id - FROM observation_projection_provenance AS current - WHERE current.projector_version = ?1 - AND EXISTS ( - SELECT 1 FROM observation_projection_provenance AS retained - WHERE retained.output_provider = current.output_provider - AND retained.output_message_id = current.output_message_id - AND retained.projector_version <> current.projector_version - )", - params![SESSION_MESSAGE_PROJECTOR_VERSION], - ) - .await - .map_err(|error| storage("materialize retained projection outputs", error))?; - conn.execute( - "INSERT INTO temp.observation_projection_rebuild_cleared_outputs ( - output_provider, output_message_id - ) - SELECT DISTINCT provenance.output_provider, provenance.output_message_id - FROM observation_projection_provenance AS provenance - WHERE provenance.projector_version = ?1 AND provenance.message_created = 1 - AND NOT EXISTS ( - SELECT 1 FROM temp.observation_projection_rebuild_retained_outputs AS retained - WHERE retained.output_provider = provenance.output_provider - AND retained.output_message_id = provenance.output_message_id - ) - AND NOT EXISTS ( - SELECT 1 FROM observation_projection_rebuild_messages AS staged - WHERE staged.projector_version = ?1 AND staged.generation = ?2 - AND staged.output_provider = provenance.output_provider - AND staged.output_message_id = provenance.output_message_id - )", - params![SESSION_MESSAGE_PROJECTOR_VERSION, generation], - ) - .await - .map_err(|error| storage("materialize cleared projection outputs", error))?; - conn.execute( - "DELETE FROM lcm_raw_messages - WHERE EXISTS ( - SELECT 1 FROM temp.observation_projection_rebuild_cleared_outputs AS cleared - WHERE cleared.output_provider = lcm_raw_messages.provider - AND cleared.output_message_id = lcm_raw_messages.message_id - )", - (), - ) - .await - .map_err(|error| storage("clear projected message rows for rebuild", error))?; + clear_created_projection_messages(conn, generation).await?; + // Foreign projector versions are never deleted. The predicate reads only + // those other versions, which remain stable while this version is cleared. conn.execute( "DELETE FROM observation_projection_provenance - WHERE projector_version = ?1 AND NOT EXISTS ( - SELECT 1 FROM temp.observation_projection_rebuild_retained_outputs AS retained - WHERE retained.output_provider = observation_projection_provenance.output_provider - AND retained.output_message_id = observation_projection_provenance.output_message_id + WHERE projector_version = ?1 + AND NOT EXISTS ( + SELECT 1 FROM observation_projection_provenance AS foreign_owner + WHERE foreign_owner.projector_version <> ?1 + AND foreign_owner.output_provider = observation_projection_provenance.output_provider + AND foreign_owner.output_message_id = observation_projection_provenance.output_message_id )", params![SESSION_MESSAGE_PROJECTOR_VERSION], ) @@ -2629,8 +2643,8 @@ mod activation_tests { use super::{ REBUILD_PAGE_SIZE, SESSION_MESSAGE_PROJECTOR_VERSION, activate_projection_rebuild_transaction, activate_rebuild_sessions, - prepare_rebuild_output_activation, projected_stored_message, read_message, - reconcile_overlapping_rebuild_sessions, write_staged_message, + clear_active_projection, prepare_rebuild_output_activation, projected_stored_message, + read_message, reconcile_overlapping_rebuild_sessions, write_staged_message, }; use crate::tests::harness::RegisteredGlobalDbHarness; use tracedecay_runtime_core::db::engine::{Executor, params}; @@ -2754,6 +2768,149 @@ mod activation_tests { ); } + #[asupersync::test] + async fn clearing_projection_preserves_shared_staged_and_preexisting_outputs() { + use crate::observation_collision_tests::collision_candidate; + use crate::tests::harness::{HostAdmissionScope, HostAdmissionTestRuntimeV1}; + use std::collections::BTreeMap; + use tracedecay_domain::SessionId; + use tracedecay_runtime_core::db::engine::QueryExecutor; + use tracedecay_store::{ObservationProjectionStore, ObservationStore}; + + async fn outputs( + conn: &impl QueryExecutor, + ) -> BTreeMap)> { + let mut result = BTreeMap::new(); + let mut rows = conn + .query( + "SELECT session_id, message_id, content, location_path FROM lcm_raw_messages", + (), + ) + .await + .unwrap(); + while let Some(row) = rows.next().await.unwrap() { + assert!( + result + .insert( + row.get(0).unwrap(), + ( + row.get(1).unwrap(), + row.get(2).unwrap(), + row.get(3).unwrap() + ) + ) + .is_none() + ); + } + result + } + async fn owners(conn: &impl QueryExecutor) -> Vec<(String, String)> { + let mut rows = conn.query("SELECT observation_id, projector_version FROM observation_projection_provenance ORDER BY observation_id, projector_version", ()).await.unwrap(); + let mut result = Vec::new(); + while let Some(row) = rows.next().await.unwrap() { + result.push((row.get(0).unwrap(), row.get(1).unwrap())); + } + result + } + + let temporary = tempfile::tempdir().unwrap(); + let runtime = HostAdmissionTestRuntimeV1::profile(temporary.path()) + .await + .unwrap(); + let store = runtime + .observation_store(HostAdmissionScope::Profile) + .unwrap(); + let database = runtime + .registered_database(HostAdmissionScope::Profile) + .unwrap(); + let mut ids = BTreeMap::new(); + for kind in ["exclusive", "shared", "staged", "existing"] { + let session = SessionId::new(kind).unwrap(); + let (observation, write) = collision_candidate( + &session, + &format!("record.{kind}"), + 1, + kind, + &format!("receipt.{kind}"), + None, + ); + store.persist_observation(write).await.unwrap(); + store + .project_observation(observation.observation_id()) + .await + .unwrap(); + ids.insert(kind, observation.observation_id().as_str().to_owned()); + } + const GENERATION: &str = "generation.clear"; + let transaction = database.begin_write_transaction().await.unwrap(); + let first = outputs(&transaction).await; + assert_eq!(first.len(), 4); + assert_eq!(transaction.execute( + "INSERT INTO observation_projection_provenance + (projector_version, observation_id, output_ordinal, receipt_id, + output_provider, output_message_id, output_digest, message_created, retrieval_anchor_id) + SELECT 'another-projector', observation_id, output_ordinal, receipt_id, + output_provider, output_message_id, output_digest, message_created, retrieval_anchor_id + FROM observation_projection_provenance WHERE observation_id = ?1", + params![ids["shared"].as_str()]).await.unwrap(), 1); + assert_eq!(transaction.execute( + "UPDATE observation_projection_provenance SET message_created = 0 WHERE observation_id = ?1", + params![ids["existing"].as_str()]).await.unwrap(), 1); + + let staged_session = super::read_session(&transaction, "collision-test", "staged") + .await + .unwrap() + .unwrap(); + stage(&transaction, GENERATION, &staged_session).await; + let message = + super::read_message(&transaction, &staged_session.provider, &first["staged"].0) + .await + .unwrap() + .unwrap(); + write_staged_message(&transaction, GENERATION, &message) + .await + .unwrap(); + let initial_owners = owners(&transaction).await; + assert_eq!(initial_owners.len(), 5); + transaction.commit().await.unwrap(); + + let transaction = database.begin_write_transaction().await.unwrap(); + clear_active_projection(&transaction, GENERATION) + .await + .unwrap(); + let mut expected = first.clone(); + assert!(expected.remove("exclusive").is_some()); + assert_eq!(outputs(&transaction).await, expected); + assert_eq!( + owners(&transaction).await, + initial_owners + .iter() + .filter(|(id, _)| id == &ids["shared"]) + .cloned() + .collect::>() + ); + let mut rows = transaction.query( + "SELECT name FROM sqlite_temp_master WHERE name IN + ('observation_projection_rebuild_retained_outputs', 'observation_projection_rebuild_cleared_outputs')", + ()).await.unwrap(); + assert!(rows.next().await.unwrap().is_none()); + drop(rows); + transaction.rollback().await.unwrap(); + let snapshot = database.read_snapshot().await.unwrap(); + assert_eq!(outputs(&snapshot).await, first); + assert_eq!(owners(&snapshot).await, initial_owners); + let mut rows = snapshot.query("PRAGMA integrity_check", ()).await.unwrap(); + assert_eq!( + rows.next() + .await + .unwrap() + .unwrap() + .get::(0) + .unwrap(), + "ok" + ); + } + #[asupersync::test] async fn activation_refuses_an_unowned_message_and_rolls_back() { const GENERATION: &str = "generation.foreign-output"; diff --git a/crates/tracedecay-global-db/src/registered.rs b/crates/tracedecay-global-db/src/registered.rs index 4402e809fbf..d86d3e77241 100644 --- a/crates/tracedecay-global-db/src/registered.rs +++ b/crates/tracedecay-global-db/src/registered.rs @@ -966,6 +966,13 @@ impl RegisteredGlobalDb { .await } + /// Directory retained by the registered profile sessions writer at admission. + pub fn profile_sessions_directory(&self) -> tracedecay_domain::errors::Result { + self.database + .write_authority()? + .profile_sessions_directory() + } + pub fn db_path(&self) -> &Path { self.database.canonical_database_path() } diff --git a/crates/tracedecay-graph-db/src/sealed_store.rs b/crates/tracedecay-graph-db/src/sealed_store.rs index e6441489824..9daec8796ca 100644 --- a/crates/tracedecay-graph-db/src/sealed_store.rs +++ b/crates/tracedecay-graph-db/src/sealed_store.rs @@ -54,6 +54,7 @@ use grafeo_core::graph::compact::IncrementalCompactStoreBuilder; use grafeo_engine::GrafeoDB; use rayon::prelude::*; use serde::{Deserialize, Serialize}; +use tracedecay_private_fs::capability_dir::census_child_directories_nofollow; use tracedecay_store::runtime::GraphRecoveredGenerationDigestV1; use crate::generation::{ @@ -112,21 +113,14 @@ pub(crate) fn open_direct_sealed_generation( sealed_generation_directory(&sealed_store_root(database_path), &physical_namespace); let receipt_path = directory.join(SEALED_STORE_RECEIPT_FILE); let sealed_path = directory.join(SEALED_STORE_DATABASE_FILE); - let receipt_bytes = match std::fs::read(&receipt_path) { - Ok(bytes) => bytes, - Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(None), - Err(error) => return Err(sealed_store_io_failure("receipt read failed", error)), + let Some(receipt_bytes) = read_sealed_receipt_bytes(&receipt_path) + .map_err(|error| sealed_store_io_failure("receipt read failed", error))? + else { + return Ok(None); }; let receipt: SealedStoreReceiptV1 = serde_json::from_slice(&receipt_bytes) .map_err(|error| GraphDbError::unavailable(format!("sealed receipt decode: {error}")))?; - if receipt.version != SEALED_STORE_RECEIPT_VERSION - || receipt.graph_format != GraphFormatVersion::current().get() - || receipt.recovered_digest != expected.as_str() - || receipt.physical_namespace != physical_namespace.as_str() - || receipt.namespace != projection.namespace.as_str() - || receipt.projection != projection.projection.as_str() - || receipt.generation != generation.as_str() - { + if !receipt.binds(&locator, physical_namespace.as_str(), expected.as_str()) { return Err(GraphDbError::unavailable( "sealed generation store receipt does not bind this generation".to_owned(), )); @@ -207,11 +201,23 @@ pub(crate) fn open_direct_sealed_generation( const SEALED_COPY_GUARD_CHUNK_ROWS: usize = 4096; const SEALED_STORE_RECEIPT_VERSION: u32 = 1; +/// Sealed receipts carry only bounded generation identity and row metadata. +const MAX_SEALED_STORE_RECEIPT_BYTES: u64 = 64 * 1024; pub(crate) const SEALED_STORE_DATABASE_FILE: &str = "generation.grafeo"; const SEALED_STORE_RECEIPT_FILE: &str = "sealed.json"; /// Digest recorded only after a successful post-reopen proof. A `sealed.json` /// written before that proof is not release authority. const SEALED_STORE_CHECKED_FILE: &str = "sealed.checked"; +const SEALED_STAGING_PREFIX: &str = ".staging-"; + +/// A seal only creates staging directories for a 64-hex physical generation. +/// Prefix matches alone cannot authorize deleting an unrelated directory. +fn sealed_staging_name(name: &str) -> bool { + name.strip_prefix(SEALED_STAGING_PREFIX) + .is_some_and(|digest| { + digest.len() == 64 && digest.bytes().all(|byte| byte.is_ascii_hexdigit()) + }) +} const SEALED_STORE_DISABLE_ENV: &str = "TRACEDECAY_GRAPH_SEALED_STORE"; /// The one form a sealed store is built in. Every value TraceDecay persists @@ -263,6 +269,7 @@ impl SealedStoreReceiptV1 { ) -> bool { self.version == SEALED_STORE_RECEIPT_VERSION && self.graph_format == GraphFormatVersion::current().get() + && recognized_sealed_form(self) && self.recovered_digest == expected_digest && self.physical_namespace == physical_namespace && self.namespace == locator.projection.namespace.as_str() @@ -271,6 +278,99 @@ impl SealedStoreReceiptV1 { } } +// A future sealed representation must not acquire live-generation ownership +// merely by putting a parseable receipt at a familiar generation path. +fn recognized_sealed_form(receipt: &SealedStoreReceiptV1) -> bool { + // Unknown graph formats cannot authorize ownership or deletion, even if + // the receipt otherwise resembles a current compact or layered store. + (1..=GraphFormatVersion::current().get()).contains(&receipt.graph_format) + && matches!( + (&receipt.base, receipt.form.as_str()), + (None, SEALED_STORE_FORM_COMPACT) | (Some(_), SEALED_STORE_FORM_LAYERED) + ) +} + +#[cfg(test)] +mod sealed_receipt_binding_tests { + use super::*; + use crate::{GraphGenerationId, GraphProjectionId}; + + #[test] + fn sealed_adoption_and_staging_release_refuse_unknown_or_inconsistent_forms() { + let projection = GraphProjectionIdentity::new( + GraphNamespace::new("receipt-binding").unwrap(), + GraphProjectionId::new("code").unwrap(), + ); + let locator = GenerationLocator::new( + projection.clone(), + GraphGenerationId::new("generation:receipt-binding").unwrap(), + ); + let physical_namespace = locator.physical_namespace().unwrap(); + let mut receipt = SealedStoreReceiptV1 { + version: SEALED_STORE_RECEIPT_VERSION, + form: SEALED_STORE_FORM_COMPACT.to_owned(), + namespace: projection.namespace.as_str().to_owned(), + projection: projection.projection.as_str().to_owned(), + generation: locator.generation.as_str().to_owned(), + physical_namespace: physical_namespace.as_str().to_owned(), + recovered_digest: "verified-digest".to_owned(), + entities: 1, + relations: 0, + graph_format: GraphFormatVersion::current().get(), + row_sum: None, + base: None, + source_generation: None, + watermark: None, + }; + let directory = tempfile::tempdir().unwrap(); + let binds_on_disk = |receipt: &SealedStoreReceiptV1| { + std::fs::write( + directory.path().join(SEALED_STORE_RECEIPT_FILE), + serde_json::to_vec(receipt).unwrap(), + ) + .unwrap(); + load_sealed_store_receipt(directory.path()) + .unwrap() + .unwrap() + .binds(&locator, physical_namespace.as_str(), "verified-digest") + }; + + assert!( + binds_on_disk(&receipt), + "verified compact receipt is accepted" + ); + receipt.form = "future-container".to_owned(); + assert!(!binds_on_disk(&receipt), "future form must never bind"); + receipt.form = SEALED_STORE_FORM_LAYERED.to_owned(); + assert!( + !binds_on_disk(&receipt), + "layered form without base must refuse" + ); + + receipt.base = Some(SealedBaseReceiptV1 { + generation: "generation:base".to_owned(), + source_generation: "source:base".to_owned(), + watermark: "watermark:base".to_owned(), + physical_namespace: "generation:base".to_owned(), + recovered_digest: "base-digest".to_owned(), + entities: 1, + relations: 0, + row_sum: "row-sum".to_owned(), + hidden_row_sum: "hidden-sum".to_owned(), + }); + receipt.form = SEALED_STORE_FORM_COMPACT.to_owned(); + assert!( + !binds_on_disk(&receipt), + "compact form with base must refuse" + ); + receipt.form = SEALED_STORE_FORM_LAYERED.to_owned(); + assert!( + binds_on_disk(&receipt), + "recognized layered form remains eligible" + ); + } +} + /// Digest and row counts that authorize a staging-row release without opening /// the sealed engine. pub(crate) struct SealedReleaseEvidence { @@ -529,135 +629,476 @@ pub struct SealedStoreCensusV1 { pub unrecognized_count: u64, } -/// Measure the sealed root beside `database_path` against the generation ids -/// of the store's current verified heads (as journaled: `:`). -/// -/// Metadata only: every entry is one receipt read plus a directory size walk, -/// never a container open. An unreadable receipt counts as unrecognized so the -/// census stays truthful when a receipt is mid-write or corrupt. +/// Keep census enumeration, receipt identity and byte measurement beneath one +/// already-open, no-follow sealed root. A path rename or symlink replacement +/// cannot substitute a foreign generation after the census begins. pub fn census_sealed_store( database_path: &Path, - head_generations: &std::collections::BTreeSet, + head_generations: &std::collections::BTreeSet<(String, String, String)>, ) -> std::io::Result { - let root = sealed_store_root(database_path); - let entries = match std::fs::read_dir(&root) { - Ok(entries) => entries, - Err(error) if error.kind() == std::io::ErrorKind::NotFound => { - return Ok(SealedStoreCensusV1::default()); - } - Err(error) => return Err(error), - }; - let mut census = SealedStoreCensusV1::default(); - let mut counted = std::collections::HashSet::new(); - for entry in entries { - let entry = entry?; - if !entry.file_type()?.is_dir() { - census.unrecognized_count += 1; - continue; - } - let path = entry.path(); - let bytes = directory_bytes(&path, &mut counted)?; - let name = entry.file_name(); - if name - .to_str() - .is_some_and(|name| name.starts_with(".staging-")) - { - census.abandoned_staging_count += 1; - census.abandoned_staging_bytes += bytes; - continue; - } - let receipt = std::fs::read(path.join(SEALED_STORE_RECEIPT_FILE)) - .ok() - .and_then(|bytes| serde_json::from_slice::(&bytes).ok()); - let Some(receipt) = receipt else { - census.unrecognized_count += 1; - continue; - }; - if head_generations.contains(&receipt.generation) { - census.head_count += 1; - census.head_bytes += bytes; - } else { - census.superseded_count += 1; - census.superseded_bytes += bytes; - } - } - Ok(census) + census_sealed_store_root(&sealed_store_root(database_path), head_generations) } -/// Bytes under `directory` not already counted in `counted`: a layered -/// generation's base container is a hard link to bytes another directory -/// already holds, so each file is charged once per census. -fn directory_bytes( - directory: &Path, - counted: &mut std::collections::HashSet<(u64, u64)>, -) -> std::io::Result { - let mut total = 0u64; - let mut pending = vec![directory.to_path_buf()]; - while let Some(directory) = pending.pop() { - for entry in std::fs::read_dir(&directory)? { - let entry = entry?; - let metadata = entry.metadata()?; - if metadata.is_dir() { - pending.push(entry.path()); - continue; +fn census_sealed_store_root( + root: &Path, + head_generations: &std::collections::BTreeSet<(String, String, String)>, +) -> std::io::Result { + const HEAD: usize = 0; + const SUPERSEDED: usize = 1; + const ABANDONED: usize = 2; + + let Some(measured) = census_child_directories_nofollow::<3>( + root, + Some(( + SEALED_STORE_RECEIPT_FILE, + MAX_SEALED_STORE_RECEIPT_BYTES as usize, + )), + |name, receipt_bytes| { + if name.to_str().is_some_and(sealed_staging_name) { + return Some(ABANDONED); } - #[cfg(unix)] + let name = name.to_str()?; + let receipt = receipt_bytes + .and_then(|bytes| serde_json::from_slice::(bytes).ok())?; + if name.len() != 64 + || !name.bytes().all(|byte| byte.is_ascii_hexdigit()) + || receipt.version != SEALED_STORE_RECEIPT_VERSION + || receipt.physical_namespace != format!("generation:{name}") + || !recognized_sealed_form(&receipt) { - use std::os::unix::fs::MetadataExt; - if metadata.nlink() > 1 && !counted.insert((metadata.dev(), metadata.ino())) { - continue; - } + return None; } - #[cfg(not(unix))] - let _ = &counted; - total = total.saturating_add(metadata.len()); + if head_generations.contains(&( + receipt.namespace, + receipt.projection, + receipt.generation, + )) { + Some(HEAD) + } else { + Some(SUPERSEDED) + } + }, + )? + else { + return Ok(SealedStoreCensusV1::default()); + }; + + Ok(SealedStoreCensusV1 { + head_count: measured.counts[HEAD], + head_bytes: measured.bytes[HEAD], + superseded_count: measured.counts[SUPERSEDED], + superseded_bytes: measured.bytes[SUPERSEDED], + abandoned_staging_count: measured.counts[ABANDONED], + abandoned_staging_bytes: measured.bytes[ABANDONED], + unrecognized_count: measured.unrecognized, + }) +} + +#[cfg(all(test, unix))] +mod sealed_census_tests { + use std::collections::BTreeSet; + use std::fs; + + use super::{ + SEALED_STORE_RECEIPT_FILE, SEALED_STORE_RECEIPT_VERSION, census_sealed_store_root, + }; + + #[test] + fn shared_sealed_bytes_belong_to_live_head_regardless_of_directory_order() { + let temp = tempfile::tempdir().unwrap(); + let root = temp.path(); + let old = "a".repeat(64); + let head = "b".repeat(64); + let wrong_namespace = "c".repeat(64); + let future_version = "d".repeat(64); + let foreign_projection = "e".repeat(64); + let unknown_form = "0".repeat(64); + let missing_layer_base = "1".repeat(64); + let future_graph_format = "2".repeat(64); + let zero_graph_format = "3".repeat(64); + for name in [ + "unrecognized", + ".staging-pending", + old.as_str(), + head.as_str(), + wrong_namespace.as_str(), + future_version.as_str(), + foreign_projection.as_str(), + unknown_form.as_str(), + missing_layer_base.as_str(), + future_graph_format.as_str(), + zero_graph_format.as_str(), + ] { + fs::create_dir(root.join(name)).unwrap(); } + let receipt = |generation: &str, namespace: &str, version: u32| { + serde_json::to_vec(&serde_json::json!({ + "version": version, + "form": "compact", + "namespace": "code", + "projection": "index", + "generation": generation, + "physical_namespace": namespace, + "recovered_digest": "unused", + "entities": 0, + "relations": 0, + "graph_format": 1, + "row_sum": null, + "base": null, + "source_generation": null, + "watermark": null + })) + .unwrap() + }; + let head_receipt = receipt( + "live", + &format!("generation:{head}"), + SEALED_STORE_RECEIPT_VERSION, + ); + let old_receipt = receipt( + "retired", + &format!("generation:{old}"), + SEALED_STORE_RECEIPT_VERSION, + ); + fs::write( + root.join(&head).join(SEALED_STORE_RECEIPT_FILE), + &head_receipt, + ) + .unwrap(); + fs::write( + root.join(&old).join(SEALED_STORE_RECEIPT_FILE), + &old_receipt, + ) + .unwrap(); + // A parseable receipt cannot claim live bytes without matching the + // generation directory, physical namespace and receipt version. + for (directory, namespace, version) in [ + ( + "unrecognized", + "generation:unrecognized".to_owned(), + SEALED_STORE_RECEIPT_VERSION, + ), + ( + wrong_namespace.as_str(), + "generation:wrong".to_owned(), + SEALED_STORE_RECEIPT_VERSION, + ), + ( + future_version.as_str(), + format!("generation:{future_version}"), + SEALED_STORE_RECEIPT_VERSION + 1, + ), + ] { + fs::write( + root.join(directory).join(SEALED_STORE_RECEIPT_FILE), + receipt("live", &namespace, version), + ) + .unwrap(); + } + // A different projection may legitimately reuse the same generation + // ID; its sealed artifact must not borrow the verified head's status. + let mut foreign_receipt: serde_json::Value = serde_json::from_slice(&receipt( + "live", + &format!("generation:{foreign_projection}"), + SEALED_STORE_RECEIPT_VERSION, + )) + .unwrap(); + foreign_receipt["projection"] = serde_json::json!("other-projection"); + let foreign_receipt = serde_json::to_vec(&foreign_receipt).unwrap(); + fs::write( + root.join(&foreign_projection) + .join(SEALED_STORE_RECEIPT_FILE), + &foreign_receipt, + ) + .unwrap(); + // Parseable, identity-bound receipts with unknown or inconsistent forms + // must not borrow a verified head's count or claim its physical bytes. + for (directory, form) in [ + (unknown_form.as_str(), "future-container"), + (missing_layer_base.as_str(), "layered"), + ] { + let mut value: serde_json::Value = serde_json::from_slice(&receipt( + "live", + &format!("generation:{directory}"), + SEALED_STORE_RECEIPT_VERSION, + )) + .unwrap(); + value["form"] = serde_json::json!(form); + fs::write( + root.join(directory).join(SEALED_STORE_RECEIPT_FILE), + serde_json::to_vec(&value).unwrap(), + ) + .unwrap(); + } + // A current receipt version with an unsupported graph format is not + // proof of a readable store, even when its generation identity matches. + for (directory, graph_format) in [ + ( + future_graph_format.as_str(), + super::GraphFormatVersion::current().get() + 1, + ), + (zero_graph_format.as_str(), 0), + ] { + let mut value: serde_json::Value = serde_json::from_slice(&receipt( + "live", + &format!("generation:{directory}"), + SEALED_STORE_RECEIPT_VERSION, + )) + .unwrap(); + value["graph_format"] = serde_json::json!(graph_format); + fs::write( + root.join(directory).join(SEALED_STORE_RECEIPT_FILE), + serde_json::to_vec(&value).unwrap(), + ) + .unwrap(); + } + fs::write(root.join(&unknown_form).join("foreign"), vec![0_u8; 4096]).unwrap(); + let shared = root.join(&head).join("shared"); + fs::write(&shared, vec![0_u8; 8192]).unwrap(); + for directory in [ + old.as_str(), + ".staging-pending", + "unrecognized", + unknown_form.as_str(), + missing_layer_base.as_str(), + future_graph_format.as_str(), + zero_graph_format.as_str(), + ] { + fs::hard_link(&shared, root.join(directory).join("shared")).unwrap(); + } + + // Head-first class charging must be independent of readdir ordering. + let census = census_sealed_store_root( + root, + &BTreeSet::from([("code".to_owned(), "index".to_owned(), "live".to_owned())]), + ) + .unwrap(); + assert_eq!((census.head_count, census.superseded_count), (1, 2)); + assert_eq!( + (census.abandoned_staging_count, census.unrecognized_count), + (0, 8) + ); + assert_eq!(census.head_bytes, 8192 + head_receipt.len() as u64); + assert_eq!( + census.superseded_bytes, + (old_receipt.len() + foreign_receipt.len()) as u64 + ); + assert_eq!(census.abandoned_staging_bytes, 0); + + let no_head = census_sealed_store_root(root, &BTreeSet::new()).unwrap(); + assert_eq!(no_head.head_bytes, 0); + assert_eq!( + no_head.superseded_bytes, + 8192 + head_receipt.len() as u64 + + old_receipt.len() as u64 + + foreign_receipt.len() as u64 + ); } - Ok(total) } -/// Removes every sealed generation beside `database_path` whose receipt does -/// not name the current graph format: after a superseded-format rebuild those -/// containers can never serve, while a generation this build already sealed -/// keeps serving. In-flight `.staging-*` seals are left to their builder. +/// Removes sealed generations with a proven older graph format. A receipt +/// written by a newer build, an unrecognized artifact, or a non-generation +/// entry is not safe to delete after a downgrade. In-flight `.staging-*` +/// directories remain owned by their builder. pub(crate) fn discard_superseded_sealed_generations( database_path: &Path, ) -> Result<(), GraphDbError> { let root = sealed_store_root(database_path); - let entries = match std::fs::read_dir(&root) { - Ok(entries) => entries, - Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(()), - Err(error) => return Err(sealed_store_io_failure("sealed root read failed", error)), - }; - for entry in entries { - let entry = - entry.map_err(|error| sealed_store_io_failure("sealed root read failed", error))?; - let path = entry.path(); - if entry - .file_name() - .to_str() - .is_some_and(|name| name.starts_with(".staging-")) - { - continue; - } - let current = load_sealed_store_receipt(&path) - .ok() - .flatten() - .is_some_and(|receipt| receipt.graph_format == GraphFormatVersion::current().get()); - if !current { - match std::fs::remove_dir_all(&path) { - Ok(()) => {} - Err(error) if error.kind() == std::io::ErrorKind::NotFound => {} - Err(error) => { - return Err(sealed_store_io_failure( - "superseded sealed generation removal failed", - error, - )); - } + tracedecay_private_fs::capability_dir::remove_selected_child_directories_nofollow( + &root, + Some(( + SEALED_STORE_RECEIPT_FILE, + MAX_SEALED_STORE_RECEIPT_BYTES as usize, + )), + |name, receipt_bytes| { + let Some(name) = name.to_str() else { + return false; + }; + if name.len() != 64 || !name.bytes().all(|byte| byte.is_ascii_hexdigit()) { + return false; } + let Some(receipt) = receipt_bytes + .and_then(|bytes| serde_json::from_slice::(bytes).ok()) + else { + return false; + }; + receipt.version == SEALED_STORE_RECEIPT_VERSION + && receipt.physical_namespace == format!("generation:{name}") + && receipt.graph_format < GraphFormatVersion::current().get() + && recognized_sealed_form(&receipt) + }, + ) + .map_err(|error| { + sealed_store_io_failure("superseded sealed generation removal failed", error) + })?; + Ok(()) +} + +#[cfg(test)] +mod superseded_sealed_cleanup_tests { + use super::*; + + #[cfg(unix)] + #[test] + fn foreign_directory_behind_a_symlinked_sealed_root_is_never_cleaned() { + let temp = tempfile::tempdir().unwrap(); + let database = temp.path().join("graph.grafeo"); + let root = sealed_store_root(&database); + let foreign = temp.path().join("foreign"); + std::fs::create_dir(&foreign).unwrap(); + + let current = GraphFormatVersion::current().get(); + assert!(current > 0); + let obsolete = "a".repeat(64); + let obsolete_path = foreign.join(&obsolete); + std::fs::create_dir(&obsolete_path).unwrap(); + std::fs::write( + obsolete_path.join(SEALED_STORE_RECEIPT_FILE), + serde_json::to_vec(&serde_json::json!({ + "version": SEALED_STORE_RECEIPT_VERSION, + "form": "compact", + "namespace": "code", + "projection": "index", + "generation": "foreign", + "physical_namespace": format!("generation:{obsolete}"), + "recovered_digest": "digest", + "entities": 0, + "relations": 0, + "graph_format": current - 1, + "row_sum": null, + "base": null, + "source_generation": null, + "watermark": null + })) + .unwrap(), + ) + .unwrap(); + for name in [".staging-abandoned", ".rows-0-foreign"] { + std::fs::create_dir(foreign.join(name)).unwrap(); + } + std::os::unix::fs::symlink(&foreign, &root).unwrap(); + + assert!(census_sealed_store(&database, &Default::default()).is_err()); + assert!(discard_superseded_sealed_generations(&database).is_err()); + sweep_abandoned_sealed_staging(&database); + sweep_abandoned_row_spills(&database); + assert!(remove_sealed_directory_checked(&root.join(&obsolete)).is_err()); + + for name in [obsolete.as_str(), ".staging-abandoned", ".rows-0-foreign"] { + assert!( + foreign.join(name).is_dir(), + "foreign artifact {name} was removed" + ); + } + } + + #[test] + fn only_proven_older_generation_artifacts_can_be_removed() { + let temp = tempfile::tempdir().unwrap(); + let database = temp.path().join("graph.grafeo"); + let root = sealed_store_root(&database); + std::fs::create_dir(&root).unwrap(); + let current = GraphFormatVersion::current().get(); + assert!(current > 0); + let obsolete = "a".repeat(64); + let future = "b".repeat(64); + let wrong_identity = "c".repeat(64); + let unreadable = "d".repeat(64); + let ordinary_file = "e".repeat(64); + let current_name = "f".repeat(64); + let unknown_form = "0".repeat(64); + let inconsistent_form = "1".repeat(64); + let unrelated = "operator-data"; + + let receipt = |name: &str, format: u32, physical_namespace: String| { + let directory = root.join(name); + std::fs::create_dir(&directory).unwrap(); + std::fs::write( + directory.join(SEALED_STORE_RECEIPT_FILE), + serde_json::to_vec(&serde_json::json!({ + "version": SEALED_STORE_RECEIPT_VERSION, + "form": "compact", + "namespace": "code", + "projection": "index", + "generation": "g", + "physical_namespace": physical_namespace, + "recovered_digest": "digest", + "entities": 0, + "relations": 0, + "graph_format": format, + "row_sum": null, + "base": null, + "source_generation": null, + "watermark": null + })) + .unwrap(), + ) + .unwrap(); + }; + receipt(&obsolete, current - 1, format!("generation:{obsolete}")); + receipt(&future, current + 1, format!("generation:{future}")); + receipt( + &wrong_identity, + current - 1, + "generation:foreign".to_owned(), + ); + receipt(¤t_name, current, format!("generation:{current_name}")); + // A parseable receipt with an older graph-format number can belong to + // a newer sealed representation, or be inconsistent with its base. + // Neither is a proven artifact that this build is allowed to remove. + for (name, form) in [ + (&unknown_form, "future-container"), + (&inconsistent_form, "layered"), + ] { + receipt(name, current - 1, format!("generation:{name}")); + let receipt_path = root.join(name).join(SEALED_STORE_RECEIPT_FILE); + let mut value: serde_json::Value = + serde_json::from_slice(&std::fs::read(&receipt_path).unwrap()).unwrap(); + value["form"] = form.into(); + std::fs::write(&receipt_path, serde_json::to_vec(&value).unwrap()).unwrap(); + } + // Whitespace-padded JSON is still a parseable old-format receipt. + // It must not authorize deletion or a full allocation in the census. + let oversized = "4".repeat(64); + receipt(&oversized, current - 1, format!("generation:{oversized}")); + let oversized_directory = root.join(&oversized); + let oversized_receipt = oversized_directory.join(SEALED_STORE_RECEIPT_FILE); + let mut padded = std::fs::read(&oversized_receipt).unwrap(); + padded.resize(MAX_SEALED_STORE_RECEIPT_BYTES as usize + 1, b' '); + assert!(serde_json::from_slice::(&padded).is_ok()); + std::fs::write(&oversized_receipt, &padded).unwrap(); + assert!(load_sealed_store_receipt(&oversized_directory).is_err()); + receipt(unrelated, current - 1, format!("generation:{unrelated}")); + std::fs::create_dir(root.join(&unreadable)).unwrap(); + std::fs::write( + root.join(&unreadable).join(SEALED_STORE_RECEIPT_FILE), + b"not json", + ) + .unwrap(); + std::fs::write(root.join(&ordinary_file), b"operator file").unwrap(); + std::fs::create_dir(root.join(".staging-active")).unwrap(); + + discard_superseded_sealed_generations(&database).unwrap(); + + assert!( + !root.join(&obsolete).exists(), + "old verified format is retired" + ); + for name in [ + future.as_str(), + wrong_identity.as_str(), + unreadable.as_str(), + ordinary_file.as_str(), + current_name.as_str(), + unknown_form.as_str(), + inconsistent_form.as_str(), + oversized.as_str(), + unrelated, + ".staging-active", + ] { + assert!(root.join(name).exists(), "must preserve {name}"); } } - Ok(()) } /// Removes every `.staging-*` directory a seal left behind under the store's @@ -672,53 +1113,43 @@ pub(crate) fn discard_superseded_sealed_generations( /// this store is in flight. pub(crate) fn sweep_abandoned_sealed_staging(database_path: &Path) { let root = sealed_store_root(database_path); - let entries = match std::fs::read_dir(&root) { - Ok(entries) => entries, - Err(error) if error.kind() == std::io::ErrorKind::NotFound => return, - Err(error) => { - tracing::warn!( - event = "sealed_staging_sweep_unreadable", - root = %root.display(), - error = %error, - "sealed store root could not be enumerated; abandoned staging stays" - ); - return; - } - }; - let mut removed = 0usize; - for entry in entries.flatten() { - let name = entry.file_name(); - let Some(name) = name.to_str() else { - continue; - }; - if !name.starts_with(".staging-") { - continue; - } - let path = entry.path(); - match std::fs::remove_dir_all(&path) { - Ok(()) => removed += 1, - Err(error) if error.kind() == std::io::ErrorKind::NotFound => {} - Err(error) => tracing::warn!( - event = "sealed_staging_sweep_failed", - path = %path.display(), - error = %error, - "abandoned sealed staging directory could not be removed" - ), - } - } - if removed > 0 { - tracing::info!( + match tracedecay_private_fs::capability_dir::remove_selected_child_directories_nofollow( + &root, + None, + |name, _| name.to_str().is_some_and(sealed_staging_name), + ) { + Ok(Some(removed)) if removed > 0 => tracing::info!( event = "sealed_staging_swept", root = %root.display(), removed, "removed abandoned sealed staging directories left by interrupted seals" - ); + ), + Ok(_) => {} + Err(error) => tracing::warn!( + event = "sealed_staging_sweep_unreadable", + root = %root.display(), + %error, + "sealed store root could not be safely swept; remaining staging stays" + ), } } /// Prefix of a batch producer's row spill under the sealed root. const ROW_SPILL_PREFIX: &str = ".rows-"; +/// A row-spill owner writes exactly `.rows--`. Unknown entries +/// sharing this prefix do not belong to the spill collector. +fn row_spill_name(name: &str) -> bool { + name.strip_prefix(ROW_SPILL_PREFIX) + .and_then(|suffix| suffix.split_once('-')) + .is_some_and(|(pid, nonce)| { + !pid.is_empty() + && !nonce.is_empty() + && pid.bytes().all(|byte| byte.is_ascii_digit()) + && nonce.bytes().all(|byte| byte.is_ascii_digit()) + }) +} + /// Removes every row spill another process left under the store's sealed /// root. Run at the eager open, whose exclusive store lock means no other /// process is producing rows for this store. Spills named for this process @@ -726,40 +1157,110 @@ const ROW_SPILL_PREFIX: &str = ".rows-"; /// whose spill removes itself when it drops. pub(crate) fn sweep_abandoned_row_spills(database_path: &Path) { let root = sealed_store_root(database_path); - let Ok(entries) = std::fs::read_dir(&root) else { - return; - }; let own = format!("{ROW_SPILL_PREFIX}{}-", std::process::id()); - for entry in entries.flatten() { - if entry - .file_name() - .to_str() - .is_some_and(|name| name.starts_with(ROW_SPILL_PREFIX) && !name.starts_with(&own)) - { - let path = entry.path(); - if let Err(error) = std::fs::remove_dir_all(&path) - && error.kind() != std::io::ErrorKind::NotFound - { - tracing::warn!( - event = "graph_row_spill_sweep_failed", - path = %path.display(), - error = %error, - "abandoned graph row spill could not be removed" - ); - } + if let Err(error) = + tracedecay_private_fs::capability_dir::remove_selected_child_directories_nofollow( + &root, + None, + |name, _| { + name.to_str() + .is_some_and(|name| row_spill_name(name) && !name.starts_with(&own)) + }, + ) + { + tracing::warn!( + event = "graph_row_spill_sweep_failed", + root = %root.display(), + %error, + "sealed row spills could not be safely swept" + ); + } +} + +#[cfg(all(test, unix))] +mod abandoned_sealed_cleanup_tests { + use super::*; + + #[test] + fn recovery_collects_only_owned_generation_staging_and_numeric_row_spills() { + use std::os::unix::fs::symlink; + + let temp = tempfile::tempdir().unwrap(); + let database = temp.path().join("graph.grafeo"); + let root = sealed_store_root(&database); + std::fs::create_dir(&root).unwrap(); + + let owned_staging = format!("{SEALED_STAGING_PREFIX}{}", "a".repeat(64)); + let foreign_pid = u64::from(std::process::id()) + 1; + let owned_spill = format!("{ROW_SPILL_PREFIX}{foreign_pid}-0"); + let live_spill = format!("{ROW_SPILL_PREFIX}{}-0", std::process::id()); + for name in [&owned_staging, &owned_spill, &live_spill] { + std::fs::create_dir(root.join(name)).unwrap(); + std::fs::write(root.join(name).join("payload"), b"owned").unwrap(); + } + + let unknown = [ + ".staging-pending".to_owned(), + format!("{SEALED_STAGING_PREFIX}{}-extra", "b".repeat(64)), + ".rows-foreign".to_owned(), + ".rows-123-extra".to_owned(), + ".rows-123-0-extra".to_owned(), + ]; + for name in &unknown { + std::fs::create_dir(root.join(name)).unwrap(); + std::fs::write(root.join(name).join("foreign"), b"keep").unwrap(); } + let outside = temp.path().join("outside"); + std::fs::create_dir(&outside).unwrap(); + std::fs::write(outside.join("foreign"), b"external").unwrap(); + let linked_staging = format!("{SEALED_STAGING_PREFIX}{}", "c".repeat(64)); + symlink(&outside, root.join(&linked_staging)).unwrap(); + let linked_spill = format!("{ROW_SPILL_PREFIX}{foreign_pid}-1"); + symlink(&outside, root.join(&linked_spill)).unwrap(); + + let before = census_sealed_store(&database, &Default::default()).unwrap(); + assert_eq!(before.abandoned_staging_count, 1); + assert!(before.unrecognized_count >= unknown.len() as u64 + 3); + + sweep_abandoned_sealed_staging(&database); + sweep_abandoned_row_spills(&database); + + assert!(!root.join(&owned_staging).exists()); + assert!(!root.join(&owned_spill).exists()); + assert!(root.join(&live_spill).is_dir()); + for name in &unknown { + assert_eq!( + std::fs::read(root.join(name).join("foreign")).unwrap(), + b"keep" + ); + } + assert!( + std::fs::symlink_metadata(root.join(&linked_staging)) + .unwrap() + .file_type() + .is_symlink() + ); + assert!( + std::fs::symlink_metadata(root.join(&linked_spill)) + .unwrap() + .file_type() + .is_symlink() + ); + assert_eq!(std::fs::read(outside.join("foreign")).unwrap(), b"external"); + let after = census_sealed_store(&database, &Default::default()).unwrap(); + assert_eq!(after.abandoned_staging_count, 0); } } fn remove_sealed_directory_checked(directory: &Path) -> Result<(), GraphDbError> { - match std::fs::remove_dir_all(directory) { - Ok(()) => Ok(()), - Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(()), - Err(error) => Err(sealed_store_io_failure( - "sealed directory removal failed", - error, - )), - } + let root = directory + .parent() + .ok_or_else(|| GraphDbError::unavailable("sealed artifact has no parent directory"))?; + let name = directory + .file_name() + .ok_or_else(|| GraphDbError::unavailable("sealed artifact has no directory name"))?; + tracedecay_private_fs::capability_dir::remove_named_child_directory_nofollow(root, name) + .map_err(|error| sealed_store_io_failure("sealed directory removal failed", error)) } /// Staging and invalid-artifact cleanup are best effort. Retiring an accepted @@ -2422,28 +2923,52 @@ fn open_sealed_store( open_sealed_store_checked(directory, identity, expected, &|| Ok(()), None) } +/// Use the shared no-follow, handle-validated bounded file reader. A pathname +/// can be swapped after a metadata check, so opening a symlink must fail at +/// the actual open operation, not only at an earlier path inspection. +fn read_sealed_receipt_bytes(path: &Path) -> std::io::Result>> { + tracedecay_private_fs::framed_log::read_bounded(path, MAX_SEALED_STORE_RECEIPT_BYTES as usize) +} + fn load_sealed_store_receipt( directory: &Path, ) -> Result, GraphDbError> { let receipt_path = directory.join(SEALED_STORE_RECEIPT_FILE); - let receipt_bytes = match std::fs::read(&receipt_path) { - Ok(bytes) => bytes, - Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(None), - Err(error) => return Err(sealed_store_io_failure("receipt read failed", error)), + let Some(receipt_bytes) = read_sealed_receipt_bytes(&receipt_path) + .map_err(|error| sealed_store_io_failure("receipt read failed", error))? + else { + return Ok(None); }; serde_json::from_slice(&receipt_bytes) .map(Some) .map_err(|error| GraphDbError::unavailable(format!("sealed receipt decode: {error}"))) } +/// A proof marker is only a digest, not a trusted file. Reuse the same +/// no-follow reader as receipts; a symlink swapped in during open must never +/// supply proof bytes, and oversized or nonregular markers do not match. fn sealed_store_check_matches( directory: &Path, expected_digest: &str, ) -> Result { - let path = directory.join(SEALED_STORE_CHECKED_FILE); - match std::fs::read_to_string(&path) { - Ok(digest) => Ok(digest == expected_digest), - Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(false), + let limit = expected_digest + .len() + .min(MAX_SEALED_STORE_RECEIPT_BYTES as usize); + match tracedecay_private_fs::capability_dir::read_nested_bounded_file_nofollow( + directory, + SEALED_STORE_CHECKED_FILE, + limit, + ) { + Ok(Some(bytes)) => Ok(bytes == expected_digest.as_bytes()), + Ok(None) => Ok(false), + Err(error) + if matches!( + error.kind(), + std::io::ErrorKind::InvalidInput | std::io::ErrorKind::InvalidData + ) => + { + Ok(false) + } Err(error) => Err(sealed_store_io_failure("sealed check read failed", error)), } } @@ -2452,8 +2977,65 @@ fn persist_sealed_store_check(directory: &Path, expected_digest: &str) -> Result if sealed_store_check_matches(directory, expected_digest)? { return Ok(()); } - std::fs::write(directory.join(SEALED_STORE_CHECKED_FILE), expected_digest) - .map_err(|error| sealed_store_io_failure("sealed check persist failed", error)) + tracedecay_private_fs::capability_dir::replace_nested_file_nofollow( + directory, + SEALED_STORE_CHECKED_FILE, + expected_digest.as_bytes(), + ) + .map_err(|error| sealed_store_io_failure("sealed check persist failed", error)) +} + +#[cfg(test)] +mod sealed_marker_tests { + use super::{ + SEALED_STORE_CHECKED_FILE, persist_sealed_store_check, sealed_store_check_matches, + }; + + #[test] + fn proof_marker_requires_exact_bounded_digest_and_replaces_stale_bytes() { + let root = tempfile::tempdir().unwrap(); + let directory = root.path().join("generation"); + std::fs::create_dir(&directory).unwrap(); + let marker = directory.join(SEALED_STORE_CHECKED_FILE); + let digest = "a".repeat(64); + + assert!(!sealed_store_check_matches(&directory, &digest).unwrap()); + std::fs::write(&marker, vec![b'a'; 1024 * 1024]).unwrap(); + assert!(!sealed_store_check_matches(&directory, &digest).unwrap()); + persist_sealed_store_check(&directory, &digest).unwrap(); + assert!(sealed_store_check_matches(&directory, &digest).unwrap()); + assert_eq!(std::fs::read(&marker).unwrap(), digest.as_bytes()); + assert_eq!(std::fs::read_dir(&directory).unwrap().count(), 1); + + std::fs::write(&marker, b"stale").unwrap(); + assert!(!sealed_store_check_matches(&directory, &digest).unwrap()); + persist_sealed_store_check(&directory, &digest).unwrap(); + assert!(sealed_store_check_matches(&directory, &digest).unwrap()); + } + + #[cfg(unix)] + #[test] + fn marker_reproof_does_not_clobber_a_foreign_symlink_target() { + let root = tempfile::tempdir().unwrap(); + let directory = root.path().join("generation"); + std::fs::create_dir(&directory).unwrap(); + let foreign = root.path().join("foreign-durable-data"); + std::fs::write(&foreign, b"must survive intact").unwrap(); + let marker = directory.join(SEALED_STORE_CHECKED_FILE); + std::os::unix::fs::symlink(&foreign, &marker).unwrap(); + let digest = "b".repeat(64); + + assert!(!sealed_store_check_matches(&directory, &digest).unwrap()); + persist_sealed_store_check(&directory, &digest).unwrap(); + assert_eq!(std::fs::read(&foreign).unwrap(), b"must survive intact"); + assert!(std::fs::symlink_metadata(&marker).unwrap().is_file()); + assert!(sealed_store_check_matches(&directory, &digest).unwrap()); + + let linked_directory = root.path().join("linked-generation"); + std::os::unix::fs::symlink(&directory, &linked_directory).unwrap(); + assert!(persist_sealed_store_check(&linked_directory, &"c".repeat(64)).is_err()); + assert_eq!(std::fs::read(&foreign).unwrap(), b"must survive intact"); + } } fn open_sealed_store_checked( diff --git a/crates/tracedecay-graph-db/tests/graph_db_suite/runtime_contract.rs b/crates/tracedecay-graph-db/tests/graph_db_suite/runtime_contract.rs index 123b4e9f3f7..c69dbf3222a 100644 --- a/crates/tracedecay-graph-db/tests/graph_db_suite/runtime_contract.rs +++ b/crates/tracedecay-graph-db/tests/graph_db_suite/runtime_contract.rs @@ -1026,7 +1026,7 @@ fn wrong_tracedecay_format_requires_reset() { } #[test] -fn superseded_format_store_is_rebuilt_fresh_keeping_only_current_sealed_generations() { +fn superseded_format_store_rebuild_removes_proven_old_seals_but_keeps_unproven() { let temp = TempDir::new().unwrap(); let path = graph_path(temp.path()); // Format 4 kept a relation's payload properties on its locator as well as @@ -1063,13 +1063,46 @@ fn superseded_format_store_is_rebuilt_fresh_keeping_only_current_sealed_generati ) .unwrap(); raw.close().unwrap(); - let sealed_generation = temp.path().join("graph.sealed").join("0".repeat(64)); + let sealed_generation_name = "0".repeat(64); + let sealed_generation = temp + .path() + .join("graph.sealed") + .join(&sealed_generation_name); std::fs::create_dir_all(&sealed_generation).unwrap(); std::fs::write( sealed_generation.join("generation.grafeo"), b"format 4 bytes", ) .unwrap(); + // A prior-format sealed directory is deletable only when its receipt + // proves both the old format and the matching physical namespace. + std::fs::write( + sealed_generation.join("sealed.json"), + serde_json::json!({ + "version": 1, + "form": "compact", + "namespace": "project", + "projection": "code", + "generation": "g0", + "physical_namespace": format!("generation:{sealed_generation_name}"), + "recovered_digest": "sha256:old", + "entities": 1, + "relations": 0, + "graph_format": previous_format as u32, + }) + .to_string(), + ) + .unwrap(); + + // An unverified artifact may belong to a future build. A format reset + // cannot infer its age or ownership from a directory name alone. + let unproven_generation = temp.path().join("graph.sealed").join("2".repeat(64)); + std::fs::create_dir_all(&unproven_generation).unwrap(); + std::fs::write( + unproven_generation.join("generation.grafeo"), + b"unknown sealed bytes", + ) + .unwrap(); // A generation this build sealed before the staging container was // replaced, as a publication racing the rebuild leaves it. @@ -1096,6 +1129,10 @@ fn superseded_format_store_is_rebuilt_fresh_keeping_only_current_sealed_generati let (_registered, db) = RegisteredGraph::open_lease(temp.path()).unwrap(); assert!(!sealed_generation.exists()); + assert!( + unproven_generation.join("generation.grafeo").exists(), + "an unverified generation must survive a format reset" + ); assert!(current_generation.join("sealed.json").exists()); assert_eq!( db.entity(&namespace(), &entity_id("stale"), live()) diff --git a/crates/tracedecay-graph-db/tests/graph_db_suite/verified_generation_contract/sealed_store.rs b/crates/tracedecay-graph-db/tests/graph_db_suite/verified_generation_contract/sealed_store.rs index 3bb165ee0f6..2491f99b9b0 100644 --- a/crates/tracedecay-graph-db/tests/graph_db_suite/verified_generation_contract/sealed_store.rs +++ b/crates/tracedecay-graph-db/tests/graph_db_suite/verified_generation_contract/sealed_store.rs @@ -1354,7 +1354,11 @@ fn sealed_census_separates_heads_from_superseded_and_abandoned_bytes() { std::fs::create_dir_all(&unreadable).unwrap(); std::fs::write(unreadable.join("generation.grafeo"), b"no receipt").unwrap(); - let heads = std::collections::BTreeSet::from(["census-g2".to_owned()]); + let heads = std::collections::BTreeSet::from([( + identity.namespace.as_str().to_owned(), + identity.projection.as_str().to_owned(), + "census-g2".to_owned(), + )]); let census = tracedecay_graph_db::census_sealed_store(&support::graph_path(temp.path()), &heads) .unwrap(); @@ -1368,6 +1372,38 @@ fn sealed_census_separates_heads_from_superseded_and_abandoned_bytes() { assert_eq!(census.unrecognized_count, 1); assert!(census.head_bytes > 0 && census.superseded_bytes > 0); + #[cfg(unix)] + { + use std::os::unix::fs::symlink; + + // A foreign directory link inside a seal must not charge its target + // to the live generation (or recursively escape the census root). + let head = std::fs::read_dir(&root) + .unwrap() + .map(Result::unwrap) + .find(|entry| { + std::fs::read_to_string(entry.path().join("sealed.json")) + .is_ok_and(|receipt| receipt.contains("\"generation\": \"census-g2\"")) + }) + .unwrap() + .path(); + let outside = temp.path().join("unrelated"); + std::fs::create_dir_all(&outside).unwrap(); + std::fs::write(outside.join("foreign.data"), vec![0_u8; 1024 * 1024]).unwrap(); + let alias = head.join("foreign-directory-link"); + symlink(&outside, &alias).unwrap(); + + let after = + tracedecay_graph_db::census_sealed_store(&support::graph_path(temp.path()), &heads) + .unwrap(); + assert_eq!( + after.head_bytes, + census.head_bytes + std::fs::symlink_metadata(&alias).unwrap().len(), + "sealed census must charge only the link, not external target bytes" + ); + assert_eq!(after.superseded_bytes, census.superseded_bytes); + } + // With no heads journaled every sealed artifact reads as superseded, and // a store with no sealed root at all is an empty census, not an error. let none = tracedecay_graph_db::census_sealed_store( @@ -1517,6 +1553,76 @@ fn retirement_deletes_the_superseded_sealed_artifact() { ); } +/// Repeated retention must converge after more than three sealed publications. +/// Older code generations become collectable without disturbing the latest +/// verified head or its still-live serving snapshot. +#[test] +fn replay_retirement_reclaims_four_sealed_generations_and_preserves_live_head() { + let temp = TempDir::new().unwrap(); + let registered = RegisteredGraph::new_mounted(temp.path()).unwrap(); + let mut authority = RelationalAuthority::default(); + let identity = projection("sealed-store:retire-many", "code"); + let mut previous_head = None; + let mut latest = None; + + for sequence in 0..5_u8 { + let generation = format!("retire-many-g{sequence}"); + let marker = format!("marker-{sequence}"); + let manifest = rich_manifest(identity.clone(), &generation, &marker); + let input = char::from(b'1' + sequence); + let record = stage_sealed_manifest( + &mut authority, + ®istered.binding, + &manifest, + &format!("publish:retire-many-{sequence}"), + previous_head, + input, + ); + let committed = + publish_sealed(®istered, temp.path(), &mut authority, &record, &manifest); + previous_head = Some(committed.head.clone()); + if sequence == 4 { + assert_snapshot_reads(&committed.snapshot, &identity, &marker); + latest = Some(committed); + } + } + + assert_eq!(sealed_receipts(temp.path()).len(), 5); + let (control, probe) = control_and_probe(); + let context = GraphPublicationOperationContextV1::new(&control, &probe).unwrap(); + for sequence in 0..4_u8 { + let generation = format!("retire-many-g{sequence}"); + let input = char::from(b'1' + sequence); + let source_generation = + CodeGenerationId::new(format!("code-generation.{generation}")).unwrap(); + let source_digest = + SealedGraphStateDigest::try_from(format!("sha256:{}", input.to_string().repeat(64))) + .unwrap(); + assert!(matches!( + registered + .registry + .retire_one_code_generation_replay( + registration(registered.binding.clone(), temp.path()), + &mut authority, + &context, + &source_generation, + &source_digest, + ) + .unwrap(), + GraphReplayCollectionOutcome::Retired(_) + )); + assert!( + receipt_for_generation(temp.path(), &generation).is_none(), + "retired sealed artifact survived: {generation}" + ); + } + + let live = latest.expect("the last publication retains a live snapshot"); + assert_snapshot_reads(&live.snapshot, &identity, "marker-4"); + assert_eq!(sealed_receipts(temp.path()).len(), 1); + assert!(receipt_for_generation(temp.path(), "retire-many-g4").is_some()); +} + /// A live direct-sealed reader (recovered through /// `recover_verified_sealed_snapshot`, which bypasses the staging database's /// verified-generation state) must gate retirement of its generation exactly diff --git a/crates/tracedecay-lcm/src/retention/replay_metadata.rs b/crates/tracedecay-lcm/src/retention/replay_metadata.rs index b7be38726d9..72910e4798a 100644 --- a/crates/tracedecay-lcm/src/retention/replay_metadata.rs +++ b/crates/tracedecay-lcm/src/retention/replay_metadata.rs @@ -25,6 +25,9 @@ const MAX_BATCH_BYTES: u64 = 4 * 1024 * 1024; #[derive(Clone, Debug, Default, PartialEq, Eq, Serialize, Deserialize)] pub struct ReplayMetadataCompactionReport { pub rows_scanned: u64, + /// Transactional store-id frontier consumed by this page, absent on idle reads. + pub frontier_before: Option, + pub frontier_after: Option, pub rows_compacted: u64, /// Eligible rows retained because their representation, sanitizer result, /// or byte bound could not prove this narrow lossless transformation. @@ -150,6 +153,7 @@ async fn step( max_rows: usize, ) -> Result { let mut after = frontier(conn).await?; + let before = after; let candidates = candidates(conn, after, max_rows).await?; let mut report = ReplayMetadataCompactionReport::default(); let mut remaining = MAX_BATCH_BYTES; @@ -199,6 +203,15 @@ async fn step( if guard_suspended { summary_convergence::restore_raw_update_guard(conn).await?; } + if report.rows_scanned > 0 { + if after <= before { + return Err(db_error( + "replay metadata page did not advance its frontier", + )); + } + report.frontier_before = Some(before); + report.frontier_after = Some(after); + } schema::set_gc_meta(conn, FRONTIER_KEY, &after.to_string()).await?; report.has_more = has_work(conn, after).await?; Ok(report) diff --git a/crates/tracedecay-lcm/src/retention/replay_metadata/tests.rs b/crates/tracedecay-lcm/src/retention/replay_metadata/tests.rs index 1bc50de1ce3..38a405796cb 100644 --- a/crates/tracedecay-lcm/src/retention/replay_metadata/tests.rs +++ b/crates/tracedecay-lcm/src/retention/replay_metadata/tests.rs @@ -369,3 +369,37 @@ async fn row_and_byte_budgets_advance_a_durable_frontier_and_finished_pages_are_ let idle = store.tick(48).await; assert_eq!(idle, ReplayMetadataCompactionReport::default()); } + +#[asupersync::test] +async fn zero_compaction_pages_advance_distinct_durable_frontiers() { + let store = Store::open().await; + for ordinal in 1..=52 { + store + .insert( + ordinal, + "canonical source", + Value::String("different replay".to_owned()), + false, + ) + .await; + } + let first = store.tick(48).await; + assert_eq!(first.rows_scanned, 48); + assert_eq!(first.rows_compacted, 0); + assert_eq!(first.frontier_before, Some(0)); + assert_eq!(first.frontier_after, Some(48)); + assert!(first.has_more); + + let second = store.tick(48).await; + assert_eq!(second.rows_scanned, 4); + assert_eq!(second.rows_compacted, 0); + assert_eq!(second.frontier_before, Some(48)); + assert_eq!(second.frontier_after, Some(52)); + assert!(!second.has_more); + assert_eq!(frontier(&store.conn).await.unwrap(), 52); + + assert_eq!( + store.tick(48).await, + ReplayMetadataCompactionReport::default() + ); +} diff --git a/crates/tracedecay-maintenance/src/retention/cold_store.rs b/crates/tracedecay-maintenance/src/retention/cold_store.rs index 4ccaa70331d..e8bd0ccd19c 100644 --- a/crates/tracedecay-maintenance/src/retention/cold_store.rs +++ b/crates/tracedecay-maintenance/src/retention/cold_store.rs @@ -4,11 +4,15 @@ //! (`store_maintenance::run_registered_code_generation_retention`). use std::collections::BTreeSet; +use std::io::{self, Read}; use std::path::{Path, PathBuf}; use std::time::Instant; +use cap_fs_ext::{DirExt, FollowSymlinks, MetadataExt, OpenOptionsFollowExt, OpenOptionsSyncExt}; +use cap_std::fs::{Dir, OpenOptions}; use serde::{Deserialize, Serialize}; use tracedecay_global_db::RegisteredGlobalDb; +use tracedecay_private_fs::capability_dir::replace_file_in_open_directory; use tracedecay_runtime_core::cancellation::{CancellationToken, MonotonicDeadline}; use super::{incident_debris, orphan_stores}; @@ -17,6 +21,7 @@ use crate::clock::now_secs_i64; const COLD_STORE_PAGE_LIMIT: usize = 8; const CHECKPOINT_DIRECTORY: &str = "maintenance"; const CHECKPOINT_FILE: &str = "retention-cold-store-cursor-v1.json"; +const MAX_CHECKPOINT_BYTES: u64 = 64 * 1024; #[derive(Debug, Clone, PartialEq, Eq, Deserialize, Serialize)] struct ColdStoreCursorV1 { @@ -75,8 +80,13 @@ pub async fn run_cold_store_page( owner_roots: &BTreeSet, cancellation: &CancellationToken, ) -> tracedecay_domain::errors::Result { - let checkpoint_path = checkpoint_path(profile_root); - let cursor = load_cursor(&checkpoint_path).unwrap_or(ColdStoreCursorV1 { + let admitted = Dir::from_std_file(profile_database.profile_sessions_directory()?); + let directory = admitted_checkpoint_directory(&admitted, profile_root).map_err(|error| { + tracedecay_domain::errors::TraceDecayError::Config { + message: format!("open maintenance cold-store cursor directory: {error}"), + } + })?; + let cursor = load_cursor_in_dir(&directory).unwrap_or(ColdStoreCursorV1 { after_project_id: None, after_unregistered_directory: None, }); @@ -188,20 +198,14 @@ pub async fn run_cold_store_page( orphan_stores::UnregisteredSweepCompletionV1::DeadlineExceeded => {} } } - let project_ids = page - .entries - .iter() - .map(|entry| entry.project_id.clone()) - .collect::>(); + // The page boundary belongs to the project registry, not the filtered + // profile-sharded store census. Projects with no matching stores still + // consume a registry page and must advance its durable resume key. let next_cursor = ColdStoreCursorV1 { - after_project_id: next_cold_store_cursor( - cursor.after_project_id.as_deref(), - &project_ids, - page.next_cursor.is_some(), - ), + after_project_id: page.next_cursor, after_unregistered_directory, }; - persist_cursor(&checkpoint_path, &next_cursor).map_err(|error| { + persist_cursor_in_dir(&directory, &next_cursor).map_err(|error| { tracedecay_domain::errors::TraceDecayError::Config { message: format!("persist maintenance cold-store cursor: {error}"), } @@ -209,20 +213,6 @@ pub async fn run_cold_store_page( Ok(report) } -fn next_cold_store_cursor( - previous: Option<&str>, - project_ids: &[String], - has_more: bool, -) -> Option { - if !has_more { - return None; - } - project_ids - .last() - .cloned() - .or_else(|| previous.map(str::to_owned)) -} - fn classify_cold_store_state( cancelled: bool, manifest_readable: bool, @@ -239,16 +229,84 @@ fn classify_cold_store_state( } } +#[cfg(test)] fn checkpoint_path(profile_root: &Path) -> PathBuf { profile_root .join(CHECKPOINT_DIRECTORY) .join(CHECKPOINT_FILE) } +fn admitted_checkpoint_directory(admitted: &Dir, profile_root: &Path) -> io::Result { + let supplied = + Dir::open_ambient_dir(profile_root, cap_std::ambient_authority())?.dir_metadata()?; + let owner = admitted.dir_metadata()?; + if (supplied.dev(), supplied.ino()) != (owner.dev(), owner.ino()) { + return Err(io::Error::other( + "profile directory differs from the admitted owner", + )); + } + checkpoint_directory(admitted) +} + +// Keep the maintenance directory capability open for the whole cursor read or +// publish. A path check followed by a path-based open or rename can be raced +// with a symlink substitution of "maintenance". +fn checkpoint_directory(root: &Dir) -> io::Result { + match root.create_dir(CHECKPOINT_DIRECTORY) { + Ok(()) => {} + Err(error) if error.kind() == io::ErrorKind::AlreadyExists => {} + Err(error) => return Err(error), + } + root.open_dir_nofollow(CHECKPOINT_DIRECTORY) +} + #[tracing::instrument(name = "maintenance.cold_store.load_cursor", level = "trace", skip_all)] -fn load_cursor(path: &Path) -> Option { - let bytes = std::fs::read(path).ok()?; - serde_json::from_slice(&bytes).ok() +fn load_cursor_in_dir(directory: &Dir) -> Option { + let mut options = OpenOptions::new(); + options.read(true).follow(FollowSymlinks::No).nonblock(true); + let file = match directory.open_with(CHECKPOINT_FILE, &options) { + Ok(file) => file, + Err(error) if error.kind() == io::ErrorKind::NotFound => return None, + Err(error) => { + tracing::warn!( + event = "retention_cold_store_cursor_invalid", + %error, + "cannot safely open the saved maintenance cursor" + ); + return None; + } + }; + let metadata = file.metadata().ok()?; + if !metadata.is_file() || metadata.len() > MAX_CHECKPOINT_BYTES { + tracing::warn!( + event = "retention_cold_store_cursor_invalid", + reason = "nonregular_or_oversized", + "saved maintenance cursor is unsafe; restarting its bounded census" + ); + return None; + } + let mut limited = file.take(MAX_CHECKPOINT_BYTES + 1); + let mut bytes = Vec::new(); + limited.read_to_end(&mut bytes).ok()?; + if bytes.len() as u64 > MAX_CHECKPOINT_BYTES { + tracing::warn!( + event = "retention_cold_store_cursor_invalid", + reason = "oversized_after_open", + "saved maintenance cursor grew during its read; restarting its bounded census" + ); + return None; + } + match serde_json::from_slice(&bytes) { + Ok(cursor) => Some(cursor), + Err(error) => { + tracing::warn!( + event = "retention_cold_store_cursor_invalid", + %error, + "saved maintenance cursor is invalid; restarting its bounded census" + ); + None + } + } } #[tracing::instrument( @@ -256,17 +314,9 @@ fn load_cursor(path: &Path) -> Option { level = "trace", skip_all )] -fn persist_cursor(path: &Path, cursor: &ColdStoreCursorV1) -> std::io::Result<()> { - let parent = path - .parent() - .ok_or_else(|| std::io::Error::other("maintenance cursor has no parent"))?; - std::fs::create_dir_all(parent)?; - let temporary = path.with_extension("json.tmp"); - let bytes = serde_json::to_vec(cursor).map_err(std::io::Error::other)?; - let mut file = std::fs::File::create(&temporary)?; - std::io::Write::write_all(&mut file, &bytes)?; - file.sync_all()?; - std::fs::rename(temporary, path) +fn persist_cursor_in_dir(directory: &Dir, cursor: &ColdStoreCursorV1) -> io::Result<()> { + let bytes = serde_json::to_vec(cursor).map_err(io::Error::other)?; + replace_file_in_open_directory(directory, CHECKPOINT_FILE, &bytes) } fn retention_window_secs(days: u64) -> i64 { @@ -280,18 +330,127 @@ fn retention_window_secs(days: u64) -> i64 { mod tests { use super::*; - #[test] - fn cursor_resumes_after_the_last_complete_project() { - let first = next_cold_store_cursor( + fn open_test_directory(path: &Path) -> io::Result { + let root = Dir::open_ambient_dir( + path.parent().unwrap().parent().unwrap(), + cap_std::ambient_authority(), + )?; + checkpoint_directory(&root) + } + + fn load_cursor(path: &Path) -> Option { + load_cursor_in_dir(&open_test_directory(path).ok()?) + } + + fn persist_cursor(path: &Path, cursor: &ColdStoreCursorV1) -> io::Result<()> { + persist_cursor_in_dir(&open_test_directory(path)?, cursor) + } + + #[asupersync::test] + async fn cold_store_cursor_advances_across_projects_without_profile_stores() { + use tracedecay_global_db::tests::harness::RegisteredGlobalDbTestRuntime; + + let temporary = tempfile::tempdir().unwrap(); + let profile = temporary.path().join("profile"); + std::fs::create_dir(&profile).unwrap(); + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + std::fs::set_permissions(&profile, std::fs::Permissions::from_mode(0o700)).unwrap(); + } + let runtime = RegisteredGlobalDbTestRuntime::profile(&profile) + .await + .unwrap(); + let database = runtime.profile_database_arc(); + let transaction = database.begin_write_transaction().await.unwrap(); + for index in 0..=COLD_STORE_PAGE_LIMIT { + let project_id = format!("project-{index:02}"); + let root = temporary + .path() + .join(&project_id) + .to_string_lossy() + .into_owned(); + transaction + .execute( + "INSERT INTO code_projects ( + project_id, canonical_root, display_root, created_at, last_seen_at + ) VALUES (?1, ?2, ?2, 1, 1)", + tracedecay_runtime_core::db::engine::params![project_id, root], + ) + .await + .unwrap(); + } + transaction.commit().await.unwrap(); + + let cancellation = CancellationToken::new(); + let owners = BTreeSet::new(); + let first = run_cold_store_page(&profile, database.as_ref(), None, &owners, &cancellation) + .await + .unwrap(); + assert_eq!(first.processed_stores, 0); + assert_eq!( + load_cursor(&checkpoint_path(&profile)) + .unwrap() + .after_project_id + .as_deref(), + Some("project-07"), + "the registry's page boundary must persist even when its census is empty" + ); + + let second = run_cold_store_page(&profile, database.as_ref(), None, &owners, &cancellation) + .await + .unwrap(); + assert_eq!(second.processed_stores, 0); + assert_eq!( + load_cursor(&checkpoint_path(&profile)) + .unwrap() + .after_project_id, None, - &["project-a".to_owned(), "project-b".to_owned()], - true, - ) - .expect("first page cursor"); - assert_eq!(first, "project-b"); + "the final registry page must finish instead of repeating its first page" + ); + } + + #[cfg(unix)] + #[asupersync::test] + async fn cold_store_cursor_uses_the_directory_its_profile_database_admitted() { + use std::os::unix::fs::symlink; + use tracedecay_global_db::tests::harness::RegisteredGlobalDbTestRuntime; + + let temporary = tempfile::tempdir().unwrap(); + let owner = temporary.path().join("owner"); + let supplied = temporary.path().join("profile"); + let foreign = temporary.path().join("other"); + let foreign_cursor = checkpoint_path(&foreign); + std::fs::create_dir_all(foreign_cursor.parent().unwrap()).unwrap(); + std::fs::write(&foreign_cursor, b"another profile's resume point").unwrap(); + let runtime = RegisteredGlobalDbTestRuntime::profile(&owner) + .await + .unwrap(); + let database = runtime.profile_database_arc(); + symlink(&owner, &supplied).unwrap(); + let owners = BTreeSet::new(); + let cancellation = CancellationToken::new(); + let run = + || run_cold_store_page(&supplied, database.as_ref(), None, &owners, &cancellation); + run().await.unwrap(); + assert!(checkpoint_path(&owner).is_file()); + + std::fs::remove_file(&supplied).unwrap(); + symlink(&foreign, &supplied).unwrap(); + assert!(run().await.is_err()); + std::fs::remove_file(&supplied).unwrap(); + std::fs::rename(&foreign, &supplied).unwrap(); + assert!(run().await.is_err()); + let foreign_cursor = checkpoint_path(&supplied); assert_eq!( - next_cold_store_cursor(Some(&first), &["project-c".to_owned()], false), - None + std::fs::read(&foreign_cursor).unwrap(), + b"another profile's resume point" + ); + assert_eq!( + std::fs::read_dir(foreign_cursor.parent().unwrap()) + .unwrap() + .count(), + 1 ); } @@ -306,10 +465,160 @@ mod tests { persist_cursor(&path, &expected).expect("persist cursor"); - assert_eq!(load_cursor(&path), Some(expected)); + assert_eq!(load_cursor(&path), Some(expected.clone())); + let updated = ColdStoreCursorV1 { + after_project_id: Some("project-c".to_owned()), + after_unregistered_directory: None, + }; + persist_cursor(&path, &updated).expect("replace existing checkpoint"); + assert_eq!(load_cursor(&path), Some(updated)); assert!(!path.with_extension("json.tmp").exists()); } + #[cfg(unix)] + #[test] + fn checkpoint_publication_does_not_follow_foreign_file_symlinks() { + use std::os::unix::fs::symlink; + + let root = tempfile::tempdir().expect("checkpoint root"); + let path = checkpoint_path(root.path()); + std::fs::create_dir_all(path.parent().unwrap()).unwrap(); + let foreign = root.path().join("foreign-data"); + std::fs::write(&foreign, b"unrelated durable bytes").unwrap(); + // Both a leftover fixed temporary name and the destination may be + // replaced by a link. Neither may grant write authority to its target. + let legacy_temporary = path.with_extension("json.tmp"); + symlink(&foreign, &legacy_temporary).unwrap(); + symlink(&foreign, &path).unwrap(); + assert!( + load_cursor(&path).is_none(), + "foreign checkpoint links cannot be adopted as cursors" + ); + + let expected = ColdStoreCursorV1 { + after_project_id: Some("project-c".to_owned()), + after_unregistered_directory: None, + }; + persist_cursor(&path, &expected).expect("replace checkpoint safely"); + assert_eq!(std::fs::read(&foreign).unwrap(), b"unrelated durable bytes"); + assert!( + std::fs::symlink_metadata(&legacy_temporary) + .unwrap() + .file_type() + .is_symlink(), + "foreign legacy temporary entries must not be overwritten" + ); + assert!( + std::fs::symlink_metadata(&path) + .unwrap() + .file_type() + .is_file(), + "a checkpoint symlink is replaced by an ordinary file" + ); + assert_eq!(load_cursor(&path), Some(expected)); + } + + #[cfg(unix)] + #[test] + fn a_linked_maintenance_directory_never_claims_foreign_cursors() { + use std::os::unix::fs::symlink; + + let root = tempfile::tempdir().expect("profile root"); + let profile = root.path().join("profile"); + let foreign = root.path().join("foreign"); + std::fs::create_dir(&profile).unwrap(); + std::fs::create_dir(&foreign).unwrap(); + let path = checkpoint_path(&profile); + let foreign_cursor = foreign.join(CHECKPOINT_FILE); + let foreign_bytes = br#"{"after_project_id":"foreign"}"#; + std::fs::write(&foreign_cursor, foreign_bytes).unwrap(); + symlink(&foreign, profile.join(CHECKPOINT_DIRECTORY)).unwrap(); + + assert_eq!( + load_cursor(&path), + None, + "a linked checkpoint parent cannot supply a foreign resume position" + ); + let expected = ColdStoreCursorV1 { + after_project_id: Some("project-local".to_owned()), + after_unregistered_directory: None, + }; + assert!( + persist_cursor(&path, &expected).is_err(), + "publication must not cross a substituted maintenance directory" + ); + assert_eq!(std::fs::read(&foreign_cursor).unwrap(), foreign_bytes); + assert_eq!( + std::fs::read_dir(&foreign).unwrap().count(), + 1, + "no staging file may be created in the foreign directory" + ); + + std::fs::remove_file(profile.join(CHECKPOINT_DIRECTORY)).unwrap(); + persist_cursor(&path, &expected).unwrap(); + assert_eq!(load_cursor(&path), Some(expected)); + assert_eq!(std::fs::read(&foreign_cursor).unwrap(), foreign_bytes); + } + + #[cfg(unix)] + #[test] + fn a_replaced_parent_cannot_redirect_an_open_checkpoint_capability() { + use std::os::unix::fs::symlink; + + let root = tempfile::tempdir().unwrap(); + let profile = root.path().join("profile"); + let foreign = root.path().join("foreign"); + std::fs::create_dir(&profile).unwrap(); + std::fs::create_dir(&foreign).unwrap(); + let path = checkpoint_path(&profile); + let directory = open_test_directory(&path).unwrap(); + let expected = ColdStoreCursorV1 { + after_project_id: Some("local".to_owned()), + after_unregistered_directory: None, + }; + persist_cursor_in_dir(&directory, &expected).unwrap(); + std::fs::rename(profile.join(CHECKPOINT_DIRECTORY), profile.join("detached")).unwrap(); + let foreign_cursor = foreign.join(CHECKPOINT_FILE); + std::fs::write(&foreign_cursor, br#"{"after_project_id":"foreign"}"#).unwrap(); + symlink(&foreign, profile.join(CHECKPOINT_DIRECTORY)).unwrap(); + + assert_eq!(load_cursor_in_dir(&directory), Some(expected.clone())); + let updated = ColdStoreCursorV1 { + after_project_id: Some("local-next".to_owned()), + after_unregistered_directory: None, + }; + persist_cursor_in_dir(&directory, &updated).unwrap(); + assert_eq!(load_cursor_in_dir(&directory), Some(updated)); + assert_eq!( + std::fs::read(&foreign_cursor).unwrap(), + br#"{"after_project_id":"foreign"}"#, + "a path substitution must not grant the foreign directory write authority" + ); + assert_eq!(std::fs::read_dir(&foreign).unwrap().count(), 1); + assert!(persist_cursor(&path, &expected).is_err()); + assert_eq!(load_cursor(&path), None); + } + + #[test] + fn malformed_and_oversized_checkpoints_do_not_authorize_unbounded_reads() { + let root = tempfile::tempdir().expect("checkpoint root"); + let path = checkpoint_path(root.path()); + std::fs::create_dir_all(path.parent().unwrap()).unwrap(); + + std::fs::write(&path, vec![b' '; (MAX_CHECKPOINT_BYTES + 1) as usize]) + .expect("oversized checkpoint"); + assert_eq!(load_cursor(&path), None); + std::fs::write(&path, b"{invalid").expect("malformed checkpoint"); + assert_eq!(load_cursor(&path), None); + + let valid = ColdStoreCursorV1 { + after_project_id: None, + after_unregistered_directory: None, + }; + persist_cursor(&path, &valid).expect("restore valid checkpoint"); + assert_eq!(load_cursor(&path), Some(valid)); + } + #[test] fn state_distinguishes_missing_unreadable_and_cancelled() { assert_eq!( diff --git a/crates/tracedecay-maintenance/src/retention/footprint.rs b/crates/tracedecay-maintenance/src/retention/footprint.rs index 7c6d4ead0c0..7423a60e7c3 100644 --- a/crates/tracedecay-maintenance/src/retention/footprint.rs +++ b/crates/tracedecay-maintenance/src/retention/footprint.rs @@ -7,6 +7,10 @@ //! cheap is smaller than one real graph container. use std::collections::BTreeMap; +#[cfg(unix)] +use std::collections::HashSet; +#[cfg(unix)] +use std::os::unix::fs::MetadataExt; use std::path::{Path, PathBuf}; use serde::{Deserialize, Serialize}; @@ -231,6 +235,15 @@ pub fn walk_regular_files_with_budget( WalkInterruptionV1::DeadlineExceeded => WalkTerminationV1::DeadlineExceeded, }; } + // The caller's root can itself be a symlink; DirEntry::file_type + // below only protects children. Never attribute a foreign tree's + // bytes to the requested store. + if !std::fs::symlink_metadata(&directory) + .is_ok_and(|metadata| metadata.file_type().is_dir()) + { + visit(&directory, WalkedEntryV1::Unavailable); + continue; + } let Ok(entries) = std::fs::read_dir(&directory) else { visit(&directory, WalkedEntryV1::Unavailable); continue; @@ -408,6 +421,8 @@ pub fn census_sqlite_temp_spill( ) -> Option { let mut bytes = 0u64; let mut unavailable_entry_count = 0usize; + #[cfg(unix)] + let mut seen_inodes = HashSet::new(); for entry in std::fs::read_dir(fd_directory).ok()? { let fd = match entry { Ok(fd) => fd, @@ -431,7 +446,18 @@ pub fn census_sqlite_temp_spill( .is_some_and(|name| name.starts_with("etilqs_") && name.ends_with(" (deleted)")); if unlinked_temp { match std::fs::metadata(fd.path()) { - Ok(metadata) => bytes = bytes.saturating_add(metadata.len()), + Ok(metadata) if metadata.is_file() => { + #[cfg(unix)] + let unique = seen_inodes.insert((metadata.dev(), metadata.ino())); + #[cfg(not(unix))] + let unique = true; + if unique { + bytes = bytes.saturating_add(metadata.len()); + } + } + // The fd can be reused between read_link and metadata; a + // directory or other non-file target is not an SQLite spill. + Ok(_) => {} // A closed descriptor no longer consumes space. Err(error) if error.kind() == std::io::ErrorKind::NotFound => {} Err(_) => unavailable_entry_count = unavailable_entry_count.saturating_add(1), @@ -716,6 +742,11 @@ mod tests { sized(&target, bytes); std::os::unix::fs::symlink(&target, fd_directory.join(fd)).expect("fd link"); } + // A reused descriptor with a matching name but a non-file target + // must not inflate the spill total or fabricate an over-budget state. + let non_file = temporary.path().join("etilqs_directory (deleted)"); + std::fs::create_dir(&non_file).expect("non-file target"); + std::os::unix::fs::symlink(&non_file, fd_directory.join("7")).expect("non-file fd link"); let entry = census_sqlite_temp_spill(&fd_directory, &BTreeMap::new()) .expect("a readable fd directory"); @@ -741,6 +772,42 @@ mod tests { ); } + #[cfg(unix)] + #[test] + fn temp_spill_census_counts_physical_files_not_fd_aliases() { + let temporary = tempfile::tempdir().expect("temporary directory"); + let fd_directory = temporary.path().join("fd"); + std::fs::create_dir(&fd_directory).expect("fd directory"); + let first = temporary.path().join("etilqs_primary (deleted)"); + let alias = temporary.path().join("etilqs_alias (deleted)"); + let second = temporary.path().join("etilqs_distinct (deleted)"); + sized(&first, 650 * MIB); + std::fs::hard_link(&first, &alias).expect("second name for same inode"); + sized(&second, 200 * MIB); + for (fd, target) in [("3", &first), ("4", &alias), ("5", &second)] { + std::os::unix::fs::symlink(target, fd_directory.join(fd)).expect("fd link"); + } + + let entry = census_sqlite_temp_spill(&fd_directory, &BTreeMap::new()) + .expect("readable fd directory"); + assert_eq!(entry.bytes, 850 * MIB); + assert_eq!(entry.unavailable_entry_count, 0); + let footprint = ProfileFootprintV1 { + entries: vec![entry], + total_bytes: 0, + unavailable_entry_count: 0, + truncated: false, + total_bound_bytes: PROFILE_TOTAL_BOUND_BYTES, + }; + assert_eq!( + findings(&footprint)[SQLITE_TEMP_SPILL_KEY] + .finding() + .state(), + DoctorEvidenceStateV1::HealthyCompleteCoverage, + "duplicated descriptors must not create a false over-budget finding", + ); + } + #[cfg(unix)] #[test] fn temp_spill_census_reports_unreadable_fd_entries_without_fabricating_health() { @@ -1070,6 +1137,49 @@ mod tests { assert_eq!(remaining, 100 - 1 - 64 - 1 - 1); } + #[cfg(unix)] + #[test] + fn symlinked_walk_root_is_partial_without_traversing_foreign_files() { + let temporary = tempfile::tempdir().expect("walk fixture"); + let outside = temporary.path().join("outside"); + std::fs::create_dir(&outside).expect("outside directory"); + sized(&outside.join("private-a"), 900); + sized(&outside.join("private-b"), 900); + let linked_root = temporary.path().join("linked-profile"); + std::os::unix::fs::symlink(&outside, &linked_root).expect("linked root"); + + let mut remaining = 1; + let mut visited = Vec::new(); + let termination = walk_regular_files_with_budget( + &linked_root, + &mut remaining, + &|| None, + &mut |path, entry| visited.push((path.to_path_buf(), entry)), + ); + assert_eq!(termination, WalkTerminationV1::Complete); + assert_eq!( + visited, + vec![(linked_root.clone(), WalkedEntryV1::Unavailable)] + ); + assert_eq!( + remaining, 1, + "foreign children must not spend the entry budget" + ); + + let footprint = census_profile_footprint(&linked_root, &BTreeMap::new(), &BTreeMap::new()); + assert_eq!(footprint.total_bytes, 0); + assert_eq!(footprint.unavailable_entry_count, 1); + assert_eq!( + footprint.full_profile_size().state, + ProfileTotalCoverageStateV1::Partial, + "unreadable store roots cannot report healthy complete coverage" + ); + assert_eq!( + std::fs::metadata(outside.join("private-a")).unwrap().len(), + 900 + ); + } + #[test] fn one_entry_budget_is_shared_across_walk_roots() { let temporary = tempfile::tempdir().expect("walk fixture"); diff --git a/crates/tracedecay-private-fs/src/capability_dir.rs b/crates/tracedecay-private-fs/src/capability_dir.rs index c0c846fb8f4..b4b4630ad83 100644 --- a/crates/tracedecay-private-fs/src/capability_dir.rs +++ b/crates/tracedecay-private-fs/src/capability_dir.rs @@ -6,12 +6,16 @@ //! swapped for a symlink cannot redirect the operation. use std::ffi::OsStr; -use std::io; +use std::io::{self, Read}; +use std::path::{Component, Path}; +use std::sync::atomic::{AtomicU64, Ordering}; + +#[cfg(unix)] +use cap_std::fs::OpenOptionsExt; -#[cfg(windows)] -use cap_fs_ext::DirExt; #[cfg(not(windows))] use cap_fs_ext::OpenOptionsMaybeDirExt; +use cap_fs_ext::{DirExt, FollowSymlinks, OpenOptionsFollowExt}; use cap_std::fs::{Dir, File, OpenOptions}; /// How many times a create-or-open that reports the entry missing is asked @@ -212,6 +216,300 @@ pub fn sync_directory(directory: &Dir) -> io::Result<()> { } } +/// Open an owned leaf through a held parent, never following a replaced root +/// symlink. Missing roots are normal before the first artifact is sealed. +pub fn open_owned_directory_nofollow(root: &Path) -> io::Result> { + let name = root + .file_name() + .ok_or_else(|| io::Error::new(io::ErrorKind::InvalidInput, "root has no leaf name"))?; + let parent_path = root + .parent() + .ok_or_else(|| io::Error::new(io::ErrorKind::InvalidInput, "root has no parent"))?; + let parent = match Dir::open_ambient_dir(parent_path, cap_std::ambient_authority()) { + Ok(parent) => parent, + Err(error) if error.kind() == io::ErrorKind::NotFound => return Ok(None), + Err(error) => return Err(error), + }; + match parent.open_dir_nofollow(name) { + Ok(opened) => Ok(Some(opened)), + Err(error) if error.kind() == io::ErrorKind::NotFound => Ok(None), + Err(error) => Err(error), + } +} + +/// Read an optional bounded regular receipt from an already-open child. +/// A replaced symlink or unreadable receipt never authorizes reclamation. +fn read_child_receipt(child: &Dir, receipt: &str, limit: usize) -> io::Result>> { + let mut options = OpenOptions::new(); + options.read(true).follow(FollowSymlinks::No); + let mut file = match child.open_with(receipt, &options) { + Ok(file) => file, + Err(error) if error.kind() == io::ErrorKind::NotFound => return Ok(None), + Err(error) => return Err(error), + }; + if !file.metadata()?.is_file() { + return Ok(None); + } + let mut bytes = Vec::new(); + file.by_ref() + .take((limit as u64).saturating_add(1)) + .read_to_end(&mut bytes)?; + if bytes.len() > limit { + return Ok(None); + } + Ok(Some(bytes)) +} + +/// Open a generation beneath a held, no-follow sealed root. Both the root +/// and the generation leaf must be real directories. +fn open_nested_directory_nofollow(directory: &Path) -> io::Result { + let root = directory + .parent() + .ok_or_else(|| io::Error::new(io::ErrorKind::InvalidInput, "directory has no root"))?; + let name = directory + .file_name() + .ok_or_else(|| io::Error::new(io::ErrorKind::InvalidInput, "directory has no name"))?; + let root = open_owned_directory_nofollow(root)? + .ok_or_else(|| io::Error::from(io::ErrorKind::NotFound))?; + root.open_dir_nofollow(name) +} + +fn checked_leaf_name(name: &str) -> io::Result<&OsStr> { + let mut components = Path::new(name).components(); + if !matches!(components.next(), Some(Component::Normal(_))) || components.next().is_some() { + return Err(io::Error::new( + io::ErrorKind::InvalidInput, + "proof file name must be one leaf", + )); + } + Ok(OsStr::new(name)) +} + +/// Read bounded regular proof bytes beneath a held root and generation. +/// Missing files and static symlinks never provide reusable proof. +pub fn read_nested_bounded_file_nofollow( + directory: &Path, + name: &str, + limit: usize, +) -> io::Result>> { + checked_leaf_name(name)?; + let child = match open_nested_directory_nofollow(directory) { + Ok(child) => child, + Err(error) if error.kind() == io::ErrorKind::NotFound => return Ok(None), + Err(error) => return Err(error), + }; + match child.symlink_metadata(name) { + Ok(metadata) if !metadata.is_file() => return Ok(None), + Ok(_) => {} + Err(error) if error.kind() == io::ErrorKind::NotFound => return Ok(None), + Err(error) => return Err(error), + } + read_child_receipt(&child, name, limit) +} + +static PROOF_TEMP_NONCE: AtomicU64 = AtomicU64::new(0); + +/// Publish a proof atomically relative to the same held generation used for +/// staging. A substituted root/child link cannot redirect or truncate bytes. +pub fn replace_nested_file_nofollow(directory: &Path, name: &str, bytes: &[u8]) -> io::Result<()> { + let child = open_nested_directory_nofollow(directory)?; + replace_file_in_open_directory(&child, name, bytes) +} + +/// Atomically replaces a single file beneath a directory held by its owner. +pub fn replace_file_in_open_directory(directory: &Dir, name: &str, bytes: &[u8]) -> io::Result<()> { + use std::io::Write; + + let leaf = checked_leaf_name(name)?; + + for _ in 0..16 { + let nonce = PROOF_TEMP_NONCE.fetch_add(1, Ordering::Relaxed); + let temporary_name = format!(".sealed-proof-{}-{nonce}.tmp", std::process::id()); + let mut options = OpenOptions::new(); + options + .write(true) + .create_new(true) + .follow(FollowSymlinks::No); + #[cfg(unix)] + options.mode(0o600); + let mut temporary = match directory.open_with(&temporary_name, &options) { + Ok(file) => file, + Err(error) if error.kind() == io::ErrorKind::AlreadyExists => continue, + Err(error) => return Err(error), + }; + let result = (|| { + temporary.write_all(bytes)?; + temporary.sync_all()?; + drop(temporary); + replace_file(directory, OsStr::new(&temporary_name), leaf)?; + sync_directory(directory) + })(); + if result.is_err() { + let _ = directory.remove_file(&temporary_name); + } + return result; + } + Err(io::Error::new( + io::ErrorKind::AlreadyExists, + "sealed proof staging names are occupied", + )) +} + +/// Capability-relative census classes. The caller decides which +/// bounded receipt identities are recognizable, while private-fs owns all +/// directory traversal, metadata and hardlink accounting. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub struct DirectoryCensus { + pub counts: [u64; N], + pub bytes: [u64; N], + pub unrecognized: u64, +} + +/// Classify direct children under one held, no-follow root, then charge live +/// classes in caller-specified priority order. A root swap cannot redirect +/// receipt reads or recursive size walks; an unreadable/oversized receipt +/// grants no classification. Substituted unsafe children fail closed. +pub fn census_child_directories_nofollow( + root: &Path, + receipt: Option<(&str, usize)>, + mut classify: impl FnMut(&OsStr, Option<&[u8]>) -> Option, +) -> io::Result>> { + let Some(directory) = open_owned_directory_nofollow(root)? else { + return Ok(None); + }; + let mut classes: [Vec; N] = std::array::from_fn(|_| Vec::new()); + let mut result = DirectoryCensus { + counts: [0; N], + bytes: [0; N], + unrecognized: 0, + }; + for entry in directory.read_dir(".")? { + let entry = entry?; + if !entry.file_type()?.is_dir() { + result.unrecognized = result.unrecognized.saturating_add(1); + continue; + } + let name = entry.file_name(); + let child = directory.open_dir_nofollow(&name)?; + let receipt_bytes = receipt + .and_then(|(file, limit)| read_child_receipt(&child, file, limit).ok().flatten()); + match classify(&name, receipt_bytes.as_deref()) { + Some(index) if index < N => { + result.counts[index] = result.counts[index].saturating_add(1); + classes[index].push(child); + } + _ => result.unrecognized = result.unrecognized.saturating_add(1), + } + } + + // A live layered generation may hard-link files from a superseded base. + // The first class to charge a multiply linked inode owns those bytes. + let mut counted = std::collections::HashSet::new(); + for (index, children) in classes.into_iter().enumerate() { + for child in children { + result.bytes[index] = + result.bytes[index].saturating_add(directory_bytes_nofollow(child, &mut counted)?); + } + } + Ok(Some(result)) +} + +fn directory_bytes_nofollow( + root: Dir, + counted: &mut std::collections::HashSet<(u64, u64)>, +) -> io::Result { + let mut total = 0_u64; + let mut pending = vec![root]; + while let Some(directory) = pending.pop() { + for entry in directory.read_dir(".")? { + let entry = entry?; + let name = entry.file_name(); + if entry.file_type()?.is_dir() { + pending.push(directory.open_dir_nofollow(&name)?); + continue; + } + let metadata = directory.symlink_metadata(&name)?; + #[cfg(unix)] + { + use cap_fs_ext::MetadataExt; + if metadata.nlink() > 1 && !counted.insert((metadata.dev(), metadata.ino())) { + continue; + } + } + #[cfg(not(unix))] + let _ = &counted; + total = total.saturating_add(metadata.len()); + } + } + Ok(total) +} + +/// Enumerate and reclaim only selected real child directories under one held +/// no-follow root. Selection (including optional receipt evidence) and removal +/// refer to the *same opened child*: a root or child swap cannot redirect the +/// deletion into a foreign pathname. An unsafe entry is preserved. +pub fn remove_selected_child_directories_nofollow( + root: &Path, + receipt: Option<(&str, usize)>, + mut eligible: impl FnMut(&OsStr, Option<&[u8]>) -> bool, +) -> io::Result> { + let Some(directory) = open_owned_directory_nofollow(root)? else { + return Ok(None); + }; + let mut removed = 0usize; + for entry in directory.read_dir(".")? { + let entry = entry?; + if !entry.file_type()?.is_dir() { + continue; + } + let name = entry.file_name(); + let child = match directory.open_dir_nofollow(&name) { + Ok(child) => child, + Err(error) if error.kind() == io::ErrorKind::NotFound => continue, + Err(error) => { + tracing::warn!( + event = "private_fs.cleanup_unsafe_child", + root = %root.display(), + child = ?name, + %error, + "unverifiable cleanup child preserved" + ); + continue; + } + }; + let receipt_bytes = match receipt { + Some((file, limit)) => read_child_receipt(&child, file, limit).ok().flatten(), + None => None, + }; + if eligible(&name, receipt_bytes.as_deref()) { + remove_open_dir_all_nofollow(child)?; + removed += 1; + } + } + Ok(Some(removed)) +} + +/// Remove an exact child of the held root. Unlike a path-based recursive +/// removal this never follows a replacement of the root or child symlink. +pub fn remove_named_child_directory_nofollow(root: &Path, name: &OsStr) -> io::Result<()> { + let mut components = Path::new(name).components(); + if !matches!(components.next(), Some(Component::Normal(_))) || components.next().is_some() { + return Err(io::Error::new( + io::ErrorKind::InvalidInput, + "removal requires one child directory name", + )); + } + let Some(directory) = open_owned_directory_nofollow(root)? else { + return Ok(()); + }; + let child = match directory.open_dir_nofollow(name) { + Ok(child) => child, + Err(error) if error.kind() == io::ErrorKind::NotFound => return Ok(()), + Err(error) => return Err(error), + }; + remove_open_dir_all_nofollow(child) +} + /// Removes the held directory without following child symlinks. pub fn remove_open_dir_all_nofollow(directory: Dir) -> io::Result<()> { #[cfg(not(windows))] @@ -241,6 +539,115 @@ mod replacement_tests { use super::*; + #[cfg(unix)] + #[test] + fn census_uses_held_root_after_foreign_symlink_swap() { + use std::os::unix::fs::symlink; + + let temp = tempfile::tempdir().unwrap(); + let owned = temp.path().join("owned"); + let moved = temp.path().join("moved"); + let foreign = temp.path().join("foreign"); + std::fs::create_dir_all(owned.join("generation")).unwrap(); + std::fs::create_dir_all(foreign.join("generation")).unwrap(); + std::fs::write(owned.join("generation").join("payload"), vec![1_u8; 41]).unwrap(); + std::fs::write( + foreign.join("generation").join("payload"), + vec![2_u8; 100_041], + ) + .unwrap(); + + let mut swapped = false; + let measured = census_child_directories_nofollow::<1>(&owned, None, |name, _| { + assert_eq!(name, OsStr::new("generation")); + assert!(!swapped); + std::fs::rename(&owned, &moved).unwrap(); + symlink(&foreign, &owned).unwrap(); + swapped = true; + Some(0) + }) + .unwrap() + .unwrap(); + assert!(swapped); + assert_eq!(measured.counts, [1]); + assert_eq!(measured.bytes, [41]); + assert_eq!( + std::fs::metadata(foreign.join("generation").join("payload")) + .unwrap() + .len(), + 100_041 + ); + assert_eq!( + std::fs::metadata(moved.join("generation").join("payload")) + .unwrap() + .len(), + 41 + ); + } + + #[cfg(unix)] + #[test] + fn census_never_descends_foreign_symlink_children() { + use std::os::unix::fs::symlink; + + let temp = tempfile::tempdir().unwrap(); + let owned = temp.path().join("owned"); + let foreign = temp.path().join("foreign"); + std::fs::create_dir_all(owned.join("generation")).unwrap(); + std::fs::create_dir_all(&foreign).unwrap(); + std::fs::write(foreign.join("payload"), vec![0_u8; 100_041]).unwrap(); + std::fs::write(owned.join("generation").join("local"), vec![0_u8; 41]).unwrap(); + symlink(&foreign, owned.join("generation").join("linked")).unwrap(); + let measured = census_child_directories_nofollow::<1>(&owned, None, |_, _| Some(0)) + .unwrap() + .unwrap(); + assert_eq!(measured.counts, [1]); + assert!(measured.bytes[0] >= 41); + assert!(measured.bytes[0] < 100_041); + } + + #[cfg(unix)] + #[test] + fn nested_proof_stays_with_open_generation_when_root_is_swapped() { + use std::os::unix::fs::symlink; + + let temp = tempfile::tempdir().unwrap(); + let root = temp.path().join("graph.sealed"); + let moved = temp.path().join("former-sealed"); + let foreign = temp.path().join("foreign"); + let generation = root.join("generation"); + std::fs::create_dir_all(&generation).unwrap(); + std::fs::create_dir_all(foreign.join("generation")).unwrap(); + std::fs::write(foreign.join("generation/sealed.checked"), b"foreign").unwrap(); + + let held = open_nested_directory_nofollow(&generation).unwrap(); + std::fs::rename(&root, &moved).unwrap(); + symlink(&foreign, &root).unwrap(); + replace_file_in_open_directory(&held, "sealed.checked", b"verified").unwrap(); + assert_eq!( + std::fs::read(moved.join("generation/sealed.checked")).unwrap(), + b"verified" + ); + assert_eq!( + std::fs::read(foreign.join("generation/sealed.checked")).unwrap(), + b"foreign" + ); + assert!(replace_nested_file_nofollow(&generation, "sealed.checked", b"unsafe").is_err()); + assert!(read_nested_bounded_file_nofollow(&generation, "sealed.checked", 64).is_err()); + + std::fs::remove_file(&root).unwrap(); + std::fs::rename(&moved, &root).unwrap(); + replace_nested_file_nofollow(&generation, "sealed.checked", b"fresh").unwrap(); + assert_eq!( + read_nested_bounded_file_nofollow(&generation, "sealed.checked", 64).unwrap(), + Some(b"fresh".to_vec()) + ); + assert_eq!( + std::fs::read(foreign.join("generation/sealed.checked")).unwrap(), + b"foreign" + ); + } + #[test] fn replacement_publishes_complete_bytes_and_refuses_parent_traversal() { let root = tempfile::tempdir().unwrap(); @@ -267,6 +674,47 @@ mod replacement_tests { assert!(!root.path().join("staged").exists()); } + #[cfg(unix)] + #[test] + fn held_cleanup_does_not_follow_a_root_swap_into_foreign_data() { + use std::os::unix::fs::symlink; + + let temp = tempfile::tempdir().unwrap(); + let root = temp.path().join("graph.sealed"); + let former = temp.path().join("former-sealed"); + let foreign = temp.path().join("foreign"); + std::fs::create_dir(&root).unwrap(); + std::fs::create_dir(&foreign).unwrap(); + for parent in [&root, &foreign] { + std::fs::create_dir(parent.join(".staging-owned")).unwrap(); + std::fs::write(parent.join(".staging-owned/keep"), b"sentinel").unwrap(); + } + let mut swapped = false; + let removed = remove_selected_child_directories_nofollow(&root, None, |name, _| { + if !swapped { + std::fs::rename(&root, &former).unwrap(); + symlink(&foreign, &root).unwrap(); + swapped = true; + } + name == OsStr::new(".staging-owned") + }) + .unwrap(); + assert!(swapped); + assert_eq!(removed, Some(1)); + assert!(!former.join(".staging-owned").exists()); + assert_eq!( + std::fs::read(foreign.join(".staging-owned/keep")).unwrap(), + b"sentinel" + ); + assert!( + remove_named_child_directory_nofollow(&root, OsStr::new(".staging-owned")).is_err() + ); + assert_eq!( + std::fs::read(foreign.join(".staging-owned/keep")).unwrap(), + b"sentinel" + ); + } + #[cfg(windows)] #[test] fn no_replace_publication_preserves_an_occupied_entry() { diff --git a/crates/tracedecay-runtime-core/src/db/access.rs b/crates/tracedecay-runtime-core/src/db/access.rs index f9bf5182162..625ac3e16f8 100644 --- a/crates/tracedecay-runtime-core/src/db/access.rs +++ b/crates/tracedecay-runtime-core/src/db/access.rs @@ -139,6 +139,7 @@ struct MaintenanceScopeState { #[derive(Debug)] struct AuthorityInner { identity: DatabaseIdentity, + profile_directory: Option, role: DatabaseAuthorityRole, token: String, } @@ -353,10 +354,37 @@ impl DatabaseAuthority { role: DatabaseAuthorityRole, intent: &str, ) -> Result { + // Cold-store checkpoints belong to the profile that admitted its sessions + // database. Retain that directory before publishing its writer authority. + let profile_directory = if identity.database_path + == identity + .profile_root + .join(crate::storage::USER_SESSIONS_DB_FILENAME) + { + Some( + tracedecay_private_fs::capability_dir::open_owned_directory_nofollow( + &identity.profile_root, + ) + .map_err(|error| { + access_io_error("open profile directory", &identity.profile_root, &error) + })? + .ok_or_else(|| { + access_error( + "open profile directory", + &identity.profile_root, + "profile directory disappeared", + ) + })? + .into_std_file(), + ) + } else { + None + }; let token = acquire_process_lease(&identity, role, intent)?; Ok(Self { inner: Arc::new(AuthorityInner { identity, + profile_directory, role, token, }), @@ -375,6 +403,23 @@ impl DatabaseAuthority { Ok(self.clone()) } + pub fn profile_sessions_directory(&self) -> Result { + let operation = "access admitted profile directory"; + self.require_active_write_scope(operation)?; + self.inner + .profile_directory + .as_ref() + .ok_or_else(|| { + access_error( + operation, + &self.inner.identity.database_path, + "a profile sessions writer is required", + ) + })? + .try_clone() + .map_err(|error| access_io_error(operation, &self.inner.identity.profile_root, &error)) + } + pub fn canonical_database_path(&self) -> &Path { &self.inner.identity.database_path } diff --git a/crates/tracedecay/src/daemon/maintenance.rs b/crates/tracedecay/src/daemon/maintenance.rs index ea2070195f9..9e293532be4 100644 --- a/crates/tracedecay/src/daemon/maintenance.rs +++ b/crates/tracedecay/src/daemon/maintenance.rs @@ -386,6 +386,10 @@ fn record_canonical_body_gc_outcome( &[ ("roots_scanned", report.roots_scanned.to_string()), ("bodies_scanned", report.bodies_scanned.to_string()), + ( + "candidates_discharged", + report.candidates_discharged.to_string(), + ), ("bodies_deleted", report.bodies_deleted.to_string()), ( "stored_bytes_deleted", @@ -428,6 +432,20 @@ fn record_replay_metadata_outcome( "retention_replay_metadata", &[ ("rows_scanned", report.rows_scanned.to_string()), + ( + "frontier_before", + report.frontier_before.map_or_else( + || "not_scanned".to_owned(), + |value| value.to_string(), + ), + ), + ( + "frontier_after", + report.frontier_after.map_or_else( + || "not_scanned".to_owned(), + |value| value.to_string(), + ), + ), ("rows_compacted", report.rows_compacted.to_string()), ("rows_skipped", report.rows_skipped.to_string()), ( diff --git a/crates/tracedecay/tests/automation_runner_test/jobs.rs b/crates/tracedecay/tests/automation_runner_test/jobs.rs index 7342786381b..b943dff53b1 100644 --- a/crates/tracedecay/tests/automation_runner_test/jobs.rs +++ b/crates/tracedecay/tests/automation_runner_test/jobs.rs @@ -987,6 +987,28 @@ async fn user_job_backend_failure_records_failed_ledger_entry() { assert_eq!(records[0].status, AutomationRunStatus::Failed); } +#[tokio::test] +async fn scheduler_prefilter_reports_unconfigured_codex_before_schedule_admission() { + let temp = tempdir().unwrap(); + let dashboard_root = temp.path().join("dashboard"); + let mut job = sample_job("unconfigured-prefilter"); + job.schedule = Some("manual".to_owned()); + + let skip = evaluate_and_record_scheduler_skip( + &dashboard_root, + &enabled_job_config(), + None, + &job, + "unconfigured-prefilter-occurrence", + None, + ) + .await + .unwrap() + .expect("missing executable must produce a typed diagnostic"); + assert_eq!(skip.report["reason"], json!("backend_unconfigured")); + assert_eq!(skip.ledger_record.status, AutomationRunStatus::Skipped); +} + #[tokio::test] async fn scheduler_trigger_skips_repeat_and_respects_lock_discipline() { let temp = tempdir().unwrap(); diff --git a/docs/LEAN-CTX-COMPARE.md b/docs/LEAN-CTX-COMPARE.md index 80d39b1052d..5ea251f4ca3 100644 --- a/docs/LEAN-CTX-COMPARE.md +++ b/docs/LEAN-CTX-COMPARE.md @@ -12,7 +12,7 @@ through its daemon and product contract. | Concern | TraceDecay | lean-ctx | |---|---|---| | Code and evidence | A daemon-owned authority serves code, project-memory, and session evidence with exact provenance, generation, coverage, and receipts. | Describes a local context layer for read modes, shell output, request-context compression, and context-use visibility. | -| Graph and durable state | Grafeo owns admitted graph/vector data; SQLite owns relational/content records. Clients do not open either store directly. | Describes a property graph and portable context packages alongside its context-management features. | +| Graph and durable state | Grafeo owns admitted graph data; SQLite owns relational/content records. Clients do not open either store directly. | Describes a property graph and portable context packages alongside its context-management features. | | Session context | Lossless Context Memory retains raw-message evidence and summary nodes, with bounded replay and drill-down through daemon operations. | Describes persistent session memory and recovery across chats. | | Workflow and multi-agent work | The V2 product includes the daemon-owned work graph and typed workflow runtime: task evidence, ownership, authorization, execution, cancellation, recovery, collaboration, and receipts are product data, not host scripts. | Describes agent handoff and shared-context features. | | Operating model | Hosts submit bounded hints and typed operations; daemon convergence, freshness, and maintenance remain explicit. | Describes agent wrappers, shell hooks, and an optional request proxy that can mediate context on the read and model-request paths. | diff --git a/docs/V2-OPERATING-MODEL.md b/docs/V2-OPERATING-MODEL.md index 62d812a2f8a..d11149e1472 100644 --- a/docs/V2-OPERATING-MODEL.md +++ b/docs/V2-OPERATING-MODEL.md @@ -10,7 +10,7 @@ capabilities are available. ## Authorities - `tracedecay-graph-db` is the only TraceDecay boundary for the workspace-pinned - embedded Grafeo `=0.5.42` runtime. It exposes typed TraceDecay identities, + embedded Grafeo runtime. It exposes typed TraceDecay identities, operations, snapshots, generations, and outcomes; Grafeo handles and types do not escape it. - Grafeo is in-process and embedded. It has no graph server, sidecar, network diff --git a/docs/dashboard.md b/docs/dashboard.md index 9ebc37ea9b8..b84cfb6d71b 100644 --- a/docs/dashboard.md +++ b/docs/dashboard.md @@ -25,8 +25,8 @@ repository and generation provenance, coverage, and receipts. Warming, partial, unavailable, denied, cancelled, retained, and recreation-required outcomes are distinct UI states, never a blank successful panel. -Structure views present Grafeo-backed graph and vector results supplied by the -daemon. Conversation and memory views hydrate authorized content from its owning +Structure views present Grafeo-backed graph results supplied by the daemon. +Conversation and memory views hydrate authorized content from its owning authority. Work controls separate proposal, authorization, execution, and repository effects. Diagnostics are read-only; maintenance actions are explicit daemon operations with authorization and receipts. diff --git a/docs/graph-at-rest/README.md b/docs/graph-at-rest/README.md index 5a17164514e..d0ad8837a24 100644 --- a/docs/graph-at-rest/README.md +++ b/docs/graph-at-rest/README.md @@ -1,9 +1,5 @@ # An at-rest form for the graph: measurement and verdict -Bench-lane investigation. Nothing here is wired into production; the two -harnesses are `#[ignore]`d and the one new crate method is gated on -`test-helpers` so it cannot compile into a shipped binary. - ## The question Activating a sealed generation replays every node and edge into grafeo's in-RAM @@ -31,17 +27,8 @@ This investigation measured what calling it would buy. > and served read-only. That dissolves blocker 2 (nothing ever writes to a > sealed artifact), the pinned fork's compact-store property hash index > dissolves blocker 1 (sealed reopens register `INDEXED_PROPERTIES` exactly -> like staging opens), and vector search deliberately never routes to sealed -> stores. Bytes round-trip the pinned dictionary codec losslessly now, but -> `COMPACT_ROUND_TRIPS_BYTES` stays **off**: a real 431-file code generation -> (~45k entities carrying serialized-record Bytes payloads) fails its -> post-reopen recovered-digest proof in compact form with "relation scalar -> endpoints do not match native topology" and stays permanently unseated -> behind activation retries, while the same corpus seals and serves in replay -> form; the toy-scale sealed-store contract compacts and reads exactly, so -> the defect only shows at generation scale. This document remains the -> measurement record for the whole-database compaction that was measured and -> rejected. +> like staging opens). This document remains the measurement record for the +> whole-database compaction that was measured and rejected. > > **Refreshes seal as layers (graph format 7):** a small refresh no longer > re-encodes its whole generation. Its sealed store is a delta container @@ -56,18 +43,6 @@ This investigation measured what calling it would buy. > becomes the next base. Stores sealed under format 6 are superseded and > rebuild cold; no migration exists. -## Verdict - -**`compact()` cannot be applied to TraceDecay's graph as it is currently -shaped.** Not "does not pay off", it returns an error. - -The blocker is TraceDecay's own schema, not grafeo. And under a schema that -does not trip it, the columnar form is a large, real win: **10.9x faster -activation at 1M rows, 2.4x lower resident memory, 185x faster adjacency.** - -So the at-rest work is worth doing, but the first ticket is a schema change, -not a storage change. - ## Numbers All figures from a single box, `--test-threads=1`, one scenario per process @@ -308,11 +283,6 @@ close/reopen verification. Small: the crate already holds `compact_snapshot_for_bench` does today. Everything downstream (section write, section reload, layered rewiring) is already implemented in grafeo. -**Phase 3, restore vector-index planning.** Either override -`has_vector_index` on the layered store so it consults the overlay and the -vector section, or hoist the check in `src/runtime.rs` off `graph_store()`. -Small, but do not skip it: gap 3 fails silently. - **Phase 4, optional fork work.** Streaming section serialize (gap 5) and mmap-backed container open (gap 6). Both are pure wins on top of a working path, and neither is needed for the activation number. @@ -424,104 +394,6 @@ contract failures, a sealed generation reopened twice came back with its format marker gone, which is the same defect reached through TraceDecay's own lifecycle rather than the minimal probe. -### The vector index is durable now, fork branch ready, pin not moved - -The claim above that the HNSW index "was never durable across a reopen -anyway" was true, and it turned out to be a wiring gap rather than a -missing format. - -**Where durability was lost.** grafeo has always *written* the index. -`build_sections` emits a `SectionType::VectorStore` carrying each index's -full HNSW topology, entry point, max level, per-node per-level neighbour -lists, and `CatalogSection::collect_indexes` records every index's -label, property, dimensions, metric, M and ef_construction beside it. The -restore side dropped both: - -- `CatalogSection::deserialize` handed the loader its `property_indexes` - and nothing else. The `vector_indexes` it had just decoded were - discarded (`catalog_section.rs`). -- `load_from_sections` guarded the topology restore on - `store.vector_index_entries()` being non-empty - (`database/mod.rs`, the `SectionType::VectorStore` block). A cold open - builds its `LpgStore` from nothing, so that map is empty at exactly the - moment the guard runs. The section bytes were read off disk and - dropped on the floor. - -So the index was durable on disk and absent in memory, and every reopen -re-indexed the whole corpus. - -**Fork, not sidecar.** The section format round-trips HNSW faithfully -already: topology is all it needs to carry, because the vectors live in -LPG node properties (persisted anyway) and are read through a -`VectorAccessor` at query time. Nothing was missing from the format, only -the restore wiring, a tracedecay-side sidecar would have duplicated a -working serializer to re-derive an artifact grafeo already had on disk, -and left every other consumer broken. - -**Branch:** `tracedecay/0.5.42-vector-index-durable`, stacked on -`tracedecay/0.5.42-close-and-overlay`, at -`f38218653dfc69fda67f9c669371036fde1ed5fe`. Three commits: - -1. re-register catalog definitions as empty indexes and let the section - fill their topology. Two incomplete-restore cases are deliberately - left absent for rebuild rather than half-restored, a definition with - no section, and a definition the section carried no topology for, - because an index that exists and covers nothing reports - `has_vector_index() == true` and answers every search with nothing. - Quantized indexes are left to a rebuild too: their codebook lives - inside the index and the section carries only topology, so restoring - one would silently downgrade it to full precision. The catalog - snapshot is v2 to record the mode; v1 files still load. -2. mark index creation, removal, and binding changes as unlogged state. - `close` may skip its flush when the WAL has not moved, and nothing - about an index reaches the WAL, so an index built after open never - reached the file, and the next open rebuilt it, and the next. Harmless - while indexes were always rebuilt; load-bearing the moment they are - restored. -3. price it: 5,000 x 64-dim vectors, release profile, **8.7ms** to - restore the index as part of opening the whole database against - **359.6ms** to rebuild the index alone on a database already open. - ~41x, and identical neighbour sets across 24 probes. - -The branch also adds an opaque per-index **binding token** that rides the -catalog section, so a caller can stamp the generation digest its index -was built over and compare after a reopen instead of trusting that a -restored topology still describes the rows. - -**The pin has moved.** The vector-durability commits were reintegrated -onto the fork's `fix/catalog-version-guard` branch (ported over the -out-of-place checkpoint and close-skip storage rewrite that landed in -the meantime) and the five `grafeo-*` revs in `[patch.crates-io]` now -point at that lineage. The reopen tests in `tests/graph_db_suite/runtime_contract.rs` -assert `Available`, restored, within the same admission bound that -proves no rebuild ran, and the fork gates the semantics with -`vector_index_reopen`, `vector_index_restore_cost`, and -`torn_vector_checkpoint` (a checkpoint killed at every injection point -over a populated HNSW index must reopen and serve search). - -### `has_vector_index`, resolved, and worse than gap 3 described - -Gap 3 said `CompactStore` inherits the trait default `false`. The layered store -is more specific than that: `LayeredStore::has_vector_index` delegates to the -*overlay* (`compact/layered.rs:1081`), which `compact()` leaves empty, so it -answers `false` for indexes the store had a moment earlier. That part is at -least truthful, the HNSW index really is gone, and it was never durable across -a reopen anyway. - -The silent failure is one layer along. `create_vector_index` builds by scanning -`nodes_by_label(label)`, and TraceDecay's vector label is -`entity_projection_label`, one of several labels on an entity node. On a -compacted base that scan matches nothing, so the build succeeds over zero -vectors, `has_vector_index` then answers `true`, and every vector search -returns empty while reporting an index exists. - -The resolution, when phase 2 becomes unblocked, is the routed one: refuse to -build an HNSW index over a store serving from a columnar base, with a typed -error rather than an empty index, and keep a generation that carries vector -indexes in its replay form. Both halves were implemented against the seal-time -compaction and reverted with it; neither is reachable while nothing calls -`compact()` outside the bench lane. - ### Why "never write after compact" is not a guarantee TraceDecay can make The obvious containment for blocker 2 is to promise the store is never mutated @@ -563,19 +435,10 @@ the heap-copy path and are a floor, not a ceiling. every reopened store ran without them. That was invisible while identity resolved through the intrinsic label index, and cost 23.7s for 64 point reads at 500k once it did not. -- **Phase 2, seal implies compact. Landed as per-generation stores; compact - form still gated.** The whole-database `compact()` this plan scoped stays - rejected; the sealed-store lane (`src/sealed_store.rs`) seals each - generation into its own single-generation database, exactly the - "per-generation stores" design blocker 2 demanded, and compacts it when - the rows carry no Bytes or Vector properties. The fork's compact property - index settles blocker 1's economics and the pinned dictionary codec - round-trips Bytes losslessly, but Bytes-carrying generations stay in replay - form behind `COMPACT_ROUND_TRIPS_BYTES` until a generation-scale compact - seal passes its post-reopen proof (see the constant's doc for the measured - scale failure). -- **Phase 3, vector planning. Not applicable to sealed stores:** vector - search never routes to a sealed artifact; HNSW indexes are rebuilt against - the staging database (`apply_sealed_copy_batch` doc). +- **Phase 2, seal implies compact. Landed as per-generation stores.** The + whole-database `compact()` this plan scoped stays rejected; the sealed-store + lane (`src/sealed_store.rs`) seals each generation into its own + single-generation database, exactly the "per-generation stores" design + blocker 2 demanded. - **Phase 4, streaming section writer, mmap container open.** Unchanged, and now clearly behind a property index for the columnar base in priority. diff --git a/docs/plans/storage/README.md b/docs/plans/storage/README.md index 67e644e9a0f..323ccda22a8 100644 --- a/docs/plans/storage/README.md +++ b/docs/plans/storage/README.md @@ -158,14 +158,10 @@ this redesign does not need to settle; the last cutover step measures it. initialized subdirectory of a worktree (a monorepo subproject, which today's `discover_project_root` supports), the pair of root commit and relative path. On this host 7 of the 14 tracedecay clones have a local-path origin, 5 https and - 2 ssh, so only the root commit converges all of them; the 14 clones and 32 linked - worktrees resolve to one project. + 2 ssh; the 14 clones and 32 linked worktrees resolve to one project. - A project never re-keys. An unshallowed clone, a first commit in an unborn repository, or a new root after a fetch adds an alias to the existing project. So durable memory is never stranded under an old id. -- `tracedecay project link ` and `tracedecay project split` record an - explicit alias or refusal in `catalog.db`, scoped to this profile, for forks that - should not share and unrelated repositories that should. - A cwd outside any repository and not explicitly initialized belongs to `projects/_profile/`. It never mints a project of its own. - Code indexes stay per worktree in `worktrees//`. Segments are @@ -275,9 +271,9 @@ in. reconnect. Batched writes take one savepoint per job, so a job's ordinary error rolls back only that job. A suspect-connection error rolls back the whole batch; each job of the batch then runs alone on a fresh connection. -- One fixed pool, `tracedecay-store-N` (default 6, clamp 2 to 16), serves every - file. A lane is per-file state: a write queue (the existing deficit-round-robin - `FairQueue`), a read queue, one write connection, at most two read connections, +- One fixed pool, `tracedecay-store-N`, serves every file. A lane is per-file + state: a write queue (the existing deficit-round-robin `FairQueue`), a read + queue, one write connection, at most two read connections, and fence state. Any pool thread can run any lane; at most one thread holds a lane's write connection at a time. One thread is reserved for foreground classes (catalog, journal, memory) so bulk derivation cannot starve Work. Open lanes are @@ -368,24 +364,10 @@ Each growth source from #4099: Each file carries `user_version` and a schema digest. The rule follows the class: - authoritative and durable classes (catalog, journal, memory, evidence): - forward-only, additive migrations in the crate, applied by the executor at open. - A non-additive change copies into `.next` and renames over the file; no - copy is kept. No user-authored state is reset by a schema change. + admit only the exact final shape. Any other shape returns typed + `ResetRequired`; there is no migration. - rebuildable classes (derived, code): a mismatch means unlink and rebuild. -Upgrade path for beta installs: `tracedecay wipe --stale` plus `tracedecay init` -remains the reset for anything this design does not carry. Durable classes are -carried, not reset. Because the store key stays `ProjectId` and the directory stays -`projects//`, each project converts in place the first time the new -binary opens it: `memory_v2_*` rows and the anchors they cite move from -`tracedecay.db` to `memory.db`, workflow definitions and native-integration -approvals move from `sessions.db` to `journal.db`, configuration revisions move from -`global.db` to `catalog.db`, and the old files are deleted after the copy commits. -Evidence is re-ingested from host files rather than copied. The conversion runs once -per file, is deleted two releases later, and is not a compatibility reader: no code -reads the old files after the conversion. The operator host was wiped on 2026-10-16 -and needs none of it. - ### 8. Public port surface | Type | Public methods | @@ -512,7 +494,8 @@ Act on, and the change made: `forget` deletes durable data. - Evidence and journal were reset or unlinked on a schema change, destroying LLM summaries, history past host retention, workflow definitions and approvals (A, B, - C). All authoritative and durable classes now migrate forward; corruption salvages. + C). Authoritative and durable classes now admit only the exact final shape, and + corruption salvages. - Facts cite retrieval anchors through a foreign key, so anchors could not live in evictable evidence (B). Cited anchors live in `memory.db`. - `FnOnce` jobs made the reconnect-and-rerun ladder and fence reruns impossible, @@ -549,15 +532,11 @@ blocking pool; that hop is intentional and the executor step removes it. byte on disk. - We accept losing cross-page snapshot consistency, because every read is a short job, in exchange for a total checkpoint fence. Pagination uses keyset cursors. -- We accept that forks sharing a root commit share a project until `project - split`, in exchange for clones converging with no configuration. - We accept partial derived search during a derived rebuild, in exchange for corruption and schema changes that heal without an operator. - We accept a 30 s ceiling per write job; bulk work pages. - We accept Grafeo as an immutable file format. This is not a second engine: it has no WAL and no mutation, like the code-index generation files. -- We accept a one-shot in-place conversion of durable rows for two releases, - because "durable memory survives" forbids a reset of that class. ## Alternatives considered @@ -588,10 +567,8 @@ changes a file that holds memory. it, and no read parks a native worker. Proof: a native task awaiting a typed read leaves the native reader-wait counter unchanged (a mutation that reads inline fails the test). -2. Memory durability. The stores that hold memory run at FULL; the memory tables - get additive migrations instead of `ResetRequired` with their first schema - change. Proof: those stores' writers report `synchronous=FULL` and the others - `NORMAL`. +2. Memory durability. The stores that hold memory run at FULL. Proof: those + stores' writers report `synchronous=FULL` and the others `NORMAL`. 3. Growth stops now. Worktree liveness by root instead of the shared `.git`, a collector for unregistered directories that never deletes durable data, response-handle sweeps every tick, `hook_analytics.jsonl` rotation, age-pruned @@ -618,14 +595,12 @@ changes a file that holds memory. `ProfileLost` within one tick with the descriptors closed. 8. Partition the sessions file and identity by repository. Split each project into journal, memory, evidence and derived; telemetry leaves the catalog; identity - adopts through aliases. The in-place conversion of durable rows ships here. - Per-stream cursors and dead letters replace the global queue: row poison - dead-letters per stream, and corrupt derived authority (ghost or divergent - provenance) rebuilds `derived.db` instead of being skipped. Delete + adopts through aliases. Per-stream cursors and dead letters replace the global + queue: row poison dead-letters per stream, and corrupt derived authority (ghost + or divergent provenance) rebuilds `derived.db` instead of being skipped. Delete `projection_queue`, `lcm-payloads/`, the string classifier, the store locator resolver and standalone registries. Proof: two full clones and a linked worktree - resolve to one project with three worktree directories; converted facts, - workflow definitions and approvals read back identically; a poisoned row + resolve to one project with three worktree directories; a poisoned row dead-letters while other streams project; corrupt provenance triggers a derived rebuild and the stream resumes. 9. Graph narrowing. The memory graph and session-relation graphs move to SQLite @@ -647,14 +622,12 @@ changes a file that holds memory. - Plan 02's "No asynchronous database API" and its interactive repository transactions: every storage API is async and every transaction is one job. -- Plan 38's "Store soft budgets are owner-configured and inert by default": budgets - have defaults and are enforced by `Storage::maintain`. +- Store soft budgets: enforcement through `Storage::maintain` is a target that + does not exist yet. Today an over-budget store only produces a Doctor finding. - Plan 39's move of graph-shaped data off SQLite: mutable graph data (memory and session relations) lives in SQLite; Grafeo remains an immutable sealed format. - Plan 40's monolithic per-worktree generation segments: segments are content-addressed per project. -- The roadmap's reset-only rule for released stores, for the durable and - authoritative classes: they migrate forward; derived and code still rebuild. - The 2026-07-15 fold of sessions into the project-wide store, which had no recorded reason: ADR-003's failure-domain split is restored by class. @@ -675,9 +648,7 @@ changes a file that holds memory. profile memory store (`user-memory.db`), commit at `synchronous=FULL`; every other store keeps `NORMAL`. `global.db` stays `NORMAL` although the catalog class is FULL, because it still takes per-hook analytics writes until telemetry splits out - in step 8. The additive-migration rule for memory ships with the first change to a - memory table (the conversion in step 8), not as machinery with no migration to - run. The class contract is unchanged. + in step 8. The class contract is unchanged. - Step 3 (2026-10-16, done). Four PRs stopped the growth this step names: - #4120. Orphan-store liveness no longer counts the shared git common directory, so a deleted worktree's store becomes collectable while its @@ -1021,11 +992,3 @@ changes a file that holds memory. get it; it is deferred until a silent-corruption incident is on record. - Are 64 GiB per profile and 8 GiB of evidence per project the right defaults? Step 10 calibrates them from the first weeks of persisted usage. - -## Next implementation step - -The rest of step 5: the native writer-wait counter becomes a refusal of any -synchronous writer wait on a native thread and both counters are deleted; then -the interactive exact-SQL transactions (56 direct sites and about 190 behind the -runtime-core async `Transaction`) become jobs. Step 7's first slice, writers and -readers as tasks on one fixed pool, is in progress in parallel. diff --git a/docs/plans/storage/evidence/why/inv-3-docs.md b/docs/plans/storage/evidence/why/inv-3-docs.md index 822c5db1444..ce9449411a1 100644 --- a/docs/plans/storage/evidence/why/inv-3-docs.md +++ b/docs/plans/storage/evidence/why/inv-3-docs.md @@ -107,13 +107,13 @@ cross-references, "page history" is `git log`/pickaxe on `docs/`. 9. **Stated goal: move graph-shaped data off SQLite and custom adjacency structures.** - "**Goal:** Replace custom adjacency structures and graph-shaped SQLite storage with one embedded Grafeo runtime boundary while retaining SQLite only for genuinely relational, transactional, and content-bearing records." - - Where: `docs/plans/tracedecay-v2/39-embedded-grafeo-graph-database.md` line 12, unchanged since the first version 955552d44b (ScriptedAlchemy, 2026-08-03, "docs(v2): plan embedded Grafeo graph cutover"). The code commit 0e3284dd2f ("feat(query): embed Grafeo graph projection") landed earlier the same day. All of these commit messages have empty bodies. + - Where: `docs/plans/tracedecay-v2/39-embedded-grafeo-graph-database.md` goal, unchanged since the first version 955552d44b (ScriptedAlchemy, 2026-08-03, "docs(v2): plan embedded Grafeo graph cutover"). The code commit 0e3284dd2f ("feat(query): embed Grafeo graph projection") landed earlier the same day. All of these commit messages have empty bodies. - Plan index (00, lines ~1003-1013): "graph-shaped data moves off custom adjacency structures and SQLite. Relational journals, receipts, raw content, configuration, and execution fencing remain in SQLite." - Relevance: the only stated "why". No doc compares Grafeo against SQLite recursive CTEs or adjacency tables on a measured workload before adoption. Task 12 ("Compare against the pre-Grafeo baseline") remained unchecked, and the 2026-08-14 audit lists "E18 POST-RC Grafeo aggregate/performance provenance ... pre-Grafeo performance comparison" as outstanding (deleted `audits/v2-implementation-audit-2026-08-14.md`). 10. **Plan 40 states what the pre-V2 SQLite graph did well and what V2 adds.** - "Master (pre-V2) proved the opposite trade-offs work. SQLite adjacency graph, per-file incremental sync, no stored body text, open-and-serve startup, but lacked what V2 genuinely adds: immutable digest-sealed generations, verify-then-publish with a watermark, exact branch/revision identity. Doc 39 already assigns the durable graph to Grafeo; the cutover never completed, leaving the sealed JSON as both canonical authority **and** de facto serving store." - - Where: `docs/plans/tracedecay-v2/40-partitioned-generation-storage-and-zero-copy-replay.md` §Problem (status "ACTIVE (decided 2026-08-22)"). + - Where: `docs/plans/tracedecay-v2/40-partitioned-generation-storage-and-zero-copy-replay.md` §Problem. - Measured trigger in the same doc: "One sealed code generation for this repository is a single `generation-.json` of **1.39 GB**"; "Warmup on this repository read tens of GB, pinned the 8 GiB cgroup boundary, and starved queries." - Relevance: the doc credits immutable generations and verify-then-publish to V2's design. It credits the earlier SQLite graph with good operational economics. diff --git a/docs/plans/tracedecay-v2/00-plan-set-index.md b/docs/plans/tracedecay-v2/00-plan-set-index.md index 72e8ce80b24..20bc503d1d3 100644 --- a/docs/plans/tracedecay-v2/00-plan-set-index.md +++ b/docs/plans/tracedecay-v2/00-plan-set-index.md @@ -106,13 +106,9 @@ invariants and do not by themselves keep a journey open. receipt-owned paths and a drift-converges acceptance test. Operational closure still requires the measured doctor journey and a clean Cursor install/upgrade/doctor pass on an operator machine.) -- Semantic search is disabled by an invalid configuration snapshot. Plan 20 - owns final-snapshot validity and explicit reset/recreation; Plan 31 owns - semantic activation. - Exact, lexical, and graph retrieval remain the required available fallback. - This snapshot does not mean semantic search is product-disabled. - (Superseded 2026-09-13: dense semantic retrieval is retired by rejected - decision 11; there is no activation to repair.) +- Dense semantic retrieval is retired by rejected decision 11. Exact, lexical, + and graph retrieval remain the required available authorities; there is no + semantic activation to repair. - A live profile was observed 237 minutes stale (the index later reported 285 minutes stale while this reconciliation was being checked). Plan 25 and the active incremental-indexing slice own cadence/freshness diagnosis; the @@ -275,8 +271,8 @@ records the rejected mechanism, the reason, and the retained replacement: installation downloads no model, loads no inference runtime, builds no vectors, and exposes no activation workflow. Released relational stores retain their data under the admission exception above; dense retrieval - is never reactivated. Plan 31 is retired as an authority; its lexical-fallback - and exact-tier obligations already belong to Plans 05, 15, and 25. The + is never reactivated. The lexical-fallback and exact-tier obligations + already belong to Plans 05, 15, and 25. The retained replacement is exactly three code-intelligence authorities and no fourth "semantic" one: lexical search (exact, fielded, phrase, proximity, bounded identifier/path splitting, and named query-time aliases that stay @@ -932,8 +928,7 @@ Unmeasured speculative optimizations and placeholder benchmarks do not ship. [15](15-search-quality-evaluation-and-retrieval-research.md), [23](23-session-lcm-temporal-retrieval-and-evaluation.md), and [25](25-code-intelligence-indexing-crate.md). - [Plan 31](31-native-fastembed-semantic-code-search.md) is retired by - rejected decision 11 and is archival. + Dense retrieval is retired by rejected decision 11. Indexing runtime is `tracedecay-code-index-runtime`; generation retention is `tracedecay-code-index-retention`. - Policy and delivery surfaces: [Plans 06](06-policy-crate.md), [08](08-tool-catalog-crate.md), @@ -999,8 +994,7 @@ Unmeasured speculative optimizations and placeholder benchmarks do not ship. [Plan 39](39-embedded-grafeo-graph-database.md)'s embedded graph storage cutover and [Plan 40](40-partitioned-generation-storage-and-zero-copy-replay.md)'s - partitioned generation storage, which supersedes the monolithic sealed - generation envelope as the long-term storage design. + partitioned generation storage. Embedded graph storage ([Plan 39](39-embedded-grafeo-graph-database.md)) threads through the active @@ -1031,10 +1025,9 @@ debris quarantine, and one-content-copy enforcement. Session retention keeps raw and projected evidence deduplicated while LCM summary nodes retain losslessly expandable source references. Plan 09 exposes the corresponding storage findings through read-only Doctor composition over Plan 26 telemetry; -Doctor never invokes cleanup, GC, retention, relink, or recovery. Store soft -budgets are owner-configured and inert by default. Direct validation uses an -isolated profile and proves size reduction without reading or mutating the -operator profile. +Doctor never invokes cleanup, GC, retention, relink, or recovery. Direct +validation uses an isolated profile and proves size reduction without reading +or mutating the operator profile. The remaining delivery proceeds by complete product journeys under this index. Branch, pull-request, merge, worktree, and SHA choreography is not roadmap diff --git a/docs/plans/tracedecay-v2/05-query-crate.md b/docs/plans/tracedecay-v2/05-query-crate.md index a563a0af211..3c4a0fcf868 100644 --- a/docs/plans/tracedecay-v2/05-query-crate.md +++ b/docs/plans/tracedecay-v2/05-query-crate.md @@ -22,15 +22,14 @@ same-host measurement showing a smaller frequently touched compile graph. - The memory delivery adds facts and provenance, the temporal-kernel delivery adds LCM/session retrieval, the code-intelligence delivery adds - lexical code search, code navigation, and current diagnostics, and the - semantic delivery adds semantic search. + lexical code search, code navigation, and current diagnostics. - The application delivery composes query use cases in application and policy. The surface delivery exposes them through CLI, MCP, HTTP, LSP, export, and live non-dashboard adapters. The dashboard delivery first ships dashboard binding and dashboard parity. - Plan 05 owns the shared execution semantics. Plan 23 remains the current - owner of temporal retrieval semantics; Plans 25 and 31 own active - lexical/code and semantic acceptance. Plans 09/10/11/12/14 and Plan 24 + owner of temporal retrieval semantics; Plan 25 owns active lexical/code + acceptance. Plans 09/10/11/12/14 and Plan 24 consume the accepted temporal-kernel behavior, not a superseded implementation shape. - The task/work delivery journey reuses these shared scope, budget, cancellation, cursor, watermark, merge, and coverage primitives to execute diff --git a/docs/plans/tracedecay-v2/15-search-quality-evaluation-and-retrieval-research.md b/docs/plans/tracedecay-v2/15-search-quality-evaluation-and-retrieval-research.md index 58ab928292b..dcfdb4df2c5 100644 --- a/docs/plans/tracedecay-v2/15-search-quality-evaluation-and-retrieval-research.md +++ b/docs/plans/tracedecay-v2/15-search-quality-evaluation-and-retrieval-research.md @@ -20,7 +20,7 @@ graph, Plan 13 diagnostic anchors, or their authorization rules. Plan 15 owns retrieval quality, composition, evaluation, and profile-selection rules. Plan 25 owns code retrieval and shared-code facts. The application, dashboard, task/work, and public surface plans consume tested behavior and -evaluation results. Plan 31 is archival. +evaluation results. ## Outcome diff --git a/docs/plans/tracedecay-v2/16-cross-project-repository-worktree-scope.md b/docs/plans/tracedecay-v2/16-cross-project-repository-worktree-scope.md index 12ffb77a405..de39953e69f 100644 --- a/docs/plans/tracedecay-v2/16-cross-project-repository-worktree-scope.md +++ b/docs/plans/tracedecay-v2/16-cross-project-repository-worktree-scope.md @@ -49,8 +49,8 @@ right data. ## Worktree index identity and reuse -Plan 16 composes the incremental index delivered by Plans 25/31; it does not -create a second watcher, parser, chunker, vector store, or scheduler. +Plan 16 composes the incremental index delivered by Plan 25; it does not +create a second watcher, parser, chunker, or scheduler. - Every linked worktree has an exact typed identity, fenced snapshot frontier, immutable code generation, freshness, and coverage. A shared Git common diff --git a/docs/plans/tracedecay-v2/25-code-intelligence-indexing-crate.md b/docs/plans/tracedecay-v2/25-code-intelligence-indexing-crate.md index 0fc40d61ea2..d4f988dfacf 100644 --- a/docs/plans/tracedecay-v2/25-code-intelligence-indexing-crate.md +++ b/docs/plans/tracedecay-v2/25-code-intelligence-indexing-crate.md @@ -75,7 +75,7 @@ regresses.) Plan 25 owns code-generation, chunking, graph, and generation-bound evidence semantics, including source-bound shared-code facts sealed into each generation. Plan 15 owns quality evaluation. Application, transport, and -dashboard plans consume the tested operations later. Plan 31 is archival. +dashboard plans consume the tested operations later. ## Outcome diff --git a/docs/plans/tracedecay-v2/26-observability-accounting-and-usage.md b/docs/plans/tracedecay-v2/26-observability-accounting-and-usage.md index 42424860fca..7acdf3d377b 100644 --- a/docs/plans/tracedecay-v2/26-observability-accounting-and-usage.md +++ b/docs/plans/tracedecay-v2/26-observability-accounting-and-usage.md @@ -145,7 +145,7 @@ vocabulary as current ownership. cancellation; analyzer startup, restart, and indexing/degraded state; cache reuse and overlay freshness; diagnostic add and clear; provider conflicts; host delivery path; partial coverage and drops; and bridge reconnect. -- Plans 25/31 emit bounded incremental-index observations for exact worktree +- Plan 25 emits bounded incremental-index observations for exact worktree identity classes without paths: event-to-reconcile and event-to-ready latency; debounced/raw event counts; overflow/rescan reason; candidate, hashed, parsed, and no-op file counts; Tree-sitter changed-range bytes; diff --git a/docs/plans/tracedecay-v2/31-native-fastembed-semantic-code-search.md b/docs/plans/tracedecay-v2/31-native-fastembed-semantic-code-search.md deleted file mode 100644 index f4e0c99c151..00000000000 --- a/docs/plans/tracedecay-v2/31-native-fastembed-semantic-code-search.md +++ /dev/null @@ -1,708 +0,0 @@ -# Native FastEmbed semantic code search (archival) - -**Status:** superseded on 2026-09-13 by -[rejected decision 11](00-plan-set-index.md#rejected-approaches). This file -records the retired dense-retrieval design. It is not current implementation, -product, storage, or test authority. - -TraceDecay no longer downloads code-search models, loads an inference runtime, -builds vector generations, exposes vector search, or runs semantic activation, -qualification, calibration, reranking, or rollback. Plans 05, 15, and 25 own -the retained exact, lexical, graph, and source-bound shared-code behavior. - -The content below is historical. It must not be used to restore a fourth -semantic code-intelligence authority. - -## Outcome - -TraceDecay augments exact lexical and graph search with local code embeddings. -Exact results remain authoritative without a model; similarity alone never proves impact, lineage, or equivalence. - -## Ownership - -- Plan 25 builds deterministic, storage-neutral `CodeSearchDocumentV1`, - `CodeSearchChunkV1`, `ChangedCodeChunkSetV1`, and - `CodeIndexCapabilityManifestV1` values from an exact code snapshot. -- Plan 04 defines deterministic changed-input projection, checkpoint, retry, - and publication semantics. Daemon/service orchestration decides when that - resumable work is scheduled. -- Plan 02 stores immutable vector generations, manifests, checkpoints, the - atomic active-generation pointer, and chunk projection receipts through - daemon-owned writer authority. -- Plans 05/15 own the common `CompactCandidate`, independent lane, deterministic - fusion, contribution, diversity, rerank, hydration, cursor, fallback, - explanation, and redundancy contracts shipped by federated retrieval. This plan adds one semantic - adapter; it does not create a parallel code-search kernel. -- Plan 20 owns versioned model/reranker profile selection, configuration - precedence, and atomic active/rollback profile pointers. This plan owns the - root-private versioned artifact manifest and SHA-256 integrity verifier, - FastEmbed runtime adapter, - bounded sessions, semantic generation service, and their typed ports. The - Plan 09 application layer later composes those accepted ports; it does not - retroactively own this plan's model implementation. -- Daemon/application orchestration schedules changed eligible chunks and - rebuilds. Plan 04 owns deterministic projection, checkpoint, retry, and - publication semantics, not scheduling policy. -- Plan 15 exclusively owns retrieval-research design, versioned corpora and - labels, metrics and strata, candidate profile comparison, thresholds, and - activation recommendations. This plan implements measured profiles and emits - results; it cannot tune or activate itself from aggregate or public benchmark - rank. -- Plans 10/11/20/21 expose the same application operations through API, - dashboard, configuration, CLI, and MCP. - -Only one root-private adapter depends on `fastembed`. Crates for indexing, store, -query, API, and UI depend on ports and stable domain values, never -FastEmbed runtime types. - -## Behavioral contracts and historical layout - -The semantic boundary must preserve the ownership and fields below. The listed -paths and Rust names are a historical design sketch, not artifact-name parity -requirements. Current owner-approved names and locations are valid when direct -contract regressions prove the same lane isolation, generation compatibility, -authorization, coverage, fallback, and receipt behavior. - -The original semantic-delivery design proposed these files: - -- `crates/tracedecay-domain/src/code_intelligence/search.rs`: - `EmbeddingProjectionKeyV1`, `SemanticSearchIndexKeyV1`, - `SemanticCapabilityManifestV1` and - code-specific semantic evidence values. Generic ranked-list, compact- - candidate, contribution, fusion, rerank, hydration, and cursor values remain - in federated retrieval's `crates/tracedecay-domain/src/retrieval.rs`. - Semantic projection uses Plan 25's `CodeChunkProjectionReceiptV1` and - `ProjectionBatchReceiptV1`; no semantic-specific receipt authority exists. -- `src/semantic_code/artifacts.rs`: versioned manifest parsing, SHA-256 member - integrity verification, import staging, atomic install, inventory, retention, - and quarantine. -- `src/semantic_code/manifest.rs`: model/runtime/projection and capability - manifest validation. -- `src/semantic_code/projector.rs`: changed-chunk batching, receipt production, - resumable checkpoints, and publication handoff. -- `src/semantic_code/fastembed_adapter.rs`: the only `fastembed` import and the - only model inference implementation. -- `src/semantic_code/session_pool.rs`: bounded sessions keyed by the complete - projection/privacy identity. -- `src/query/retrieval/semantic.rs`: the code-semantic implementation of the - federated `Retriever` port. It emits generic `CompactCandidate` values carrying a - typed code anchor and semantic evidence; it consumes neither lexical nor - graph candidates. -- `src/semantic_code/service.rs`: temporary root-private semantic orchestration for - artifact/runtime admission, projection generation, shadow execution, and - activation/rollback. It exposes typed ports consumed by the application - layer and contains no transport binding. -- `src/config/retrieval.rs`: Plan-20-owned versioned profile definitions and atomic - active/rollback profile pointers. -- `src/store/vector_generations.rs`: Plan-02-owned implementation of the - semantic projection read/write ports. Query and semantic modules do not - depend on its physical schema. - -The semantic adapter extends the federated retrieval contract rather than replacing -it: - -```rust -pub struct SemanticRetrievalRequestV1 { - pub query_digest: QueryDigest, - pub query_view: EphemeralSanitizedQueryViewV1, - pub authorized_scope: AuthorizedScopeReceiptV1, - pub authorization_epoch: AuthorizationEpoch, - pub scope_digest: ScopeDigest, - pub code_generation: CodeGenerationId, - pub projection_key: EmbeddingProjectionKeyV1, - pub capability_manifest_digest: ManifestDigest, - pub budget: RetrievalBudget, - pub continuation: Option, -} - -pub struct SemanticCodeRetriever { - // Root-private FastEmbed session and semantic projection read ports. -} - -// SemanticCodeRetriever implements -// Retriever. - -pub struct CodeSemanticEvidenceV1 { - pub projection_key: EmbeddingProjectionKeyV1, - pub vector_generation: VectorGenerationId, - pub chunk_id: CodeSearchChunkId, - pub distance: CanonicalSemanticDistanceV1, - pub search_kind: SemanticSearchKindV1, -} -``` - -The adapter implements the one federated `Retriever` port and receives one -frozen authorized scope, code generation, semantic -projection, and bounded ephemeral sanitized query view. It returns the generic -Plan 15 batch with one `CompactCandidate` and one occurrence-keyed -`CodeSemanticEvidenceV1` per result. It cannot mint exact -admission, read another lane's candidates, fuse, diversify, rerank, hydrate, or -retain raw query text/query vectors. Store conformance uses separate lexical, -graph, vector, receipt, and hydration representations; no table, vector index, -or materialized join becomes authority. -Deadline and cancellation limits are fields of the shared `RetrievalBudget`; -this plan does not introduce semantic-only budget or continuation types. - -## Deterministic documents and generations - -Each chunk records repository/project/worktree/ref/snapshot identity, immutable -code generation, file and symbol identity, source span, language/extractor and -chunker versions, sensitivity decision, content digest, stable ordinal, and -bounded sanitized text. Symbol boundaries are preferred; oversized symbols use -versioned structural splits. Generated, vendor, binary, ignored, fixture, and -unsupported content has an explicit classification. - -A vector-generation identity includes the ordered eligible-document manifest, -model/tokenizer/runtime manifest, dimension, metric, normalization, chunker, -privacy domain/key epoch, and source watermark. The manifest also pins query -and document instructions/prefixes, pooling, truncation side and length, -precision/quantization, runtime/backend/thread/device identity, and search or -ANN parameters when present. Builds checkpoint in bounded batches. Partial or -mixed generations are never queryable. Publication verifies membership, -dimensions, finite values, digests, and watermark before one atomic pointer -swap; deletion creates a tombstone and unchanged inputs do no embedding work. - -`EmbeddingProjectionKeyV1` pins every vector-affecting input: model artifact, -tokenizer/config and instruction digests, pooling, truncation side/length, -runtime/backend/build revision, deterministic device class, dimension, metric, -normalization, precision/quantization, chunk schema/chunker revision, privacy -domain, and key epoch. Its canonical digest is Plan 25's -`ProjectionKeyV1.profile_digest`. Search/ANN structure parameters use a -separate `SemanticSearchIndexKeyV1`; changing them rebuilds only derived search -structures and query caches. Thread/batch settings remain execution-manifest -evidence and must produce the same vector digest or fail publication. Vectors -and projection rows are replayable derived data; canonical chunks and exact -code-generation evidence remain authority. - -For each `ChangedCodeChunkSetV1`, the projector returns one ordered -`CodeChunkProjectionReceiptV1` per added/changed/deleted chunk and an explicit -aggregate reused count in `ProjectionBatchReceiptV1`. A receipt records source -generation and manifest, projection key, chunk ID, prior/current digest, -`embed | tombstone`, completed/skipped/failed outcome, and output digest. -Deleted receipts bind `prior_generation`/`prior_chunk_digest`, the batch -`request_digest`, and have no current digest. -Reused chunks produce no per-chunk receipt. A no-op request with an unchanged -target projection key invokes no inference; a key-only replay expands reused -eligible chunks into explicit embed operations before dispatch. A one- -symbol edit embeds only its changed chunks and affected file-level chunks. A -model, tokenizer, instructions, runtime-compatibility, dimension, metric, -normalization, precision, privacy-domain, or key-epoch change creates a new -projection generation by replaying retained canonical chunks without reparsing -unchanged source. Old retained projections remain immutable and addressable by -their exact key; partial receipt sets never activate. - -### Worktree-aware projection scheduling - -- Consume Plan 25's reconciled `ChangedCodeChunkSetV1`; do not watch files, - rescan repositories, invoke Git status, or infer changes inside the semantic - runtime. -- Batch only added/changed eligible chunks through FastEmbed - `TextEmbedding::embed`. Construct the model through - `try_new_from_user_defined` from explicitly installed local ONNX/tokenizer/ - config bytes. Query and projection paths never invoke the hub, discover an - ambient model cache, or download artifacts. -- A no-op changed set performs zero model calls. Deletions produce tombstones - without inference. A projection-key change replays retained eligible chunks - without parser/extractor work; a search-index-key-only change rebuilds only - the derived exact-flat/search structure. -- Reuse vector bytes across worktrees only when canonical chunk content, - projection key, privacy domain, and key epoch match. Every projection row, - receipt, and immutable vector generation remains bound to its exact source - worktree snapshot and code generation. The committed retrieval profile names - that vector generation, while each admitted query separately pins the exact - source code generation; neither identity is inferred from a process cache. -- Coalesce superseded projection batches by exact worktree and source - generation. Bound queued bytes, batch size, session count, model memory, and - publication concurrency. Interactive exact/lexical/graph queries have - priority; semantic query admission is fail-fast and never enters the - projection waiter queue. -- Keep a prior semantic generation queryable only when its complete projection - key and source compatibility still match the frozen code request. Otherwise - omit semantics until the new complete generation atomically publishes. - -## Model and offline lifecycle - -Configuration selects a versioned embedding profile and, independently, an -optional reranker profile. With automatic acquisition enabled, the daemon -queues the selected catalog model's immutable Hugging Face repository revision -in the background. Manifests pin actual model/tokenizer/config bytes, licenses, -runtime/build identity, dimensions, normalization, metric, device, threads, -batching, and resource ceilings. Implementation selects maintained -crate/runtime versions during the semantic delivery; activated model and reranker profiles still -require a passing Plan 15 evaluation. This plan contains no stale crate or -model-version pin. - -Every acquisition/import path verifies artifacts before installation or -activation. Queries never download a model or open an ambient cache. Offline -startup remains healthy and baseline-complete. Compatible warmed sessions -are pooled under bounded memory, concurrency, idle, and cancellation policy. -Load failure, OOM, corruption, missing bytes, or incompatible pins disables the -affected semantic stage without silently selecting another model. - -Artifact handling is explicit: - -- A versioned canonical manifest lists profile kind, model/tokenizer/config file digests, - byte lengths, licenses, upstream source metadata, runtime compatibility, - projection inputs, and the complete resource ceiling. The manifest and each - declared member are identified by SHA-256 solely to detect drift, corruption, - or incomplete installation. These versioned manifests and integrity checks - are the complete artifact-verification contract. -- Daemon-owned background acquisition resolves only the cataloged repository - and immutable revision into an explicit cache under the lifecycle root. It - stages each member, independently checks its declared length and SHA-256, - and atomically publishes the complete verified install. Query/runtime paths - perform no download, import, extraction, source discovery, or network - inference. -- Explicit local-path and configured-HTTPS import APIs remain available as - manual operations, but no production surface currently wires them. They are - not the delivered acquisition journey. -- Interrupted imports are resumable only when the source supplies immutable - length, digest, and range identity; otherwise staging is discarded. Archive - traversal, symlink/hardlink entries, absolute paths, duplicate members, - undeclared members, size expansion beyond the manifest, and digest mismatch - quarantine the import without exposing it to runtime discovery. -- Installed artifacts live in one Plan-02-owned user store keyed by versioned - manifest digest, never an ambient Hugging Face/ORT/FastEmbed cache. The - daemon's lifecycle-root Hugging Face cache is an acquisition source only, - not runtime or installation authority. Inventory records distinguish - `staged | verified | installed | quarantined | retained_for_rollback`. - Garbage collection may remove only unreferenced non-active/non-rollback - artifacts after a daemon lease and append-only receipt. -- Activation is an authenticated Plan 20 compare-and-swap from the expected - active/rollback profile digests. It verifies artifact inventory, accepted - Plan 15 report, runtime/platform compatibility, complete projection - generation, resource ceiling, and executable rollback before publishing. - Rollback uses the same checks and must pass a cold-start/offline drill. - -The mandatory Plan 25 `CodeIndexCapabilityManifestV1` admits lexical/graph -retrieval. Optional `SemanticCapabilityManifestV1` pins the authorized scope -digest, code and vector generations, projection and search-index keys, -supported chunk grains/languages, coverage, partial states, privacy domain/key -epoch, and manifest digest. Fusion, candidate budgets, diversity, reranking, -and hydration are validated separately from the active Plan 15/20 profile. The semantic service validates the base -manifest before any channel and validates the semantic augmentation only before -semantic/rerank work. Missing semantic capability yields lexical/graph mode; an -explicit strict-semantic request yields the typed unavailable result. - -## Query and redundancy - -Search resolves exact scope and frozen generation first, reproduces the -versioned exact+lexical+graph baseline, then adds a compatible semantic -candidate list. Fusion is stable and explainable; Plan 15's exact-admission -authority keeps exact identifiers, paths, quoted phrases, errors, tool names, -and configuration keys in a non-demotable tier. The first production semantic -baseline is deterministic exact flat-vector search unless measured -current/10x evidence shows it violates a reviewed resource budget. Optional -reranking is bounded to a configured top-N candidate set, is admitted only -after candidate-controlled gain with no protected-stratum regression, and -preserves the pre-rerank list byte-for-byte when unavailable. Raw similarity, -logits, margins, or fused scores are not confidence; calibrated abstention -requires a versioned cohort/generation-bound profile and reports invalid or -shifted calibration explicitly. Strict semantic requests return a typed -unavailable result. - -Semantic projection and indexing run asynchronously. Existing exact, lexical, -and graph operations remain callable and return without joining, polling, or -waiting for semantic work. The semantic lane is omitted until one complete -compatible immutable vector generation becomes current through a single atomic -publication. Staged, partial, indexing, stale, failed, cancelled, or -incompatible generations contribute no candidates, score, cap pressure, -cursor bytes, or rank effects. A request for strict semantic alone may return -typed unavailable; it may not delay or replace the normal federated path. - -`code.redundancy` reuses the same active generation. It canonicalizes pairs, -removes self/overlapping chunks, and reports `exact_clone`, -`structural_near_duplicate`, `semantic_analogue`, or `insufficient_evidence`. -Semantic-only matches remain review candidates, never automatic rewrites or CI -violations. Disabled semantics preserves the structural baseline and ordering. - -The Plan-05 query pipeline executes these phases without combining them: - -1. Resolve and authorize project/repository/worktree/ref scope; freeze code, - exact, lexical, graph, and compatible semantic generations; validate the - base and semantic capability manifests, authorization epochs, and budgets. -2. Reuse federated retrieval's separately inspectable `RetrieverOutcome` values for - `ExactLiteral`, `Lexical`, and `Graph`; produce one independent - `Semantic` outcome. No semantic code calls, wraps, filters, or mutates - another lane. -3. Validate/canonicalize the semantic outcome under the generic Plan 15 lane - contract: finite canonical scores, candidate identity dedupe, generation and - authorization equality, stable lane-local ordering, complete coverage, and - a bounded continuation. -4. Pass all compact candidates to the one federated fusion implementation. The - existing exact tier remains lexicographically first and cannot be demoted by - graph/semantic scores, diversity, or reranking. This plan evaluates semantic - weighting/fusion candidates, including exact rational RRF if proposed, on - the same recorded candidate inputs; no algorithm, constant, or weight - activates without a passing Plan 15 evaluation. -5. Use the complete comparator and diversity policy from the tested generic - profile. The cursor binds emitted identities, ordered overflow, every lane - continuation, profile/generation/projection digests, and authorization - epoch. Input insertion order, hash order, and completion order cannot affect - IDs, order, contributions, explanations, coverage, or cursor bytes. -6. Retain every generic `CandidateContribution`, including lane-local raw score - domain/rank, calibrated fixed-point feature when valid, weighted - contribution, profile/projection identity, evidence class, and exclusion - reason. Graph path coverage/authority and semantic projection/distance - provenance remain typed lane evidence. A fused score is not probability or - confidence. -7. Optionally rerank only the profile-bounded non-protected candidate set while - preserving the pre-rerank list and contributions. Absence/failure returns - those bytes unchanged. -8. After fusion, diversity, reranking, and page selection, reauthorize and - hydrate only selected hits through the federated hydration port. Hydration may add bounded - symbol text, file context, declarations, and graph neighbors from the same - generation; it cannot add candidates or change rank. - -Plan 15's generic `FusionProfile` pins the selected algorithm, lane features/ -weights, candidate budgets, diversity caps, protected exact classes, rerank -budget, hydration budget, and total comparator. Plan 15 recommends that profile -from direct evaluation; Plan 05 implements it, and Plan 31 supplies -only the semantic lane, artifact/runtime/projection implementation, and -measurements. - -## Authorization and local/private boundary - -- Owning source stores authorize exact scope, operation, privacy domain, key - epoch, and code generation. Plan 20's accepted profile and the semantic service - separately authorize semantic projection generation and runtime profile. The - semantic query composition validates both receipts before invoking the semantic - lane; source denial invokes zero channels, while - semantic denial invokes no semantic/rerank port and preserves authorized - lexical/graph execution. -- Candidate references carry authorization receipt ID and epoch without private - payload. Immediately before hydration, the shared query pipeline rechecks current - authorization, scope, privacy domain/key epoch, and frozen generation. - The owning store performs atomic receipt validation plus bounded payload/ - neighbor read; every neighbor must be inside the authorized scope. Revocation - before or during a multi-read discards the complete hydrated hit and returns - a typed denial with no source text or neighbor payload. -- All caches are local and privacy-domain/key-epoch separated. Model artifacts - key by versioned manifest digest; sessions by embedding projection key; vectors - and checkpoints by projection key plus source generation/chunk digest. - Result caches additionally key by authorization receipt/epoch, authorized - scope and query digests, fusion/rerank/request revisions, and pagination. - Hydration caches additionally key by candidate identity, requested fields, - and neighbor budget. Domain/epoch changes produce zero hits in every cache; - authorization changes produce zero result/hydration hits. No ambient model - cache, network inference, browser runtime, external process, or cross-domain - cache is permitted. -- Raw queries, source text, vectors, private paths/symbols, and hydrated - explanations never enter telemetry or checked-in benchmark artifacts. - A raw query is sanitized into a bounded ephemeral query view before model - inference. `EphemeralSanitizedQueryViewV1` contains bounded sanitized bytes - plus sanitizer and normalization revisions; it is non-serializable, - non-cacheable, and valid only for the authorized request. `QueryDigest` is a - privacy-domain/key-epoch keyed MAC over that view, never an unkeyed content - hash. `QueryDigest` is owned by - `crates/tracedecay-domain/src/retrieval.rs` and may appear only in in-process - request state, authenticated cursor identity, and privacy-separated local - cache keys. `EphemeralSanitizedQueryViewV1` is owned by - `code_intelligence/search.rs`. Neither value may enter receipts, telemetry, - durable stores, or checked-in artifacts. Query text and vectors remain in - memory only for that request. - Operational receipts contain opaque identities, revisions, outcomes, counts, - and digests only. Only sanitized fixtures and aggregate Plan 15 reports may - enter Git. -- Semantic errors, timeout, cancellation, OOM, corruption, or missing artifacts cannot - broaden scope. When the selected profile permits fallback, the lexical/graph - lane outcomes and Plan 15's named exact/lexical/graph fallback subpayload are byte-identical. - The enclosing response may add only a typed semantic/rerank outcome outside - that subpayload, its digest, and cursor identity; strict mode fails closed. - -## Plan 15 evaluation handoff and fresh-store reset - -Semantic retrieval ships a versioned sanitized corpus covering exact names, natural-language -intent, mixed queries, renamed symbols, same-name cross-scope cases, no-answer -queries, generated/vendor noise, large symbols, unsupported languages, and -incremental edits. It measures exact-hit retention, precision/recall/MRR/nDCG, -wrong-scope and no-answer error, worst strata, build/update time, p50/p95/p99, -CPU/RSS, model/vector/cache bytes, cancellation, and offline behavior. - -Plan 15 owns this corpus's partition/label policy, metrics, thresholds, and -activation recommendation. This plan owns reproducible Linux execution and truthful -`pass`, `fail`, or `pending` summaries. Activation requires a passing Plan 15 -evaluation showing no scope/privacy or protected -exact/no-answer/wrong-scope regression, demonstrated semantic gain, and -declared current/10x Linux resource observations. Sensitive or -ineligible bytes never enter documents, artifacts, metrics, explanations, or -model-assisted routes. - -Late interaction, quantization, and specialized ANN remain measured candidate -profiles, not semantic-delivery defaults. ANN is admitted only when it beats exact search's -reviewed resource budget while meeting exact-oracle average, tail, minimum, -and zero-recall-query gates under immutable-generation compatibility. No HNSW, -DiskANN, ScaNN, vector database, precision, or quantization choice is -mandatory. Public benchmark rank cannot select a production profile. - -Non-final vectors are never trusted, republished, or converted. They return -`ResetRequired`; an operator explicitly resets or recreates the generation from -current canonical documents without using the old vector as input. - -## Current delivery audit - -The audit below separates callable delivery from pending direct execution. A -present source file or type name is not completion by itself; the cited direct -regressions invoke the relevant boundary. Static fixture validation is useful, -but the current Linux evaluation remains pending. - -**Production lifecycle correction (updated 2026-07-26).** Commit `dd4adbe2a` -ships daemon-owned immutable `hf-hub` background acquisition. -`shared_lifecycle_owner()` opens a lifecycle-root-scoped -`HfHubModelMemberSourceV1`; startup queues acquisition when configured, and the -source resolves only the cataloged repository/revision before the lifecycle -checks every member's declared length and SHA-256 and atomically installs the -package. Status and Doctor expose downloading, verifying, installed, and -failed states. A clean temporary profile downloaded and digest-verified the -full default Jina model end to end, and packaged offline acceptance installs -the same verified model from the daemon-owned cache. The explicit local and -configured-HTTPS import APIs remain manual and production-unwired; they are not -being certified by this correction. Installed model bytes remain semantically -omitted until compatible vector indexing publishes readiness, preserving the -asynchronous non-blocking contract. - -**Why the old green gate was misleading.** Before `dd4adbe2a`, -`HfHubModelMemberSourceV1::fetch_member` unconditionally returned -`semantic model lifecycle operation rejected`, so no shipped binary could -acquire a model. The distribution gate's packaged-semantic half tested only -typed fallback/strict-unavailable behavior, the exact terminal absence produced -by that rejection, and therefore encoded the bug as its passing specification. -The corrected gate now runs packaged background acquisition and verifies the -installed Jina members; restoring the unconditional rejection makes that gate -fail, and restoring the source makes it pass. - -- **Library-first FastEmbed:** delivered. The root manifest keeps - `fastembed` optional with upstream defaults disabled, the - `semantic-fastembed` feature selects the native runtime, and - `FastEmbedEmbeddingRuntime::open_session` uses FastEmbed's user-defined local - byte constructor. Native model execution and resource evidence remain - pending. -- **Default production posture:** delivered in the root feature manifest. - FastEmbed/ORT is available on Linux and macOS. Windows production builds keep - the daemon, hosts, exact/lexical/graph retrieval, and the ORT-free Model2Vec - backend; selecting a FastEmbed model reports the typed unavailable runtime - state and never silently substitutes another embedding backend. -- **Daemon-owned immutable acquisition:** delivered. Startup queues the - selected catalog revision without blocking startup or query paths; the - lifecycle-root source cache is explicit, and verification plus atomic install - remain independent of `hf-hub`. -- **Local verified model bytes only:** delivered at the runtime boundary. - Model, tokenizer, and config bytes come from the installed manifest members; - runtime construction has no hub, ambient-cache, download, external-process, - or network-inference path. Background acquisition now creates the verified - installed manifest and members; explicit local/HTTPS imports remain - production-unwired. Direct tests validate constructor and integrity reads, - while cross-platform runtime coverage remains normal CI work. -- **Exact-flat semantic baseline:** delivered by - `SemanticCodeRetriever`/`SemanticVectorReadPort::scan_exact_flat`, with direct - deterministic ordering, provenance, generation, and coverage regression in - `exact_flat_scan_is_deterministic_and_emits_generic_semantic_evidence`. -- **Immutable generations and verified publication:** active integration work - uses the staged vector-generation store and the Plan 39 verified Grafeo - publication boundary. Staged work is not queryable, cancellation preserves - every prior verified generation, and failed publication cannot expose a - partial generation. Publication only adds immutable catalog state and may - advance the relational verified graph head; it never activates semantic - routing. Plan 20's linked configuration transition and activation receipt - are the sole active/rollback authority, while process-local runtime pointers - are observed readiness caches only. -- **Asynchronous, non-blocking indexing:** delivered at the projection, - generation-selection, runtime-routing, and bounded-session boundaries. - `only_a_current_receipt_routes_to_semantic_search` routes indexing, degraded, - unavailable, and rollback states to the frozen exact/lexical/graph fallback, while - `saturated_runtime_omits_semantics_without_entering_the_waiter_queue` proves - query work does not wait for semantic capacity. -- **Calibrated abstention:** delivered. Missing or shifted calibration invokes - no semantic authority and preserves fallback; distance and margin rejection - are versioned and generation-bound. Strict semantic returns typed unavailable. - Linux threshold evaluation remains pending. -- **Byte-stable exact/lexical/graph fallback:** delivered at the semantic service boundary. - Direct regressions retain the caller-owned validated fallback object through - augmentation and every tested abstention path. The versioned fallback bytes - and digest remain pending direct lexical verification. -- **Evaluation before activation:** still pending. Current runtime - routing requires an observed current activation receipt and rejects an - indexing receipt, but the Plan 15 Linux evaluation, current/10x measurements, - normal cross-platform CI, and rollback drill are still incomplete. - `result-pending.json` therefore keeps activation false and truthfully reports - `pending`. - (Update 2026-08-07: the named artifact is stale, no `result-pending.json` - exists in-tree (`git ls-files` returns zero hits). The behavior it stood for - is intact and is now asserted by the evaluation harness itself: - `tests/search_eval_cli_test.rs:53` - (`compare_reports_unmeasured_semantic_and_rerank_stages_as_pending`) requires - the `compare` CLI to exit non-zero and report unmeasured semantic/rerank - stages as `pending`. The gate is unchanged; only the file reference is - obsolete.) - -## Planned behavioral delivery and direct verification - -The native semantic delivery remains unfinished. The checkpoints below are required product behavior. -Paths, type spellings, fixture filenames, benchmark entrypoints, and test-suite -registration are non-normative historical suggestions. Completion follows the -callable semantic operation, direct regressions, the Plan 15 Linux evaluation, -and normal CI. - -1. **Contracts and capability admission:** add semantic-only domain values, - ports, manifest validators, ephemeral query-view rules, and split in-memory - adapters. Reuse federated retrieval's generic retrieval/fusion/hydration behavior. Direct - regressions cover contract validation, capability admission, and storage - independence. -2. **Artifact and runtime foundation:** implement one root-private artifact - verifier, manifest validator, FastEmbed adapter, and bounded session pool. - Direct regressions cover versioned manifest and SHA-256 member verification, - local and explicit HTTPS import, - traversal/expansion rejection, interrupted staging, atomic install, - quarantine/GC, cold and warm sessions, OOM, cancellation, - offline startup, no ambient cache, and Linux/Windows native-runtime - compatibility. -3. **Incremental vector projection:** implement the projector and vector-generation - store against Plan 04 projection/checkpoint semantics, Plan 02 - receipt/publication authority, and this plan's runtime ports. Daemon/service - orchestration owns asynchronous worktree-fair bounded scheduling; do not - assign it to Plan 04. Direct regressions cover changed-chunk batching, - deletion without inference, no-op zero inference, cross-worktree physical - reuse without identity reuse, superseded-batch cancellation, model-key - replay, search while indexing without waiting, omission of incomplete/ - incompatible generations, atomic publication, cancellation/failure - isolation, and offline lifecycle. -4. **Exact-flat semantic retrieval and shadow composition:** implement one - independent semantic outcome and compose it with frozen federated lane outputs - through the shared fusion/diversity/cursor behavior. Direct regressions - cover lane isolation, contribution provenance, protected exact results, - diversity/pagination, and byte-identical fallback. -5. **Late hydration, privacy, and rollback:** reuse federated hydration with semantic - profile admission, generation checks, authorization recheck, - domain-keyed caches, payload-safe receipts, active/rollback pointer CAS, and - cold offline rollback. Direct regressions cover hydration, authorization, - privacy-domain isolation, activation, and rollback. -6. **Developer evaluation:** use reproducible checked-in Linux workloads, - explicit reset/recreation checks, exact-flat oracle comparisons, focused - channel/fusion/calibration/rerank ablations, privacy/non-interference checks, - and current/10x resource observations. Optional ANN and reranker branches - begin only after the exact-flat shadow checkpoint and cannot enter the - critical path without a passing comparison. -7. **Activation and verification:** only a passing Plan 15 result may make - semantics eligible for the existing configuration activation. Run staged - shadow/cohort behavior, rollback, fresh-store reset, privacy, architecture, direct - tests, Linux evaluation, and normal all-feature cross-platform CI. Status - and Doctor behavior are exercised through the production lifecycle states; - activation still requires the remaining evaluation and indexing evidence. - This plan does not create a temporary public semantic endpoint or reserve later - surface contracts. - -A reproducible projection workload measures clean, warm one-symbol, deletion, -no-op, model-key replay, cancellation, and incompatible rebuild. A query -workload reports lexical, graph, semantic, fusion, rerank, and hydration time -separately at current and 10x corpus sizes, -including p50/p95/p99, CPU, peak RSS, model/vector/cache bytes, candidates per -channel, chunks embedded/reused/deleted, hydration fetch count, and fallback. -Channel ablations use equal candidate budgets; exact flat-vector search is the -semantic oracle. End-to-end performance work consumes these production -measurements, while Plan 15 owns quality/resource trade-off and activation -recommendations. - -The query workload also overlaps queries with a blocked projection worker and -with staged, partial, failed, cancelled, stale, and incompatible generations. -It records exact/lexical/graph completion independently, requires zero wait on -the semantic worker, compares the visible federated fallback bytes and rank before -and during indexing, and observes semantic candidates only after the complete -compatible generation is verified and Plan 20 has committed the matching -activation receipt. Those are ordered publication and configuration -transactions, not a graph-local pointer swap. - -Each workload manifest pins corpus/query digests, exact file/chunk/query counts, -language/source strata, seed, model/projection/fusion revisions, hardware and -runtime summary, cache state, and concurrency. The 10x workload contains -exactly ten times the eligible chunks of the current workload without copying -quality labels across partitions. Reports retain the raw Linux samples needed -for every statistic and label unsupported p99 or uncertainty claims `pending`. - -### Hard activation barrier - -No semantic comparison starts until callable exact/lexical/graph behavior -passes direct regressions and its versioned profile, exact-tier contract, -fallback-subpayload bytes, and quality fixtures are identified by real -content/revision digests. Historical artifact-name parity is not part of this -barrier. No activation occurs unless Plan 15 reports `pass`, -authorization/scope leakage is zero, protected exact results are -unchanged, the exact/lexical/graph fallback subpayload is byte-identical, -generation compatibility holds, search-during-indexing leaves fallback bytes and rank -unchanged, incomplete or stale generations contribute nothing, all -resource ceilings pass, and cold offline rollback succeeds. - -A `fail` or `pending` result leaves semantics disabled and no semantic profile -eligible. - -## Acceptance - -Semantic retrieval is complete when projection, atomic publication, lexical-fallback-preserving -search, bounded generic fusion/reranking/redundancy, artifact/offline lifecycle, -configuration, production status/Doctor behavior, corpus/resource/privacy -tests, fault recovery, rollback, explicit reset/recreation, the Linux developer -evaluation, and normal CI pass. Application, public transport, and dashboard -adapters remain required parts of the same product. No separate -semantic endpoint, vector database, browser inference runtime, or model-specific -transport is introduced. Queries never silently substitute a model/revision, -download at query time, cascade to an unmeasured representation, or treat -semantic similarity as identity, impact, lineage, or equivalence. - -- A no-op chunk manifest performs zero embedding calls. A one-symbol edit - embeds exactly its changed symbol chunks and affected file-level chunks and - tombstones explicit deletions. Changing only vector-affecting model profile - fields with unchanged canonical chunk inputs replays all eligible chunks with - zero parser/extractor calls; chunker/sanitizer/sensitivity changes follow - Plan 25's canonical rebuild path. -- Receipt fixtures reject missing, duplicate, extra, wrong-generation, wrong- - digest, and wrong-key entries; crash/cancellation leaves the previous active - retrieval configuration unchanged and no partial projection queryable. - Queries issued while - projection is blocked complete through exact/lexical/graph without waiting - and match the versioned federated fallback bytes and rank. Only a complete, - recovered-state-verified generation named by the current linked activation - receipt may add the semantic lane. -- The semantic adapter emits a separately inspectable generic - `RetrieverOutcome>`; the exact, - lexical, and graph outcomes are unchanged. Disabling or failing semantic - retrieval preserves Plan 15's named federated fallback subpayload bytes. Fused - explanations reproduce every declared generic profile - contribution and the complete comparator for every result. -- Exact-hit retention and first-relevant-protected-hit Recall@10 are 100% over - Plan 15's versioned protected-query set, with numerator, denominator, and - support reported separately for symbols, qualified names, compiler/runtime - errors, CLI flags, paths, quoted phrases, tool names, and configuration keys. - For multi-target queries, every relevant protected hit admitted by the - requested page size precedes every fuzzy, graph-only, or semantic-only hit. -- One hundred runs with shuffled candidate insertion and channel completion - order produce byte-identical channel lists, fused IDs/order/contributions, - diversity spillback, explanations, and cursors. -- Fixture profiles with `max_non_exact_per_source = 3` and - `max_non_exact_per_file = 2` enforce those caps before pagination, preserve - protected exact hits, and deterministically refill from overflow candidates. - A three-page fixture proves no duplicate, omission, or cap drift after cursor - resume. -- A page requesting five hydrated hits reads payloads and neighbors for at most - those five hits and the profile's declared neighbor budget. Hydration cannot - change rank; every neighbor matches the frozen generation and preserves graph - ordered path identity, coverage, and weakest edge authority. -- Denied authorization invokes zero retrieval ports and zero payload reads. - Revocation before hydration returns no payload. Privacy-domain/key-epoch - changes yield zero session/vector/result/hydration cache hits, and cross- - domain candidates, vectors, receipts, metrics, or explanations are zero. -- Split-adapter conformance produces identical results when lexical postings, - graph evidence, vectors, receipts, and hydration payloads use separate stores, - proving embeddings and no single physical table are authority. -- Artifact fixtures reject malformed versioned manifests, undeclared or - duplicate members, path traversal, links, size expansion, - digest/length mismatch, incompatible runtime/platform pins, interrupted - publication, and deletion of active/rollback artifacts. Query execution - performs zero network/import/cache-discovery operations. -- Distribution acceptance starts from an isolated profile, queues the - background worker, installs every digest-verified default Jina member, and - proves semantics remain omitted until indexing readiness. Its offline mode - consumes only the lifecycle-root cache and must fail if production - acquisition regresses to unconditional rejection. -- The checked-in benchmark corpus contains only sanitized fixtures and expected - opaque anchors; raw private queries/source are absent. Activation requires a - passing Plan 15 evaluation and cannot be inferred from public rank or - aggregate gain. diff --git a/docs/plans/tracedecay-v2/38-storage-retention-size-and-efficiency.md b/docs/plans/tracedecay-v2/38-storage-retention-size-and-efficiency.md index c7f9a8191f3..f53b66bc913 100644 --- a/docs/plans/tracedecay-v2/38-storage-retention-size-and-efficiency.md +++ b/docs/plans/tracedecay-v2/38-storage-retention-size-and-efficiency.md @@ -147,14 +147,9 @@ measurements, not inferred table sizes. ## Product contract -1. **Branch store lifecycle.** Deleting a branch (or a worktree whose branch - is gone) schedules its branch-DB removal through the daemon. A periodic - daemon sweep reconciles `branches/` against live git refs. `branch gc` - remains the manual verb; the automatic path is the default. - - **Dated amendment (2026-08-07, recorded decision, supersedes the - per-branch-DB model above).** Per-branch SQLite copies are retired as the - write-side mechanism. Non-default-branch writes now land in the single +1. **Branch store lifecycle.** Per-branch SQLite copies are retired as the + write-side mechanism (dated amendment, 2026-08-07). Non-default-branch + writes now land in the single project store's next branch-graph publication epoch, fenced by the sync lease and sealed through the branch-meta `graph_source` CAS (`feat(graph): fence branch graph publication and mutation epochs`, diff --git a/docs/plans/tracedecay-v2/39-embedded-grafeo-graph-database.md b/docs/plans/tracedecay-v2/39-embedded-grafeo-graph-database.md index c37a3d432bb..ffb9ec389a1 100644 --- a/docs/plans/tracedecay-v2/39-embedded-grafeo-graph-database.md +++ b/docs/plans/tracedecay-v2/39-embedded-grafeo-graph-database.md @@ -1,14 +1,5 @@ # Embedded Grafeo Graph Database Implementation Plan -> **SUPERSEDED BY LANDING (2026-08-13).** The checklist below predates the -> cutover and was never ticked; the work shipped regardless. Evidence: -> `crates/tracedecay-graph-db` is a workspace member pinned to Grafeo -> `=0.5.42` with `generation_runtime.rs` and the sealed-graph state digest; -> the daemon consumes it through -> `src/daemon/code_index_scheduler/graph_activation.rs`; and the SQLite graph -> authority was deleted in `79683eb527`. Treat the unchecked boxes as -> historical planning, not open work. - **Goal:** Replace custom adjacency structures and graph-shaped SQLite storage with one embedded Grafeo runtime boundary while retaining SQLite only for genuinely relational, transactional, and content-bearing records. **Architecture:** `tracedecay-graph-db` is the only workspace crate allowed @@ -20,9 +11,8 @@ content, and source manifests remain in their domain stores; Grafeo is the sole persisted and queried graph/vector projection over those sources. SQLite does not store shadow adjacency, graph indexes, or vector indexes. -**Tech Stack:** Rust 2024, the published Grafeo `=0.5.42` crates from -crates.io, TraceDecay domain/store ports, Tokio cancellation, Criterion, and -cargo-nextest. +**Tech Stack:** Rust 2024, the Grafeo crates, TraceDecay domain/store ports, +Tokio cancellation, Criterion, and cargo-nextest. ## Global Constraints @@ -35,9 +25,6 @@ cargo-nextest. `.grafeo.wal` sidecar while the file is open and checkpoints it into the database file on close. Do not inject an already-open file, replace Grafeo's storage engine, or add a private/secure filesystem adapter. -- Depend on the exact published `=0.5.42` release. Do not pin an unpublished Git - revision, maintain a TraceDecay fork, vendor Grafeo, or reimplement its - storage, WAL, checkpoint, or recovery machinery. - `tracedecay-graph-db` is the sole production direct `grafeo-*` dependency. - Do not introduce another persisted/query graph or vector database. A bounded in-memory algorithm is not a storage authority, but should use the standard @@ -79,8 +66,6 @@ Official implementation references: - [Grafeo repository and crate layout](https://github.com/GrafeoDB/grafeo) - [Embedded Rust API](https://grafeo.dev/user-guide/rust/) -- [Vector and hybrid search](https://grafeo.dev/user-guide/vector-search/) -- [Published Rust API for `0.5.42`](https://docs.rs/grafeo/0.5.42/grafeo/) - [Persistent storage and WAL](https://grafeo.dev/user-guide/persistence/persistent/) --- @@ -250,11 +235,10 @@ Expected: fail because the crate and interfaces do not exist. - [ ] **Step 2: Add the workspace crate and centralize Grafeo dependencies** -Use the exact published Grafeo `=0.5.42` crates in the new crate. Add -`tracedecay-graph-db` to `[workspace].members` and centralize those exact -versions in `[workspace.dependencies]`. Enable Grafeo's public `grafeo-file` -feature together with `wal`. No other workspace crate directly depends on -Grafeo, and no Grafeo dependency uses a Git source. +Use the workspace Grafeo crates in the new crate. Add `tracedecay-graph-db` +to `[workspace].members` and centralize those dependencies in +`[workspace.dependencies]`. Enable Grafeo's public `grafeo-file` feature +together with `wal`. No other workspace crate directly depends on Grafeo. - [ ] **Step 3: Implement typed open, snapshot, batch, traversal, and vector adapters** @@ -395,67 +379,6 @@ cargo clippy -p tracedecay-graph-db -p tracedecay-code-index -p tracedecay-query git commit -am "refactor(code-graph): route Grafeo through graph-db" ``` -## Task 4: Move semantic vectors and hybrid retrieval - -**Files:** -- Modify: `crates/tracedecay-semantic/src/projector.rs` -- Modify: `crates/tracedecay-semantic/src/runtime_query.rs` -- Modify: `crates/tracedecay-application/src/store/vector_generations.rs` -- Modify: `src/store/vector_generations.rs` -- Modify: `crates/tracedecay-query/src/retrieval/semantic/execution_authority.rs` -- Delete: SQLite code-semantic vector payload/state tables superseded by graph-db - -**Interfaces:** -- Consumes: `EmbeddingProjectionKey`, verified `EmbeddingVector`, source - generation, model/artifact digest, metric, dimensions, and normalization. -- Produces: recovered-state-verified vector publication and bounded Grafeo - vector similarity; TraceDecay composes hybrid retrieval above it. - -```rust -pub trait SemanticVectorStore { - fn publish_generation( - &self, - generation: &VectorGenerationId, - vectors: Vec, - cancellation: &CancellationToken, - ) -> Result; - - fn search( - &self, - request: &SemanticVectorSearchRequest, - ) -> Result; -} -``` - -- [ ] **Step 1: Add failing vector behavior tests** - -Cover cosine/dot/euclidean behavior as declared by the manifest, dimension mismatch, non-finite values, stale generation, cancelled query, filtered namespace, deterministic ties, reopen, and serving the previous generation during rebuild. - -- [ ] **Step 2: Move vector payloads and indexes to graph-db** - -Store the vector on the corresponding projected entity or a typed vector -entity linked to it. Keep model lifecycle/install metadata and publication -receipts relational. Publish vector and code graph generations in one graph-db -batch when they share a generation. - -- [ ] **Step 3: Implement hybrid retrieval above the storage primitive** - -Grafeo returns bounded vector candidates. `tracedecay-query` retains -BM25/lexical retrieval and performs authorization, fusion, normalization, -hydration, explanations, and stable cursors. - -- [ ] **Step 4: Delete SQLite vector payload duplication** - -Remove `semantic_vector_payload_v1`, `semantic_vector_state_slice_v1`, evaluation duplicates, and legacy vector blobs after every production reader moves. Retain only relational generation/model lifecycle records that are not duplicate vector authority. - -- [ ] **Step 5: Verify and commit** - -```bash -cargo nextest run -p tracedecay-semantic -p tracedecay-query -p tracedecay-application --no-fail-fast -cargo bench --bench code_search --no-run -git commit -am "refactor(semantic): move vector search to graph-db" -``` - ## Task 5: Move Git topology and evidence relations **Files:** @@ -958,30 +881,6 @@ recovered-digest proof in compact form with "relation scalar endpoints do not match native topology" while the same rows seal and serve in replay form. Toy-scale contracts pass, so the defect is scale- or ordering-dependent. -**Decision.** Live-container reclaim is `compact()` under the maintenance -lease, not a bespoke re-seal or a quarantine-and-rebuild: - -1. Reproduce the endpoint mismatch in the fork with a generation-scale - fixture (the sealed-store contract's `rich_manifest` grown to the failing - size), fix it there, and gate `COMPACT_ROUND_TRIPS_BYTES` on the recovered - digest proving equal in compact and replay form for that fixture. -2. Add a maintenance-cadence step beside `run_branch_compaction`: when a - project's live container exceeds twice the summed bytes of its verified - heads' sealed artifacts (both figures are already censused, see the - `RetentionBacklog` finding's sealed evidence), take the project store - maintenance lease, quiesce the graph owner, `compact()`, close, and reopen. - The reopen reconstructs the layered store from the `CompactStore` section; - later writes land in the overlay, so the store stays mutable. -3. Prove the swap with the existing recovered-digest verification before the - compacted container replaces the live file; a mismatch keeps the original - and reports a typed degradation. Vector generations, whose only durable - home is the live container, are covered by the same proof and never - re-embedded. - -**Non-goals.** No re-projection from canonical replay authorities as a -compaction path (that is corruption recovery and would re-embed vectors), and -no quarantine of a healthy container. - ### Correction (2026-09-12, later the same day, measured) The decision above rests on a wrong premise. Read against the pinned fork: @@ -1039,14 +938,10 @@ The decision above rests on a wrong premise. Read against the pinned fork: base rather than a replay into RAM, deletes land in the overlay's deletions section, and `recompact()` merges on the maintenance cadence. Blocked on validating TraceDecay's staging writes against a - `LayeredStore` (README gaps 3 and 4: `has_vector_index` planning and - the MVCC-free base). + `LayeredStore` (README gap 4, the MVCC-free base). b. Rebuild a fresh container by streaming the retained authorities, sealed heads and their dependency closure from their compact stores, - pending replays from `canonical_replay_source`, vector generations - from their sealed stores, then prove every generation's recovered - digest against the journal before swapping files under the - maintenance lease. Memory is bounded by the largest generation, not - the container. Blocked on vector generations: the sealed lane never - routes vector search to sealed stores, so a rebuilt live container - must re-stage the vector rows and rebuild the HNSW index on open. + and pending replays from `canonical_replay_source`, then prove every + generation's recovered digest against the journal before swapping + files under the maintenance lease. Memory is bounded by the largest + generation, not the container. diff --git a/docs/plans/tracedecay-v2/40-partitioned-generation-storage-and-zero-copy-replay.md b/docs/plans/tracedecay-v2/40-partitioned-generation-storage-and-zero-copy-replay.md index 2cdba660c4a..36e520ba0cf 100644 --- a/docs/plans/tracedecay-v2/40-partitioned-generation-storage-and-zero-copy-replay.md +++ b/docs/plans/tracedecay-v2/40-partitioned-generation-storage-and-zero-copy-replay.md @@ -1,16 +1,5 @@ # 40. Partitioned generation storage and zero-copy replay retirement -Status: ACTIVE (decided 2026-08-22). Supersedes the monolithic sealed-generation -envelope as the long-term storage design; stages 0–1 land first as compatible -fixes inside the current `format_revision`. - -Stage state (reconciled 2026-08-31 against the tree, not the commit log): -stages 0a, 0b, and 0c are landed; stages 1 and 2 are landed; stage 3 is -untouched. Partitioned writers emit `format_revision` 7 while readers retain -explicit revision-5/6 monolithic compatibility. The -§Problem measurements below describe the pre-stage-0 state and are retained -as the original evidence, not as a description of the current tree. - ## Problem (measured, 2026-08-21/22, this repository) - One sealed code generation for this repository is a single @@ -63,55 +52,6 @@ representation and the replay economics: 5. **No second whole-generation copy, ever.** Replay reads canonical artifacts; retention must never require a byte-duplicating insurance copy. -## Staged delivery (each stage ships alone, oldest-first compatible) - -- **Stage 0, zero-copy replay retirement** (this change set, three commits): - - **0a** `hydrate_sealed_code_generation` resolves the sealed payload from - the canonical `code-generations-v1/` root first and falls back to the - replay pool. Readers become location-agnostic; digest verification on read - makes both locations equally trustworthy. Retirement moves strictly - canonical→pool, so a canonical-then-pool probe cannot miss a live file. - - **0b** Retention retires a superseded generation by **atomic rename into - the replay pool** (no-clobber; content-identical `AlreadyExists` collapses - to unlink) instead of unlinking, whenever the graph projection has not yet - durably consumed it. Rename is metadata-only and crash-atomic: the file is - always in exactly one of the two roots. The existing release queue keeps - deleting pool entries once the graph append is durable. - - **0c** Delete the eager staging copy from the graph publish path - (`install_project_graph_replay_seal_at` / `stage_project_graph_replay_seal` - and the copy machinery in `seals.rs`). Steady state stages **zero bytes**; - the pool only ever holds generations that were retired while still needed. -- **Stage 1, single-parse activation**: graph publication consumes the - already-decoded active generation (the projection manifest already rides - along on first publication; extend the same guarantee to the recovery - branches via the Stage-0a fallback) so cold activation parses the sealed - payload at most once. -- **Stage 2, manifest + segment split** (`format_revision` bump): split the - sealed envelope per §Decision 1–2; integer-keyed, content-addressed - segments; the 2 GiB whole-envelope bound and whole-file rewrite per - generation disappear; unchanged-file segments are shared across - generations. -- **Stage 3. Grafeo startup authority**: manifest ↔ Grafeo head validation - replaces replay on clean startup per §Decision 3–4. - -## Invariant mapping (old mechanism → new mechanism) - -| Invariant | Today | After | -|---|---|---| -| Content addressing | digest-named monolithic JSON | digest-named manifest + digest-named segments | -| Atomic publish | temp+rename of envelope + pointer | segments/Grafeo commit first, then manifest temp+rename | -| Crash recovery | eager 1.39 GB pool copy + journal replay | rename-retired canonical files + scoped lazy replay | -| Retention | unlink source; release pool copy via queue | rename-to-pool when graph pending; queue unchanged | -| Verification | full-file digest on every decode | per-component digest on read (unchanged per file) | -| Branch identity | durable index entry evidence | unchanged (manifest carries it) | - -## Non-goals - -- No change to sealing semantics, projector revisions, or the relational - journal contract in Stage 0. -- Stage 0 does not change `format_revision`; on-disk envelopes stay readable - both directions across the stage-0 commits. - ## Dated amendment (2026-09-12, recorded decision): stage 2b, shared, compressed segments **Measured.** For one worktree of a 150 MB / 5,190-file repository the scoped diff --git a/evals/intent_engine/HTTP_RUNNER.md b/evals/intent_engine/HTTP_RUNNER.md index e8f355dab66..b03f705ee17 100644 --- a/evals/intent_engine/HTTP_RUNNER.md +++ b/evals/intent_engine/HTTP_RUNNER.md @@ -47,6 +47,11 @@ duplicate/foreign cases, and malformed probability maps also refuse; no migratio or silent salvage is performed. Choose a new tag for another experiment or an old unidentifiable log. Completed output files are never overwritten. +A raw `--sample-out` capture must be outside the `results/` tree. The scorer +treats `results/*.json` as completed experiments, and a sample at the final +result pathname can block publication of a correctly completed run. The runner +refuses such a destination before creating the partial log or issuing inference. + An exclusive lock on the partial log rejects concurrent writers for the same tag/configuration. Rows are flushed after each response; this is not a crash-durable database or a concurrent-writer protocol. The runner writes and syncs the final result to a temporary file in the results directory, diff --git a/evals/intent_engine/common.py b/evals/intent_engine/common.py index 1678498ee6a..72670c2794f 100644 --- a/evals/intent_engine/common.py +++ b/evals/intent_engine/common.py @@ -2,6 +2,31 @@ import json import math + +def strict_json_loads(data): + """Read evaluation inputs and evidence without ambiguous JSON fields.""" + def unique_fields(pairs): + fields = {} + for key, value in pairs: + if key in fields: + raise ValueError(f"duplicate JSON field: {key}") + fields[key] = value + return fields + + def refuse_constant(value): + raise ValueError(f"invalid JSON numeric constant: {value}") + + def finite_float(token): + value = float(token) + if not math.isfinite(value): + raise ValueError(f"nonfinite JSON number: {token}") + return value + + return json.loads( + data, object_pairs_hook=unique_fields, + parse_constant=refuse_constant, parse_float=finite_float, + ) + CATS = ["session_recall", "memory_store", "call_graph", "impact", "refactor_edit", "review_changes", "type_orientation", "project_context", "unexpected_changes"] @@ -68,4 +93,5 @@ def json_window(c, turns=None): "recent_tools": [t["text"] for t in turns if t["role"] == "tool"]} def load(path="dataset.json"): - return json.load(open(path)) + with open(path, encoding="utf-8") as source: + return strict_json_loads(source.read()) diff --git a/evals/intent_engine/run_http.py b/evals/intent_engine/run_http.py index 07f9340f7f8..88a9aa40d08 100644 --- a/evals/intent_engine/run_http.py +++ b/evals/intent_engine/run_http.py @@ -22,7 +22,7 @@ HERE = os.path.dirname(os.path.abspath(__file__)) sys.path.insert(0, HERE) -from common import calibrated, noul_questions, prompt_only, text_window +from common import calibrated, noul_questions, prompt_only, strict_json_loads, text_window STATES = { "A_noul_prompt": prompt_only, @@ -30,6 +30,10 @@ "B2_noul_text_promptfirst": lambda c: text_window(c, prompt_first=True), } +# Each server response contains only a bounded set of typed decision answers. +# Refuse an oversized or streaming response before parsing untrusted JSON. +MAX_DECISION_RESPONSE_BYTES = 1024 * 1024 + def digest(value): return hashlib.sha256( @@ -42,6 +46,56 @@ def request_bytes(body): return json.dumps(body, allow_nan=False).encode("utf-8") +def sample_read_limit(requests): + """Bound saved raw captures relative to the transmitted request and HTTP cap.""" + largest_request = max(len(request_bytes(body)) for body, _ in requests.values()) + # Indentation and JSON escaping can expand the parsed HTTP body. Keep + # a conservative allowance without accepting arbitrarily large files. + return 64 * 1024 + 8 * (MAX_DECISION_RESPONSE_BYTES + largest_request) + + +def verify_existing_sample(path, header, requests, done, questions): + """A resumed capture must describe a durable response from this experiment. + + Never silently reuse an unrelated file as the model's first raw answer. + Opening a regular file without following links also avoids blocking on + hostile sample paths such as named pipes. + """ + flags = os.O_RDONLY | getattr(os, "O_NONBLOCK", 0) | getattr(os, "O_NOFOLLOW", 0) + descriptor = os.open(path, flags) + try: + if not stat.S_ISREG(os.fstat(descriptor).st_mode): + raise ValueError(f"sample path is not a regular file: {path}") + limit = sample_read_limit(requests) + if os.fstat(descriptor).st_size > limit: + raise ValueError("sample capture exceeds bounded evidence size") + with os.fdopen(descriptor, "rb") as source: + descriptor = None + contents = source.read(limit + 1) + if len(contents) > limit: + raise ValueError("sample capture exceeds bounded evidence size") + sample = strict_json_loads(contents) + finally: + if descriptor is not None: + os.close(descriptor) + case_id = sample.get("case_id") if isinstance(sample, dict) else None + if (not isinstance(case_id, str) or case_id not in requests + or sample.get("experiment_sha256") != header["experiment_sha256"] + or sample.get("request_sha256") != requests[case_id][1] + or sample.get("request") != requests[case_id][0]): + raise ValueError("sample capture does not belong to this experiment") + # --sample-out records the first response, not merely any completed row. + # A later response with matching request/digest is still mislabelled evidence. + if case_id != next(iter(requests)): + raise ValueError("sample capture must correspond to the first case") + if case_id not in done: + raise ValueError("sample capture has no durable matching response row") + response = sample.get("response") + noul_probabilities(response, questions) + if done[case_id].get("response_sha256") != digest(response): + raise ValueError("sample capture disagrees with durable response evidence") + + def publish_sample(path, serialized): """Publish a completely rendered sample without clobbering an existing one. @@ -58,11 +112,12 @@ def publish_sample(path, serialized): output.write(serialized) try: os.link(staged, path) - except FileExistsError: - # A dangling symlink occupies the pathname but exists() says - # false. Do not treat that unusable capture as already published. - if not os.path.isfile(path): - raise ValueError(f"sample path exists but is not a regular file: {path}") + except FileExistsError as error: + # Another run may have published a different capture after + # preflight. Never silently use it as evidence for this experiment. + if not os.path.isfile(path) or os.path.islink(path): + raise ValueError(f"sample path exists but is not a regular file: {path}") from error + raise ValueError(f"sample capture was concurrently published: {path}") from error finally: os.unlink(staged) @@ -170,13 +225,24 @@ def post(url, body, timeout=1800, bearer_token=None): ) started = time.perf_counter() with opener.open(req, timeout=timeout) as response: - out = json.loads(response.read()) + declared = response.headers.get("Content-Length") + if declared is not None: + try: + declared = int(declared) + except ValueError as error: + raise ValueError("decision response has an invalid Content-Length") from error + if declared < 0 or declared > MAX_DECISION_RESPONSE_BYTES: + raise ValueError("decision response exceeds the 1 MiB size limit") + payload = response.read(MAX_DECISION_RESPONSE_BYTES + 1) + if len(payload) > MAX_DECISION_RESPONSE_BYTES: + raise ValueError("decision response exceeds the 1 MiB size limit") + out = strict_json_loads(payload) return out, (time.perf_counter() - started) * 1000 def load_server_config(path): with open(path, encoding="utf-8") as source: - config = json.load(source) + config = strict_json_loads(source.read()) if not isinstance(config, dict): raise ValueError("--server-config must contain a JSON object") for field in ("checkpoint", "server_revision", "backend", "dtype"): @@ -212,11 +278,11 @@ def noul_probabilities(response, questions): raise ValueError("noul answers must cover exactly the requested questions") values = {} for key, answer in answers.items(): - # Older decision servers omit the kind, but a present tag must agree - # with the requested question. A Score response with an incidental - # noul field is not evidence of a successful Noul prediction. + # The current decision contract tags every answer. An untyped + # response could be an unrelated decision with an incidental noul + # value, so it cannot qualify as Noul model evidence. if (not isinstance(answer, dict) - or ("type" in answer and answer["type"] != "noul") + or answer.get("type") != "noul" or "noul" not in answer): raise ValueError(f"invalid noul answer for {key}") values[key] = answer["noul"] @@ -226,6 +292,10 @@ def noul_probabilities(response, questions): def serialize_result(row, questions, temperature): """Admit fresh and resumed rows before any durable output is changed.""" probabilities(row.get("probs"), questions) + # Every durable HTTP row must retain its raw response fingerprint. + response_sha = row.get("response_sha256") + if not isinstance(response_sha, str) or not re.fullmatch(r"[0-9a-f]{64}", response_sha): + raise ValueError("result row is missing a valid response digest") for field in ("ms", "server_ms", "input_tokens", "state_tokens", "dropped"): if field not in row: raise ValueError(f"result row is missing {field} for {row.get('id')}") @@ -305,14 +375,24 @@ def open_partial_for_resume(path): def resume_rows(partial, header, cases, requests, questions, temperature): """Validate the whole existing log before appending or issuing HTTP work.""" - first = partial.readline() - if not first.endswith("\n") or json.loads(first) != header: + header_limit = len(json.dumps(header, allow_nan=False)) + 1 + first = partial.readline(header_limit + 1) + if len(first) > header_limit: + raise ValueError("partial output has an oversized header") + if not first.endswith("\n") or strict_json_loads(first) != header: raise ValueError("partial output belongs to another or unrecorded experiment; use a new tag") + # Fresh rows contain only bounded probabilities and telemetry plus the + # known case ID. Do not load an arbitrary-sized untrusted log line. + row_limit = 64 * 1024 + max(len(json.dumps(case_id)) for case_id in cases) done = {} - for line_number, line in enumerate(partial, 2): + line_number = 1 + while line := partial.readline(row_limit + 1): + line_number += 1 + if len(line) > row_limit: + raise ValueError(f"partial output has an oversized row at line {line_number}") if not line.endswith("\n"): raise ValueError(f"partial output has an incomplete row at line {line_number}") - row = json.loads(line) + row = strict_json_loads(line) case_id = row.get("id") if isinstance(row, dict) else None if not isinstance(case_id, str) or case_id not in cases or case_id in done: raise ValueError(f"partial output has an unknown or duplicate case at line {line_number}") @@ -381,17 +461,33 @@ def main(argv=None): server_config = load_server_config(args.server_config) bearer_token = (load_bearer_token(args.bearer_token_env, args.url) if args.bearer_token_env is not None else None) - if args.sample_out and os.path.lexists(args.sample_out) and not os.path.isfile(args.sample_out): + if (args.sample_out and os.path.lexists(args.sample_out) + and (os.path.islink(args.sample_out) or not os.path.isfile(args.sample_out))): raise ValueError(f"sample path exists but is not a regular file: {args.sample_out}") + if args.sample_out: + # The scorer treats results/*.json as completed experiments. Publishing + # a raw sample under that tree can masquerade as a result, or occupy + # the final result path and permanently prevent exact resume. + result_root = os.path.normcase(os.path.realpath(os.path.join(HERE, "results"))) + sample_target = os.path.normcase(os.path.realpath(args.sample_out)) + try: + inside_results = os.path.commonpath([result_root, sample_target]) == result_root + except ValueError: + inside_results = False # Paths on different Windows drives. + if inside_results: + raise ValueError("--sample-out must be outside the results directory") with open(os.path.join(HERE, "dataset.json"), "rb") as source: dataset_bytes = source.read() - dataset = json.loads(dataset_bytes) + dataset = strict_json_loads(dataset_bytes) dataset_sha = hashlib.sha256(dataset_bytes).hexdigest() questions = noul_questions() case_ids = validate_dataset(dataset, questions) if args.subset: with open(args.subset, encoding="utf-8") as source: - ids = json.load(source)["ids"] + subset = strict_json_loads(source.read()) + if not isinstance(subset, dict): + raise ValueError("subset must be a JSON object with ids") + ids = subset.get("ids") if (not isinstance(ids, list) or not ids or any(not isinstance(case_id, str) for case_id in ids) or len(set(ids)) != len(ids) or not set(ids).issubset(case_ids)): @@ -403,13 +499,19 @@ def main(argv=None): cases = {case["id"]: case for case in dataset} os.makedirs(os.path.join(HERE, "results"), exist_ok=True) for cfg in args.configs: + # One --sample-out captures one response from the first configuration. + # Later configurations must not misattribute that capture to themselves. + sample_path = args.sample_out if cfg == args.configs[0] else None name = f"{args.tag}_{cfg}" path = os.path.join(HERE, "results", name + ".json") if os.path.lexists(path): raise ValueError(f"completed output already exists: {path}; use a new tag") + if (sample_path and os.path.lexists(sample_path) + and not os.path.lexists(path + ".partial")): + raise ValueError(f"sample capture already exists for a new run: {sample_path}") require_exclusive_publication(os.path.dirname(os.path.abspath(path))) - if args.sample_out and not os.path.isfile(args.sample_out): - require_exclusive_publication(os.path.dirname(os.path.abspath(args.sample_out))) + if sample_path and not os.path.lexists(sample_path): + require_exclusive_publication(os.path.dirname(os.path.abspath(sample_path))) requests = {} for case in dataset: body = {"state": STATES[cfg](case), "questions": questions} @@ -452,6 +554,17 @@ def main(argv=None): done = {} else: done = resume_rows(partial, header, cases, requests, questions, args.noul_temperature) + if sample_path and os.path.lexists(sample_path): + if created: + raise ValueError(f"sample capture already exists for a new run: {sample_path}") + verify_existing_sample(sample_path, header, requests, done, questions) + elif sample_path and done: + # A new capture would be the next response, not the first + # response of this experiment. Never mislabel it as raw evidence. + raise ValueError( + "cannot capture the first response after rows were recorded; " + "resume without --sample-out or use a new --tag" + ) for index, case in enumerate(dataset): if case["id"] in done: continue @@ -463,7 +576,10 @@ def main(argv=None): probs = noul_probabilities(response, questions) server_ms = response.get("latency_ms") if server_ms is None: - server_ms = (response.get("timing") or {}).get("inference_ms") + timing = response.get("timing") + if timing is not None and not isinstance(timing, dict): + raise ValueError("response timing must be an object or null") + server_ms = (timing or {}).get("inference_ms") usage = response.get("usage") if usage is None: usage = {} @@ -476,6 +592,7 @@ def main(argv=None): "input_tokens": usage.get("input_tokens"), "state_tokens": usage.get("state_tokens"), "dropped": usage.get("state_tokens_dropped"), "probs": probs, + "response_sha256": digest(response), } if args.noul_temperature is not None: row["probs_raw"] = probs @@ -484,11 +601,20 @@ def main(argv=None): # either. A non-finite response must not leave a sample file # that a later valid retry mistakes for a completed capture. serialized_row = serialize_result(row, questions, args.noul_temperature) - if args.sample_out and not os.path.exists(args.sample_out): - serialized_sample = json.dumps( - {"request": body, "response": response}, indent=1, allow_nan=False - ) - publish_sample(args.sample_out, serialized_sample) + if sample_path and not os.path.lexists(sample_path): + # The first raw response cannot be recaptured from a later + # case if its published sample disappeared during inference. + if index != 0: + raise ValueError("first-response sample disappeared; cannot capture a later case") + serialized_sample = json.dumps({ + "case_id": case["id"], + "experiment_sha256": header["experiment_sha256"], + "request_sha256": request_sha, + "request": body, "response": response, + }, indent=1, allow_nan=False) + if len(serialized_sample.encode("utf-8")) > sample_read_limit(requests): + raise ValueError("sample capture exceeds bounded evidence size") + publish_sample(sample_path, serialized_sample) partial.write(serialized_row + "\n") partial.flush() done[case["id"]] = row diff --git a/evals/intent_engine/score.py b/evals/intent_engine/score.py index da9375d9624..7fd1f2b52ef 100644 --- a/evals/intent_engine/score.py +++ b/evals/intent_engine/score.py @@ -6,7 +6,7 @@ """ import bisect, hashlib, json, glob, math, os, re, statistics, sys, tempfile sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) -from common import CATS, calibrated +from common import CATS, calibrated, strict_json_loads TH = [0.05, 0.1, 0.15, 0.2, 0.25, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9] @@ -100,6 +100,9 @@ def validate_http_provenance(result, rows): request_sha = row.get("request_sha256") if not isinstance(request_sha, str) or not re.fullmatch(r"[0-9a-f]{64}", request_sha): raise ValueError(f"HTTP case {row['id']} has an invalid request fingerprint") + response_sha = row.get("response_sha256") + if not isinstance(response_sha, str) or not re.fullmatch(r"[0-9a-f]{64}", response_sha): + raise ValueError(f"HTTP case {row['id']} is missing a valid response digest") requests.append([row["id"], request_sha]) if experiment.get("requests_sha256") != evidence_digest(requests): raise ValueError("HTTP request fingerprints disagree with experiment identity") @@ -210,7 +213,10 @@ def lat(rows): observed_truncated = sum(value > 0 for value in dropped) result = { "median_ms": round(statistics.median(ms)), - "p90_ms": round(ms[int(0.9 * (len(ms) - 1))]), + # Nearest-rank tails cannot understate short runs by taking the + # first value of a two-sample run. + "p90_ms": round(ms[math.ceil(0.90 * len(ms)) - 1]), + "p95_ms": round(ms[math.ceil(0.95 * len(ms)) - 1]), "total_cases": len(rows), "cases_truncated": observed_truncated if len(dropped) == len(rows) else None, "observed_cases_truncated": observed_truncated, @@ -304,10 +310,15 @@ def ch(rs): if __name__ == "__main__": here = os.path.dirname(os.path.abspath(__file__)) - fixed = json.load(open(os.path.join(here, "fixed_thresholds.json"))) if os.path.exists(os.path.join(here, "fixed_thresholds.json")) else None + fixed = None + thresholds_path = os.path.join(here, "fixed_thresholds.json") + if os.path.exists(thresholds_path): + with open(thresholds_path, encoding="utf-8") as source: + fixed = strict_json_loads(source.read()) summary = {}; allres = {} for f in sorted(glob.glob(os.path.join(here, "results", "*.json"))): - res = json.load(open(f)) + with open(f, encoding="utf-8") as source: + res = strict_json_loads(source.read()) if res["config"] in allres: raise ValueError("duplicate configuration %r in result files" % res["config"]) allres[res["config"]] = res @@ -317,12 +328,12 @@ def ch(rs): summary[k] = score_config(res, fixed) publish_summary(os.path.join(here, "summary.json"), summary) f2 = lambda x: " - " if x is None else "%.2f" % x - print("config macroAUC ctxAUC | t0.5: mF1 top1 top1ctx noneFH | t0.6: mF1 top1 top1ctx noneFH | best: mF1 top1 top1ctx noneFH | ECE | med/p90 ms") + print("config macroAUC ctxAUC | t0.5: mF1 top1 top1ctx noneFH | t0.6: mF1 top1 top1ctx noneFH | best: mF1 top1 top1ctx noneFH | ECE | med/p90/p95 ms") for k, s in summary.items(): line = "%-26s %s %s |" % (k, f2(s["macro_auc"]), f2(s["macro_auc_ctx"])) for nm in ("t0.5", "t0.6", "best_t_in_sample"): m = s[nm]; line += " %s %s %s %s |" % (f2(m["macro_f1"]), f2(m["all"]["top1_acc"]), f2(m["ctx"]["top1_acc"]), f2(m["all"]["none_false_hint_rate"])) - line += " LOOmF1 %s | %.3f | %d/%d" % (f2(s["macro_loo_f1"]), s["reliability"]["ece"], s["latency"]["median_ms"], s["latency"]["p90_ms"]) + line += " LOOmF1 %s | %.3f | %d/%d/%d" % (f2(s["macro_loo_f1"]), s["reliability"]["ece"], s["latency"]["median_ms"], s["latency"]["p90_ms"], s["latency"]["p95_ms"]) line += " AUC categories %d/%d ctx %d/%d" % (s["macro_auc_n_categories"], len(CATS), s["macro_auc_ctx_n_categories"], len(CATS)) if "choice_top1_acc" in s: line += " argmax top1 all/ctx %s/%s noneFH %s" % (f2(s["choice_top1_acc"]["all"]), f2(s["choice_top1_acc"]["ctx"]), f2(s["choice_none_false_hint_rate"])) diff --git a/evals/intent_engine/test_run_http.py b/evals/intent_engine/test_run_http.py index 6d5e710698c..be102d9364a 100644 --- a/evals/intent_engine/test_run_http.py +++ b/evals/intent_engine/test_run_http.py @@ -48,7 +48,7 @@ def write(self, name, value): (self.root / name).write_text(json.dumps(value), encoding="utf-8") def response(self, url, body): - return {"answers": {key: {"noul": 0.25} for key in body["questions"]}, + return {"answers": {key: {"type": "noul", "noul": 0.25} for key in body["questions"]}, "usage": {"input_tokens": 12}, "latency_ms": 0}, 5.0 def run_main(self, extra=()): @@ -111,6 +111,150 @@ def test_invalid_dataset_is_refused_before_network_or_partial_output(self): finally: self.write("dataset.json", self.dataset) + def test_ambiguous_json_inputs_refuse_before_inference(self): + dataset_path = self.root / "dataset.json" + server_path = self.root / "server.json" + subset_path = self.root / "subset.json" + original_dataset = dataset_path.read_text(encoding="utf-8") + original_server = server_path.read_text(encoding="utf-8") + variants = [ + (dataset_path, original_dataset.replace( + '"labels": ["impact"]', '"labels": [], "labels": ["impact"]', 1 + ), (), "duplicate JSON field"), + (server_path, original_server.replace( + '"checkpoint": "weights@immutable-revision"', + '"checkpoint": "other", "checkpoint": "weights@immutable-revision"', 1 + ), (), "duplicate JSON field"), + (subset_path, '{"ids":["first"],"ids":["second"]}', + ("--subset", str(subset_path)), "duplicate JSON field"), + (dataset_path, original_dataset.replace( + '"current_prompt": "What breaks?"', + '"current_prompt": "What breaks?", "debug": NaN', 1 + ), (), "invalid JSON numeric constant"), + (dataset_path, original_dataset.replace( + '"current_prompt": "What breaks?"', + '"current_prompt": "What breaks?", "debug": 1e999', 1 + ), (), "nonfinite JSON number"), + ] + for path, contents, extra, diagnostic in variants: + original = path.read_bytes() if path.exists() else None + try: + if original is not None: + self.assertNotEqual(contents, original.decode("utf-8")) + path.write_text(contents, encoding="utf-8") + with self.subTest(path=path.name, diagnostic=diagnostic): + with patch.object(run_http, "post") as post: + with self.assertRaisesRegex(ValueError, diagnostic): + self.run_main(extra) + post.assert_not_called() + self.assertFalse(self.partial.exists()) + self.assertFalse(self.output.exists()) + finally: + if original is None: + path.unlink() + else: + path.write_bytes(original) + + def test_ambiguous_partial_json_refuses_without_mutating_resume(self): + self.interrupt() + original = self.partial.read_text(encoding="utf-8") + header, row = original.splitlines() + variants = [ + header.replace( + '"experiment_sha256":', '"experiment_sha256": "fake", "experiment_sha256":', 1 + ) + "\n" + row + "\n", + header + "\n" + row.replace( + '"labels":', '"labels": [], "labels":', 1 + ) + "\n", + header + "\n" + row.replace( + '"ms":', '"unrelated": Infinity, "ms":', 1 + ) + "\n", + header + "\n" + row.replace( + '"ms":', '"unrelated": 1e999, "ms":', 1 + ) + "\n", + ] + try: + for ambiguous in variants: + with self.subTest(evidence=ambiguous[-120:]): + self.assertNotEqual(ambiguous, original) + self.partial.write_text(ambiguous, encoding="utf-8") + self.assert_refused_unchanged() + finally: + self.partial.write_text(original, encoding="utf-8") + with patch.object(run_http, "post", side_effect=self.response) as post: + self.run_main() + post.assert_called_once() + self.assertEqual(len(json.loads(self.output.read_text())["rows"]), 2) + + def test_oversized_resume_lines_refuse_before_network_without_damage(self): + self.interrupt() + original = self.partial.read_bytes() + header = original.splitlines(keepends=True)[0] + invalid = ( + (b"x" * 200_000 + b"\n", "oversized header"), + (header + b"x" * 200_000 + b"\n", "oversized row"), + ) + for contents, diagnostic in invalid: + with self.subTest(diagnostic=diagnostic): + self.partial.write_bytes(contents) + with patch.object(run_http, "post") as post: + with self.assertRaisesRegex(ValueError, diagnostic): + self.run_main() + post.assert_not_called() + self.assertEqual(self.partial.read_bytes(), contents) + self.assertFalse(self.output.exists()) + self.partial.write_bytes(original) + with patch.object(run_http, "post", side_effect=self.response) as post: + self.run_main() + post.assert_called_once() + self.assertEqual(len(json.loads(self.output.read_text())["rows"]), 2) + + def test_sample_capture_cannot_impersonate_scored_results(self): + result_root = self.output.parent + for sample in ( + self.output, + result_root / "foreign.json", + result_root / "nested" / "another.json", + result_root / ".." / "results" / "alias.json", + ): + with self.subTest(sample=str(sample)): + with patch.object(run_http, "post") as post: + with self.assertRaisesRegex(ValueError, "--sample-out must be outside"): + self.run_main(["--sample-out", str(sample)]) + post.assert_not_called() + self.assertFalse(self.partial.exists()) + self.assertFalse(self.output.exists()) + self.assertFalse(result_root.exists()) + + # Capturing raw evidence elsewhere still publishes a valid result. + sample = self.root / "first-response.json" + with patch.object(run_http, "post", side_effect=self.response) as post: + self.run_main(["--sample-out", str(sample)]) + self.assertEqual(post.call_count, len(self.dataset)) + self.assertEqual(json.loads(sample.read_text())["case_id"], "first") + self.assertEqual(len(json.loads(self.output.read_text())["rows"]), 2) + + def test_oversized_sample_capture_refuses_before_resume_inference(self): + sample = self.root / "first-response.json" + extra = ["--sample-out", str(sample)] + self.interrupt(extra) + original_sample = sample.read_bytes() + original_partial = self.partial.read_bytes() + oversized = b"x" * (9 * run_http.MAX_DECISION_RESPONSE_BYTES + 65_536) + sample.write_bytes(oversized) + with patch.object(run_http, "post") as post: + with self.assertRaisesRegex(ValueError, "sample capture exceeds"): + self.run_main(extra) + post.assert_not_called() + self.assertEqual(sample.read_bytes(), oversized) + self.assertEqual(self.partial.read_bytes(), original_partial) + self.assertFalse(self.output.exists()) + sample.write_bytes(original_sample) + with patch.object(run_http, "post", side_effect=self.response) as post: + self.run_main(extra) + post.assert_called_once() + self.assertEqual(len(json.loads(self.output.read_text())["rows"]), 2) + def test_exact_resume_only_requests_the_missing_case(self): self.interrupt() with patch.object(run_http, "post", side_effect=self.response) as post: @@ -128,6 +272,23 @@ def test_exact_resume_only_requests_the_missing_case(self): self.assertEqual(result["rows"][0]["server_ms"], 0) self.assertFalse(self.partial.exists()) + def test_late_first_response_capture_refuses_without_changing_resume(self): + self.interrupt() + before = self.partial.read_bytes() + sample = self.root / "late-first-response.json" + with patch.object(run_http, "post") as post: + with self.assertRaisesRegex(ValueError, "cannot capture the first response"): + self.run_main(["--sample-out", str(sample)]) + post.assert_not_called() + self.assertEqual(self.partial.read_bytes(), before) + self.assertFalse(sample.exists()) + self.assertFalse(self.output.exists()) + + # Omitting the unavailable raw capture still permits an exact resume. + with patch.object(run_http, "post", side_effect=self.response) as post: + self.run_main() + post.assert_called_once() + self.assertEqual(len(json.loads(self.output.read_text())["rows"]), 2) def test_completed_rows_cannot_be_relabelled_for_scoring(self): with patch.object(run_http, "post", side_effect=self.response): @@ -536,6 +697,18 @@ def test_invalid_settings_are_refused_without_inference(self): self.run_main() post.assert_not_called() + def test_malformed_subset_document_refuses_before_network_or_evidence(self): + subset_path = self.root / "subset.json" + for malformed in ([], None, "first", False, {"case_ids": ["first"]}): + with self.subTest(subset=malformed): + self.write("subset.json", malformed) + with patch.object(run_http, "post") as post: + with self.assertRaisesRegex(ValueError, "subset must"): + self.run_main(["--subset", str(subset_path)]) + post.assert_not_called() + self.assertFalse(self.partial.exists()) + self.assertFalse(self.output.exists()) + def test_invalid_subsets_cannot_silently_change_the_cohort(self): for ids in ([], ["absent"], ["first", "first"], "first"): with self.subTest(ids=ids), patch.object(run_http, "post") as post: @@ -757,11 +930,15 @@ def test_malformed_answers_do_not_become_completed_rows(self): self.assertEqual(self.partial.read_bytes(), before) def test_invalid_answer_kinds_and_shapes_preserve_resumable_rows(self): - self.interrupt() - before = self.partial.read_bytes() sample = self.root / "rejected-response.json" + self.interrupt(["--sample-out", str(sample)]) + before = self.partial.read_bytes() + original_sample = sample.read_bytes() response, elapsed = self.response("", {"questions": run_http.noul_questions()}) invalid = [] + untyped = copy.deepcopy(response) + del untyped["answers"]["impact"]["type"] + invalid.append(untyped) for declared_type in ("score", None, False, [], {"kind": "noul"}): malformed = copy.deepcopy(response) malformed["answers"]["impact"]["type"] = declared_type @@ -783,7 +960,7 @@ def test_invalid_answer_kinds_and_shapes_preserve_resumable_rows(self): self.run_main(["--sample-out", str(sample)]) self.assertEqual(self.partial.read_bytes(), before) self.assertFalse(self.output.exists()) - self.assertFalse(sample.exists()) + self.assertEqual(sample.read_bytes(), original_sample) # The rejected response did not poison the partial log; valid # inference can continue from the already-settled first case. @@ -803,6 +980,27 @@ def test_explicit_noul_answer_kind_still_records_valid_predictions(self): self.assertEqual(len(rows), len(self.dataset)) self.assertTrue(all(row["probs"]["impact"] == 0.25 for row in rows)) + def test_malformed_server_timing_refuses_without_publishing_evidence(self): + sample = self.root / "first-response.json" + response, elapsed = self.response("", {"questions": run_http.noul_questions()}) + response.pop("latency_ms") + for malformed in ([], "unknown", True, 23): + with self.subTest(timing=malformed): + with patch.object(run_http, "post", + return_value=({**response, "timing": malformed}, elapsed)): + with self.assertRaisesRegex(ValueError, "response timing must be an object"): + self.run_main(["--sample-out", str(sample)]) + self.assertEqual(len(self.partial.read_text().splitlines()), 1) + self.assertFalse(sample.exists()) + self.assertFalse(self.output.exists()) + + # Refusal must not poison a later valid run or publish misleading samples. + with patch.object(run_http, "post", side_effect=self.response) as post: + self.run_main(["--sample-out", str(sample)]) + self.assertEqual(post.call_count, len(self.dataset)) + self.assertEqual(len(json.loads(self.output.read_text())["rows"]), len(self.dataset)) + self.assertTrue(sample.exists()) + def test_malformed_response_does_not_poison_first_sample_on_resume(self): sample = self.root / "first-response.json" response, elapsed = self.response("", {"questions": run_http.noul_questions()}) @@ -832,14 +1030,194 @@ def test_nonfinite_raw_response_does_not_publish_sample_or_result(self): self.assertFalse(sample.exists()) self.assertEqual(len(self.partial.read_text().splitlines()), 1) - def test_existing_sample_is_never_replaced(self): + def test_existing_sample_from_other_run_refuses_without_inference(self): sample = self.root / "first-response.json" sample.write_text('{"previous": true}') - with patch.object(run_http, "post", side_effect=self.response): - self.run_main(["--sample-out", str(sample)]) + with patch.object(run_http, "post") as post: + with self.assertRaisesRegex(ValueError, "sample capture already exists"): + self.run_main(["--sample-out", str(sample)]) + post.assert_not_called() self.assertEqual(sample.read_text(), '{"previous": true}') + self.assertFalse(self.partial.exists()) + self.assertFalse(self.output.exists()) + + def test_missing_response_digest_refuses_resume_before_inference(self): + self.interrupt() + original = self.partial.read_bytes() + lines = original.splitlines() + self.assertEqual(len(lines), 2) + for invalid in ("missing", "malformed"): + with self.subTest(invalid=invalid): + saved = json.loads(lines[1]) + if invalid == "missing": + del saved["response_sha256"] + else: + saved["response_sha256"] = "not-a-sha256" + altered = lines[0] + b"\n" + json.dumps(saved).encode("utf-8") + b"\n" + self.partial.write_bytes(altered) + with patch.object(run_http, "post") as post: + with self.assertRaisesRegex(ValueError, "response digest"): + self.run_main() + post.assert_not_called() + self.assertEqual(self.partial.read_bytes(), altered) + self.assertFalse(self.output.exists()) + self.partial.write_bytes(original) + with patch.object(run_http, "post", side_effect=self.response) as post: + self.run_main() + post.assert_called_once() + self.assertFalse(self.partial.exists()) + self.assertEqual(len(json.loads(self.output.read_text())["rows"]), 2) + + def test_sample_identity_and_raw_response_must_match_durable_resume(self): + sample = self.root / "first-response.json" + extra = ["--sample-out", str(sample)] + self.interrupt(extra) + original = json.loads(sample.read_text()) + header = json.loads(self.partial.read_text().splitlines()[0]) + first_row = json.loads(self.partial.read_text().splitlines()[1]) + self.assertEqual(original["case_id"], "first") + self.assertEqual(original["experiment_sha256"], header["experiment_sha256"]) + self.assertEqual(run_http.digest(original["response"]), + first_row["response_sha256"]) + before = self.partial.read_bytes() + changes = ( + ("experiment", lambda capture: capture.__setitem__("experiment_sha256", "0" * 64)), + ("request", lambda capture: capture["request"].__setitem__("state", "unrelated")), + ("response", lambda capture: capture["response"]["answers"]["impact"].__setitem__( + "noul", 0.9)), + ) + for reason, change in changes: + with self.subTest(reason=reason): + altered = copy.deepcopy(original) + change(altered) + sample.write_text(json.dumps(altered)) + with patch.object(run_http, "post") as post: + with self.assertRaisesRegex(ValueError, "sample capture"): + self.run_main(extra) + post.assert_not_called() + self.assertEqual(self.partial.read_bytes(), before) + self.assertFalse(self.output.exists()) + sample.write_text(json.dumps(original)) + with patch.object(run_http, "post", side_effect=self.response) as post: + self.run_main(extra) + post.assert_called_once() + self.assertFalse(self.partial.exists()) self.assertEqual(len(json.loads(self.output.read_text())["rows"]), 2) + def test_resumed_sample_cannot_impersonate_a_later_durable_case(self): + sample = self.root / "first-response.json" + extra = ["--sample-out", str(sample)] + self.interrupt(extra) + # Leave both rows durable but simulate a failure to publish the result. + with patch.object(run_http, "post", side_effect=self.response): + with patch.object(run_http, "publish_final_result", + side_effect=OSError("publication interrupted")): + with self.assertRaisesRegex(OSError, "publication interrupted"): + self.run_main(extra) + self.assertEqual(len(self.partial.read_text().splitlines()), 3) + self.assertFalse(self.output.exists()) + + genuine = sample.read_bytes() + forged = json.loads(genuine) + second_request = { + "state": run_http.STATES[self.config](self.dataset[1]), + "questions": run_http.noul_questions(), + } + forged["case_id"] = "second" + forged["request"] = second_request + forged["request_sha256"] = hashlib.sha256( + run_http.request_bytes(second_request) + ).hexdigest() + second_row = json.loads(self.partial.read_text().splitlines()[2]) + self.assertEqual(forged["request_sha256"], second_row["request_sha256"]) + self.assertEqual(run_http.digest(forged["response"]), + second_row["response_sha256"]) + sample.write_text(json.dumps(forged), encoding="utf-8") + before = self.partial.read_bytes() + with patch.object(run_http, "post") as post: + with self.assertRaisesRegex(ValueError, "first case"): + self.run_main(extra) + post.assert_not_called() + self.assertEqual(self.partial.read_bytes(), before) + self.assertFalse(self.output.exists()) + + # The genuine first-case capture still allows exact completion. + sample.write_bytes(genuine) + with patch.object(run_http, "post") as post: + self.run_main(extra) + post.assert_not_called() + self.assertFalse(self.partial.exists()) + self.assertEqual(len(json.loads(self.output.read_text())["rows"]), 2) + + def test_missing_first_sample_is_not_replaced_with_later_response(self): + sample = self.root / "first-response.json" + calls = 0 + + def lose_capture_during_second_request(url, body): + nonlocal calls + calls += 1 + if calls == 2: + self.assertTrue(sample.exists()) + sample.unlink() + return self.response(url, body) + + with patch.object(run_http, "post", side_effect=lose_capture_during_second_request): + with self.assertRaisesRegex(ValueError, "sample disappeared"): + self.run_main(["--sample-out", str(sample)]) + self.assertEqual(calls, 2) + self.assertFalse(sample.exists()) + self.assertFalse(self.output.exists()) + saved = [json.loads(line) for line in self.partial.read_text().splitlines()] + self.assertEqual([row["id"] for row in saved[1:]], ["first"]) + + # Recovery without a sample still reuses the first measured row + # instead of claiming the second response was the first capture. + with patch.object(run_http, "post", side_effect=self.response) as post: + self.run_main() + post.assert_called_once() + self.assertFalse(self.partial.exists()) + self.assertEqual( + [row["id"] for row in json.loads(self.output.read_text())["rows"]], + ["first", "second"], + ) + + def test_multiple_configs_capture_only_first_configuration(self): + sample = self.root / "first-response.json" + first_config = "A_noul_prompt" + with patch.object(run_http, "post", side_effect=self.response) as post: + with contextlib.redirect_stdout(io.StringIO()): + run_http.main(self.base + [ + "--sample-out", str(sample), first_config, self.config, + ]) + self.assertEqual(post.call_count, len(self.dataset) * 2) + captured = json.loads(sample.read_text()) + first = json.loads((self.root / "results" / + f"probe_{first_config}.json").read_text()) + second = json.loads(self.output.read_text()) + self.assertEqual(captured["experiment_sha256"], first["experiment_sha256"]) + self.assertNotEqual(captured["experiment_sha256"], second["experiment_sha256"]) + self.assertEqual(run_http.digest(captured["response"]), + first["rows"][0]["response_sha256"]) + + def test_sample_link_race_cannot_adopt_foreign_capture(self): + sample = self.root / "first-response.json" + original_link = run_http.os.link + + def competing_sample(source, destination): + if destination == str(sample): + sample.write_text('{"foreign": true}') + raise FileExistsError(destination) + return original_link(source, destination) + + with patch.object(run_http, "post", side_effect=self.response): + with patch.object(run_http.os, "link", side_effect=competing_sample): + with self.assertRaisesRegex(ValueError, "concurrently published"): + self.run_main(["--sample-out", str(sample)]) + self.assertEqual(sample.read_text(), '{"foreign": true}') + self.assertFalse(self.output.exists()) + self.assertEqual(len(self.partial.read_text().splitlines()), 1) + self.assertEqual(list(self.root.glob(".noul-sample-*")), []) + @unittest.skipUnless(os.name == "posix", "dangling links require portable POSIX support") def test_dangling_sample_link_refuses_without_inference_or_partial_output(self): sample = self.root / "first-response.json" @@ -878,6 +1256,7 @@ def test_calibration_remains_finite_at_extreme_positive_temperatures(self): def test_transmitted_bytes_match_the_recorded_request_fingerprint(self): response = io.BytesIO(json.dumps({"answers": {}}).encode()) + response.headers = {"Content-Length": str(len(response.getbuffer()))} body = {"state": "Unicode λ", "questions": {"a": {}, "b": {}}} with patch("urllib.request.build_opener") as build_opener: build_opener.return_value.open.return_value = response @@ -1050,5 +1429,109 @@ def log_message(self, *_): server.server_close() + def test_duplicate_json_response_fields_cannot_publish_evaluation_evidence(self): + response, _ = self.response("", {"questions": run_http.noul_questions()}) + valid = json.dumps(response).encode("utf-8") + duplicate_answers = ( + b'{"answers":{},"answers":' + + json.dumps(response["answers"]).encode("utf-8") + b'}' + ) + duplicate_noul = valid.replace( + b'"noul": 0.25', b'"noul": 0.1, "noul": 0.25', 1 + ) + self.assertNotEqual(duplicate_noul, valid) + + class Answer(BaseHTTPRequestHandler): + payload = valid + + def do_POST(self): + self.rfile.read(int(self.headers["Content-Length"])) + payload = type(self).payload + self.send_response(200) + self.send_header("Content-Length", str(len(payload))) + self.end_headers() + self.wfile.write(payload) + + def log_message(self, *_): + pass + + server = HTTPServer(("127.0.0.1", 0), Answer) + worker = Thread(target=server.serve_forever, daemon=True) + worker.start() + sample = self.root / "first-response.json" + extra = [ + "--url", f"http://127.0.0.1:{server.server_port}/v1/systemone", + "--sample-out", str(sample), + ] + try: + for ambiguous in (duplicate_answers, duplicate_noul): + Answer.payload = ambiguous + with self.subTest(payload=ambiguous[:100]): + with self.assertRaisesRegex(ValueError, "duplicate JSON field"): + self.run_main(extra) + self.assertEqual(len(self.partial.read_text().splitlines()), 1) + self.assertFalse(sample.exists()) + self.assertFalse(self.output.exists()) + Answer.payload = valid + self.run_main(extra) + result = json.loads(self.output.read_text(encoding="utf-8")) + self.assertEqual([row["id"] for row in result["rows"]], ["first", "second"]) + self.assertTrue(sample.exists()) + self.assertFalse(self.partial.exists()) + finally: + server.shutdown() + worker.join(timeout=2) + server.server_close() + + + def test_decision_response_rejects_oversized_or_invalid_http_bodies(self): + class Answer(BaseHTTPRequestHandler): + def do_POST(self): + self.rfile.read(int(self.headers["Content-Length"])) + valid = b'{"answers":{}}' + self.send_response(200) + if self.path == "/declared": + # The server claims a huge body but sends valid, small JSON. + # A check after an unrestricted read would accept this. + self.send_header("Content-Length", str(run_http.MAX_DECISION_RESPONSE_BYTES + 1)) + self.end_headers() + self.wfile.write(valid) + elif self.path == "/invalid": + self.send_header("Content-Length", "not-a-number") + self.end_headers() + self.wfile.write(valid) + else: + # No Content-Length: a streaming server must also be bounded. + payload = (b'{"answers":{},"padding":"' + + b"x" * run_http.MAX_DECISION_RESPONSE_BYTES + b'"}') + self.end_headers() + try: + self.wfile.write(payload) + except BrokenPipeError: + pass + + def log_message(self, *_): + pass + + server = HTTPServer(("127.0.0.1", 0), Answer) + worker = Thread(target=server.serve_forever, daemon=True) + worker.start() + try: + base = f"http://127.0.0.1:{server.server_port}" + body = {"state": "private transcript", "questions": {}} + for path, message in ( + ("/declared", "size limit"), + ("/streamed", "size limit"), + ("/invalid", "invalid Content-Length"), + ): + with self.subTest(path=path): + with self.assertRaisesRegex(ValueError, message): + run_http.post(base + path, body, timeout=2) + finally: + server.shutdown() + worker.join(timeout=2) + server.server_close() + + if __name__ == "__main__": unittest.main() diff --git a/evals/intent_engine/test_score.py b/evals/intent_engine/test_score.py index c5f7100b5f5..a00d7e0310d 100644 --- a/evals/intent_engine/test_score.py +++ b/evals/intent_engine/test_score.py @@ -30,7 +30,8 @@ def row(case_id, labels=(), context=False, choice=None): def signed_http_result(config, rows, temperature=None): # Fixture independently reproduces the fingerprint scheme in run_http.py. rows = [ - {**item, "request_sha256": hashlib.sha256(item["id"].encode()).hexdigest()} + {**item, "request_sha256": hashlib.sha256(item["id"].encode()).hexdigest(), + "response_sha256": hashlib.sha256((item["id"] + ":response").encode()).hexdigest()} for item in rows ] @@ -121,6 +122,22 @@ def test_singleton_has_no_leave_one_out_training_evidence(self): self.assertIsNone(result["per_cat"][category][metric]) self.assertEqual(result["t0.5"]["all"]["top1_acc"], 1.0) + def test_tail_latency_reports_nearest_rank_for_small_and_full_cohorts(self): + # Old floor((n - 1) * p) selected the fastest of two requests. + # Tail percentiles must come from observed samples. + for samples, expected in ( + ([10], (10, 10, 10)), + ([10, 20], (15, 20, 20)), + (list(range(1, 21)), (10, 18, 19)), + ): + with self.subTest(samples=samples): + rows = [{**row(str(i)), "ms": elapsed} + for i, elapsed in enumerate(samples)] + timing = score.score_config({"config": "tails", "rows": rows})["latency"] + actual = (timing["median_ms"], timing["p90_ms"], timing["p95_ms"]) + self.assertEqual(actual, expected) + self.assertEqual(timing["total_cases"], len(samples)) + def test_empty_results_and_partial_choice_coverage_are_refused(self): with self.assertRaisesRegex(ValueError, "empty.*no rows to score"): score.score_config({"config": "empty", "rows": []}) @@ -218,6 +235,8 @@ def test_cli_reports_the_scored_subset_without_loading_the_full_dataset(self): self.assertEqual(scored.returncode, 0, scored.stderr) summary = json.loads((root / "summary.json").read_text(encoding="utf-8")) self.assertIsNone(summary["sample"]["choice_top1_acc"]["no_ctx"]) + self.assertEqual(summary["sample"]["latency"]["p95_ms"], 100) + self.assertIn("med/p90/p95 ms", scored.stdout) self.assertIn( f"AUC categories 1/{len(score.CATS)} ctx 1/{len(score.CATS)}", scored.stdout, ) @@ -228,6 +247,44 @@ def test_cli_reports_the_scored_subset_without_loading_the_full_dataset(self): self.assertIn("AUC eligible categories", compared.stdout) self.assertFalse((root / "dataset.json").exists()) + def test_ambiguous_completed_result_refuses_to_replace_valid_summary(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + self.stage_scripts(root) + results = root / "results" + results.mkdir() + original = json.dumps({"config": "sample", "rows": [row("a")]}) + variants = [ + (original.replace( + '"config": "sample"', '"config": "other", "config": "sample"', 1 + ), "duplicate JSON field"), + (original.replace( + '"labels": []', '"labels": ["impact"], "labels": []', 1 + ), "duplicate JSON field"), + (original.replace( + '"ms": 100', '"ignored": NaN, "ms": 100', 1 + ), "invalid JSON numeric constant"), + (original.replace( + '"ms": 100', '"ignored": -1e999, "ms": 100', 1 + ), "nonfinite JSON number"), + ] + summary = root / "summary.json" + for contents, diagnostic in variants: + self.assertNotEqual(contents, original) + (results / "sample.json").write_text(contents, encoding="utf-8") + summary.write_bytes(b"last verified summary\\n") + with self.subTest(diagnostic=diagnostic): + attempt = self.run_script(root, "score.py") + self.assertNotEqual(attempt.returncode, 0) + self.assertIn(diagnostic, attempt.stderr) + self.assertEqual(summary.read_bytes(), b"last verified summary\\n") + (results / "sample.json").write_text(original, encoding="utf-8") + scored = self.run_script(root, "score.py") + self.assertEqual(scored.returncode, 0, scored.stderr) + self.assertEqual( + json.loads(summary.read_text())["sample"]["latency"]["total_cases"], 1 + ) + def test_empty_cli_result_does_not_overwrite_an_existing_summary(self): with tempfile.TemporaryDirectory() as directory: root = Path(directory) @@ -404,6 +461,35 @@ def test_http_provenance_cannot_be_downgraded_to_local_scoring(self): self.assertIn("missing its experiment identity", attempt.stderr) self.assertEqual(summary.read_bytes(), b"last verified summary\n") + def test_http_response_digest_required_before_scoring_or_publishing(self): + original = signed_http_result("B_noul_text", [row("first"), row("second")]) + self.assertEqual(score.score_config(original)["latency"]["total_cases"], 2) + for bad_digest in (None, "", "not-a-sha256", "G" * 64, 1): + changed = json.loads(json.dumps(original)) + if bad_digest is None: + del changed["rows"][0]["response_sha256"] + else: + changed["rows"][0]["response_sha256"] = bad_digest + with self.subTest(bad_digest=bad_digest): + with self.assertRaisesRegex(ValueError, "response digest"): + score.score_config(changed) + + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + self.stage_scripts(root) + (root / "results").mkdir() + damaged = json.loads(json.dumps(original)) + del damaged["rows"][0]["response_sha256"] + (root / "results" / "damaged.json").write_text( + json.dumps(damaged), encoding="utf-8" + ) + summary = root / "summary.json" + summary.write_bytes(b"last verified summary\n") + attempt = self.run_script(root, "score.py") + self.assertNotEqual(attempt.returncode, 0) + self.assertIn("response digest", attempt.stderr) + self.assertEqual(summary.read_bytes(), b"last verified summary\n") + def test_http_result_fingerprints_bind_case_order_and_experiment(self): original = signed_http_result("B_noul_text", [row("first"), row("second")]) self.assertEqual(score.score_config(original)["latency"]["total_cases"], 2)