diff --git a/contextgraph-conformance/src/bin/contextgraph-example-docs.rs b/contextgraph-conformance/src/bin/contextgraph-example-docs.rs index 732ff71..3fe7b16 100644 --- a/contextgraph-conformance/src/bin/contextgraph-example-docs.rs +++ b/contextgraph-conformance/src/bin/contextgraph-example-docs.rs @@ -17,8 +17,8 @@ use clap::{Parser, ValueEnum}; use contextgraph_host::wire::Envelope; use contextgraph_types::capability::QueryCapability; use contextgraph_types::{ - Capabilities, ContextFrame, ContextQueryResult, DataFlow, EgressScope, FrameKind, - PROTOCOL_VERSION, Provenance, ProviderInfo, + Capabilities, ContextFrame, ContextQueryResult, DataFlow, FrameKind, FrameVerdict, + PROTOCOL_VERSION, Provenance, ProviderInfo, Verdict, VerifyRequest, VerifyResponse, }; /// Ways this fixture can deliberately violate the protocol, each tripping a @@ -41,12 +41,26 @@ enum Misbehave { /// Exit on receiving a malformed line (trips /// `malformed-input-tolerance`). CrashOnGarbage, + /// Answer `valid` to every `context/verify` entry without comparing + /// digests — the rubber stamp that makes reuse unsafe (trips + /// `verify-honesty`). + RubberStampVerify, + /// Advertise `verify` support at the handshake but answer `unknown` to + /// everything — a provider that claims a capability it does not have + /// (trips `verify-honesty`). + HollowVerify, /// Declare an off-machine egress scope (`third-party-index`) alongside /// `egress: false` — a scope that contradicts the data-flow posture /// (trips `consent-scope`). ScopeLie, } +/// The digests this fixture currently serves, one per canned frame. Verify +/// answers are derived from these, so the fixture's `context/verify` verdicts +/// and the frames it serves can never drift apart. +const GETTING_STARTED_DIGEST: &str = "sha256:getting-started-v1"; +const CONFIGURATION_DIGEST: &str = "sha256:configuration-v1"; + #[derive(Parser)] #[command( name = "contextgraph-example-docs", @@ -115,6 +129,21 @@ fn main() { }, ); } + Envelope::Verify { request } => { + let response = match args.misbehave { + // Claims every held frame is still good without comparing + // a single digest — the lie `verify-honesty` catches. + Some(Misbehave::RubberStampVerify) => { + VerifyResponse::uniform(&request, Verdict::Valid) + } + // Advertises verify but can never vouch for anything. + Some(Misbehave::HollowVerify) => { + VerifyResponse::uniform(&request, Verdict::Unknown) + } + _ => verify_honestly(&request), + }; + write_envelope(&mut stdout, &Envelope::Verified { response }); + } Envelope::Shutdown => std::process::exit(0), // handshake_ack / frames / error are host→provider-invalid inputs; // a provider ignores them. @@ -163,9 +192,57 @@ fn capabilities() -> Capabilities { graph: false, embeddings_fingerprint: None, subscribe: false, + // This fixture can compare a presented digest against the one it + // currently serves, so it advertises pull-based verification (§4). It + // cannot watch its sources, so it does not advertise `subscribe` — the + // exact provider shape verify exists for. + verify: true, + } +} + +/// The digest this fixture serves for a frame id *right now*, or `None` if it +/// does not serve that frame at all. +fn current_digest(frame_id: &str) -> Option<&'static str> { + match frame_id { + "frm_getting_started" => Some(GETTING_STARTED_DIGEST), + "frm_configuration" => Some(CONFIGURATION_DIGEST), + _ => None, } } +/// Answer a `context/verify` request honestly, by comparing each presented +/// digest against the one this provider currently serves (§4). +/// +/// This is the whole provider-side contract: the digest is provider-declared +/// and opaque, so only the provider can say whether the bytes behind an +/// identity still match. A digest that differs from the current one is exactly +/// what a mutated source looks like from here. +fn verify_honestly(request: &VerifyRequest) -> VerifyResponse { + VerifyResponse::new( + request + .frames + .iter() + .map(|frame| { + let verdict = match current_digest(&frame.frame_id) { + // Never served, or no longer served: nothing to revalidate. + None => Verdict::Gone, + Some(current) => match frame.content_digest.as_deref() { + // Nothing presented to compare against. + None => Verdict::Unknown, + Some(presented) if presented == current => Verdict::Valid, + // The source moved on. Offer the current digest so the + // host can tell what it would be re-fetching. + Some(_) => Verdict::Stale { + replacement_digest: Some(current.to_string()), + }, + }, + }; + FrameVerdict::new(frame.clone(), verdict) + }) + .collect(), + ) +} + fn canned_frames(misbehave: Option) -> Vec { let bad_score = misbehave == Some(Misbehave::BadScore); let empty_citation = misbehave == Some(Misbehave::EmptyCitation); @@ -186,7 +263,7 @@ fn canned_frames(misbehave: Option) -> Vec { "Install the reference binding with `cargo add contextgraph-types`, then implement \ the four required methods." .into(), - content_digest: Some("sha256:getting-started-v1".into()), + content_digest: Some(GETTING_STARTED_DIGEST.into()), uri: Some("file:///docs/getting-started.md".into()), score: if bad_score { 1.5 } else { 0.82 }, token_cost, @@ -216,7 +293,7 @@ fn canned_frames(misbehave: Option) -> Vec { content: "Providers declare their data-flow direction at the handshake so hosts can \ gate consent before sending any query." .into(), - content_digest: Some("sha256:configuration-v1".into()), + content_digest: Some(CONFIGURATION_DIGEST.into()), uri: Some("file:///docs/configuration.md".into()), score: 0.61, token_cost, diff --git a/contextgraph-conformance/src/lib.rs b/contextgraph-conformance/src/lib.rs index 1e1caee..2e02329 100644 --- a/contextgraph-conformance/src/lib.rs +++ b/contextgraph-conformance/src/lib.rs @@ -18,6 +18,10 @@ //! - **frame-validity** — queried frames pass `contextgraph-types` validation: score //! in `[0, 1]`, a non-empty title, a non-empty `citation_label` (§3.4 — //! "NEVER a bare uuid"). +//! - **verify-honesty** — a provider advertising `verify` answers `valid` for +//! frames it just served and `stale` when their digests are mutated +//! (`docs/context-reuse.md` §4). Skipped when `verify` is not advertised — +//! that is the declared fallback, not a failure. //! - **budget-honesty** — returned frames' summed `token_cost` never exceeds //! the query budget (§3.3 — "never lies about cost"). //! - **shutdown-clean** — the provider tears down without error (§3.2). @@ -44,6 +48,7 @@ pub use report::{CheckResult, CheckStatus, ConformanceReport}; pub const CHECK_HANDSHAKE: &str = "handshake"; pub const CHECK_CONSENT_SCOPE: &str = "consent-scope"; pub const CHECK_FRAME_VALIDITY: &str = "frame-validity"; +pub const CHECK_VERIFY_HONESTY: &str = "verify-honesty"; pub const CHECK_BUDGET_HONESTY: &str = "budget-honesty"; pub const CHECK_SHUTDOWN: &str = "shutdown-clean"; pub const CHECK_MALFORMED: &str = "malformed-input-tolerance"; @@ -108,7 +113,7 @@ pub async fn run_conformance(target: ProviderTarget) -> ConformanceReport { )); } checks.push(check_consent_scopes(&info)); - run_query_and_shutdown_checks(host, &id, &mut checks).await; + run_query_and_shutdown_checks(host, &id, &caps, &mut checks).await; } Err(error) => { checks.push(CheckResult::fail( @@ -116,8 +121,9 @@ pub async fn run_conformance(target: ProviderTarget) -> ConformanceReport { format!("could not establish provider: {error}"), )); for name in [ - CHECK_CONSENT_SCOPE, CHECK_FRAME_VALIDITY, + CHECK_VERIFY_HONESTY, + CHECK_CONSENT_SCOPE, CHECK_BUDGET_HONESTY, CHECK_SHUTDOWN, ] { @@ -205,12 +211,18 @@ fn capture_identity( )) } -async fn run_query_and_shutdown_checks(host: Host, id: &str, checks: &mut Vec) { +async fn run_query_and_shutdown_checks( + host: Host, + id: &str, + caps: &Capabilities, + checks: &mut Vec, +) { let query = sample_query(); match host.query_provider(id, &query).await { Ok(result) => { let (ok, evidence) = check_frames(&result); checks.push(CheckResult::from_bool(CHECK_FRAME_VALIDITY, ok, evidence)); + checks.push(check_verify_honesty(&host, id, caps, &result).await); if result.respects_budget(query.max_tokens) { checks.push(CheckResult::pass( @@ -236,6 +248,7 @@ async fn run_query_and_shutdown_checks(host: Host, id: &str, checks: &mut Vec { let evidence = format!("query failed: {error}"); checks.push(CheckResult::fail(CHECK_FRAME_VALIDITY, evidence.clone())); + checks.push(CheckResult::fail(CHECK_VERIFY_HONESTY, evidence.clone())); checks.push(CheckResult::fail(CHECK_BUDGET_HONESTY, evidence)); } } @@ -257,6 +270,128 @@ async fn run_query_and_shutdown_checks(host: Host, id: &str, checks: &mut Vec CheckResult { + if !caps.verify { + return CheckResult::skip( + CHECK_VERIFY_HONESTY, + "provider does not advertise `verify`; a host falls back to re-querying its frames (§4)", + ); + } + + let held: Vec = result + .frames + .iter() + .filter(|frame| frame.content_digest.is_some()) + .map(|frame| FrameId::new(id, frame.id.clone(), frame.content_digest.clone())) + .collect(); + if held.is_empty() { + return CheckResult::skip( + CHECK_VERIFY_HONESTY, + "provider served no frame carrying a `content_digest`, so nothing is verifiable (§1 D4)", + ); + } + + // Ask 1: the real digests. Every frame the provider just served must + // verify valid — otherwise it cannot vouch for its own output. + let unchanged = host.verify_frames(&held).await; + if !unchanged.dropped.is_empty() { + let detail: Vec = unchanged + .dropped + .iter() + .map(|dropped| format!("{} => {:?}", dropped.frame.frame_id, dropped.reason)) + .collect(); + return CheckResult::fail( + CHECK_VERIFY_HONESTY, + format!( + "provider advertises `verify` but did not answer `valid` for {} of {} frame(s) it had just served with unchanged digests: {}", + unchanged.dropped.len(), + held.len(), + detail.join(", ") + ), + ); + } + + // Ask 2: the same frames with mutated digests — what a changed source + // looks like from the provider's side. + let mutated: Vec = held + .iter() + .map(|frame| { + FrameId::new( + id, + frame.frame_id.clone(), + frame + .content_digest + .as_ref() + .map(|digest| format!("{digest}{MUTATED_SUFFIX}")), + ) + }) + .collect(); + let changed = host.verify_frames(&mutated).await; + + if !changed.retained.is_empty() { + return CheckResult::fail( + CHECK_VERIFY_HONESTY, + format!( + "provider answered `valid` for {} frame(s) whose content digest it never served — a rubber stamp that lets a host cite stale evidence", + changed.retained.len() + ), + ); + } + let not_stale: Vec = changed + .dropped + .iter() + .filter(|dropped| !matches!(dropped.reason, DropReason::Stale { .. })) + .map(|dropped| format!("{} => {:?}", dropped.frame.frame_id, dropped.reason)) + .collect(); + if !not_stale.is_empty() { + return CheckResult::fail( + CHECK_VERIFY_HONESTY, + format!( + "a digest mismatch on a frame the provider still serves MUST verify `stale` (§4 V1); got: {}", + not_stale.join(", ") + ), + ); + } + + CheckResult::pass( + CHECK_VERIFY_HONESTY, + format!( + "provider verified {n} unchanged frame(s) `valid` and all {n} mutated digest(s) `stale`, carrying no frame bodies", + n = held.len() + ), + ) +} + /// Wire-level probe: complete the handshake on a fresh connection, inject a /// malformed line, then send a valid query. A conforming provider ignores or /// cleanly errors on the garbage and stays alive to answer the query; a diff --git a/contextgraph-conformance/tests/conformance_suite.rs b/contextgraph-conformance/tests/conformance_suite.rs index 41815b3..808468a 100644 --- a/contextgraph-conformance/tests/conformance_suite.rs +++ b/contextgraph-conformance/tests/conformance_suite.rs @@ -43,6 +43,7 @@ async fn a_well_behaved_provider_is_fully_conformant() { CHECK_HANDSHAKE, CHECK_CONSENT_SCOPE, CHECK_FRAME_VALIDITY, + CHECK_VERIFY_HONESTY, CHECK_BUDGET_HONESTY, CHECK_SHUTDOWN, CHECK_MALFORMED, @@ -111,3 +112,27 @@ async fn an_incompatible_protocol_version_fails_the_handshake() { CheckStatus::Skipped ); } + +#[tokio::test] +async fn rubber_stamping_every_verify_as_valid_fails_verify_honesty() { + // The dangerous lie: a provider that answers `valid` without comparing + // digests lets a host go on citing evidence that changed underneath it. + let report = run_conformance(target(&["--misbehave", "rubber-stamp-verify"])).await; + assert!(!report.passed()); + assert_eq!(status_of(&report, CHECK_VERIFY_HONESTY), CheckStatus::Fail); + // Nothing else is disturbed — the frames it serves are still well-formed + // and honestly costed, so only the verify check catches this. + for name in [CHECK_HANDSHAKE, CHECK_FRAME_VALIDITY, CHECK_BUDGET_HONESTY] { + assert_eq!(status_of(&report, name), CheckStatus::Pass, "{name}"); + } +} + +#[tokio::test] +async fn advertising_verify_while_vouching_for_nothing_fails_verify_honesty() { + // The other direction: a provider that claims the capability but answers + // `unknown` to everything cannot revalidate its own just-served frames. + let report = run_conformance(target(&["--misbehave", "hollow-verify"])).await; + assert!(!report.passed()); + assert_eq!(status_of(&report, CHECK_VERIFY_HONESTY), CheckStatus::Fail); + assert_eq!(status_of(&report, CHECK_HANDSHAKE), CheckStatus::Pass); +} diff --git a/contextgraph-conformance/tests/verify_wire.rs b/contextgraph-conformance/tests/verify_wire.rs new file mode 100644 index 0000000..80752af --- /dev/null +++ b/contextgraph-conformance/tests/verify_wire.rs @@ -0,0 +1,133 @@ +//! `context/verify` wire-level tests (`docs/context-reuse.md` §4). +//! +//! Two things the type-level tests can't cover: that the published reference +//! messages stay in lockstep with the types (the repo's schema/examples sync +//! convention), and that a real stdio provider answers a verify exchange +//! honestly over an actual pipe. + +use contextgraph_host::wire::Envelope; +use contextgraph_host::{DropReason, Host}; +use contextgraph_types::{FrameId, Verdict}; + +fn fixture() -> String { + env!("CARGO_BIN_EXE_contextgraph-example-docs").to_string() +} + +/// Every published reference message must still parse as an `Envelope`, and +/// the verify exchange must round-trip byte-for-byte — the examples are the +/// cross-language contract, so drift between them and the types is a defect. +#[test] +fn published_reference_messages_match_the_types() { + let raw = std::fs::read_to_string(concat!( + env!("CARGO_MANIFEST_DIR"), + "/../examples/reference-messages.json" + )) + .expect("reference-messages.json is readable"); + let messages: Vec = + serde_json::from_str(&raw).expect("reference messages are a JSON array"); + + let mut saw_verify = false; + let mut saw_verified = false; + for (i, message) in messages.iter().enumerate() { + let kind = message["type"].as_str().expect("every message has a type"); + let parsed: Envelope = serde_json::from_value(message.clone()) + .unwrap_or_else(|e| panic!("reference message {i} (`{kind}`) does not parse: {e}")); + + if matches!(kind, "verify" | "verified") { + let reserialized = serde_json::to_value(&parsed).unwrap(); + assert_eq!( + &reserialized, message, + "reference message {i} (`{kind}`) does not round-trip through the types" + ); + saw_verify |= kind == "verify"; + saw_verified |= kind == "verified"; + } + } + assert!( + saw_verify && saw_verified, + "the reference messages must document a full verify exchange" + ); +} + +/// A verify request carries identities, never bodies — the economic claim of +/// §4, asserted against the actual serialized wire form. +#[test] +fn a_verify_exchange_carries_no_frame_bodies_on_the_wire() { + let request = contextgraph_types::VerifyRequest::new(vec![FrameId::new( + "docs", + "frm_getting_started", + Some("sha256:getting-started-v1".into()), + )]); + let line = serde_json::to_string(&Envelope::Verify { request }).unwrap(); + for body_field in ["\"content\"", "\"title\"", "\"provenance\"", "\"score\""] { + assert!( + !line.contains(body_field), + "a verify envelope must not carry {body_field}: {line}" + ); + } +} + +/// Drive the real stdio fixture end-to-end: a frame it just served verifies +/// `valid`, a mutated digest verifies `stale` and is evicted by the host, and +/// a frame it does not serve at all comes back `gone`. +#[tokio::test] +async fn a_real_stdio_provider_answers_a_verify_exchange_honestly() { + let mut host = Host::new(); + host.add_stdio("docs", &fixture(), &[]) + .await + .expect("fixture handshakes"); + + let served = FrameId::new( + "docs", + "frm_getting_started", + Some("sha256:getting-started-v1".into()), + ); + let mutated = FrameId::new( + "docs", + "frm_configuration", + Some("sha256:not-what-i-serve".into()), + ); + let absent = FrameId::new("docs", "frm_never_existed", Some("sha256:whatever".into())); + + let outcome = host + .verify_frames(&[served.clone(), mutated.clone(), absent.clone()]) + .await; + + assert_eq!( + outcome.retained, + vec![served], + "an unchanged frame must survive revalidation" + ); + // The host demonstrably drops the frame verified stale. + assert!(outcome.was_dropped(&mutated)); + assert_eq!( + outcome.drop_reason(&mutated), + Some(&DropReason::Stale { + replacement_digest: Some("sha256:configuration-v1".into()) + }), + "a mutated digest must come back stale, carrying the current digest" + ); + assert_eq!(outcome.drop_reason(&absent), Some(&DropReason::Gone)); + + // Only the stale frame is worth re-querying; the gone one is not there. + let requery: Vec<&FrameId> = outcome.requery().collect(); + assert_eq!(requery, vec![&mutated]); + + let _ = host.shutdown().await; +} + +/// The capability gate: pointed at a provider that does not advertise +/// `verify`, the host must fall back to re-query rather than trust the frames. +#[tokio::test] +async fn the_default_provider_impl_vouches_for_nothing() { + // The trait default answers `unknown` for everything, so a provider that + // implements nothing can never accidentally bless a stale frame. + use contextgraph_types::{VerifyRequest, VerifyResponse}; + let request = VerifyRequest::new(vec![FrameId::new("p", "f", Some("sha256:a".into()))]); + let response = VerifyResponse::uniform(&request, Verdict::Unknown); + assert_eq!( + response.verdict_for(&request.frames[0]), + Some(&Verdict::Unknown) + ); + assert!(!Verdict::Unknown.permits_reuse()); +} diff --git a/contextgraph-host/src/host.rs b/contextgraph-host/src/host.rs index 97f775b..0ad829c 100644 --- a/contextgraph-host/src/host.rs +++ b/contextgraph-host/src/host.rs @@ -11,11 +11,12 @@ //! dropped for a budget lie, or crashing mid-query never poisons the others //! (task deliverable 5). +use std::collections::HashMap; use std::time::Duration; use contextgraph_types::{ - ConsentReceipt, ContextFrame, ContextQuery, ContextQueryResult, DataFlow, EgressScope, - ProviderUsage, ServedFrame, UsageReport, + ContextFrame, ContextQuery, ContextQueryResult, DataFlow, FrameId, ProviderUsage, ServedFrame, + UsageReport, Verdict, VerifyRequest, }; use crate::consent::{ConsentDecision, ConsentRecord, ConsentStore}; @@ -131,6 +132,117 @@ impl Host { self.providers.is_empty() } + /// Revalidate frames this host already holds, so unchanged context can be + /// reused without re-querying it (`docs/context-reuse.md` §4). + /// + /// Identities are grouped by provider and each capable provider is asked + /// once. The rule is **default-deny**: a frame is retained only on an + /// explicit [`Verdict::Valid`], and every other outcome — a negative + /// verdict, a missing digest, a provider that doesn't support verify, an + /// unregistered provider, a failed request — drops the frame with a reason + /// (requirement V2). Reasons that + /// [warrant a re-query](DropReason::warrants_requery) tell the host which + /// dropped frames are worth fetching again. + /// + /// This method holds **no state**: it neither caches frames nor tracks turn + /// boundaries. When to re-verify is the host's policy (§4 gives informative + /// guidance); the protocol's job is only to answer the question when asked. + /// No frame body travels in either direction. + pub async fn verify_frames(&self, held: &[FrameId]) -> VerifyOutcome { + use futures_util::future::join_all; + + // Group by provider, preserving first-seen provider order so the + // outcome is deterministic for a given input. + let mut order: Vec<&str> = Vec::new(); + let mut grouped: HashMap<&str, Vec> = HashMap::new(); + for frame in held { + let id = frame.provider_id.as_str(); + if !grouped.contains_key(id) { + order.push(id); + } + grouped.entry(id).or_default().push(frame.clone()); + } + + let legs = order.into_iter().map(|provider_id| { + let frames = grouped.remove(provider_id).unwrap_or_default(); + self.verify_one_provider(provider_id, frames) + }); + + let mut outcome = VerifyOutcome::default(); + for leg in join_all(legs).await { + outcome.retained.extend(leg.retained); + outcome.dropped.extend(leg.dropped); + } + outcome + } + + /// Verify one provider's slice of the held set, converting every failure + /// mode into dropped frames rather than a propagated error — one provider's + /// verify failure never affects another's. + async fn verify_one_provider(&self, provider_id: &str, frames: Vec) -> VerifyOutcome { + let mut outcome = VerifyOutcome::default(); + + let Some(provider) = self.provider(provider_id) else { + outcome.drop_all(frames, DropReason::UnknownProvider); + return outcome; + }; + if !provider.capabilities().verify { + // The declared fallback: a provider that can't verify gets its + // frames re-queried rather than trusted (§4, requirement V3). + outcome.drop_all(frames, DropReason::VerifyUnsupported); + return outcome; + } + + // A frame with no digest can't be revalidated — §1's D4 makes that a + // re-query, not a reuse. Filter before asking, so the request only + // carries answerable identities. + let (verifiable, undigested): (Vec, Vec) = + frames.into_iter().partition(FrameId::is_verifiable); + outcome.drop_all(undigested, DropReason::NoDigest); + if verifiable.is_empty() { + return outcome; + } + + let request = VerifyRequest::new(verifiable.clone()); + let response = match tokio::time::timeout( + self.per_provider_timeout, + provider.verify(&request), + ) + .await + { + Ok(Ok(response)) => response, + Ok(Err(error)) => { + outcome.drop_all(verifiable, DropReason::VerifyFailed(error.to_string())); + return outcome; + } + Err(_) => { + let error = HostError::Timeout { + id: provider_id.to_string(), + timeout_ms: self.per_provider_timeout.as_millis() as u64, + }; + outcome.drop_all(verifiable, DropReason::VerifyFailed(error.to_string())); + return outcome; + } + }; + + for frame in verifiable { + // Correlate by full identity, never by position. A provider that + // omits an answer gets `Unknown` — silence is not validity. + match response.verdict_for(&frame) { + Some(Verdict::Valid) => outcome.retained.push(frame), + Some(Verdict::Stale { replacement_digest }) => outcome.drop_one( + frame, + DropReason::Stale { + replacement_digest: replacement_digest.clone(), + }, + ), + Some(Verdict::Gone) => outcome.drop_one(frame, DropReason::Gone), + Some(Verdict::Unknown) | None => outcome.drop_one(frame, DropReason::Unknown), + } + } + outcome + } + /// Query a single provider by id, honoring the consent gate and the /// per-provider timeout. Querying an unconsented egress provider is /// [`HostError::ConsentRequired`] (legacy boolean) or @@ -453,6 +565,102 @@ pub enum ProviderResult { Failed(HostError), } +/// Why a held frame was dropped by [`Host::verify_frames`] +/// (`docs/context-reuse.md` §4). +/// +/// The first three mirror the provider's [`Verdict`]s; the rest are host-side +/// reasons a frame could not be revalidated at all. Either way the frame leaves +/// the composed context — the difference matters only for deciding whether to +/// re-query it. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum DropReason { + /// The provider answered `stale`: the frame exists but its content changed. + /// Carries the provider's current digest when it offered one. + Stale { replacement_digest: Option }, + /// The provider answered `gone` — the frame no longer exists. + Gone, + /// The provider answered `unknown`, or returned no verdict for the frame at + /// all. Silence is not validity. + Unknown, + /// The frame carries no `content_digest`, so it cannot be revalidated + /// (§1, requirement D4). + NoDigest, + /// The provider does not advertise the `verify` capability, so the host + /// falls back to re-querying its frames (§4, requirement V3). + VerifyUnsupported, + /// No provider with this frame's `provider_id` is registered with the host. + UnknownProvider, + /// The verify request itself failed — a transport error or a timeout. + VerifyFailed(String), +} + +impl DropReason { + /// Whether re-querying the provider could recover usable content for this + /// frame. False only for [`Gone`](Self::Gone) — every other reason means + /// the host simply doesn't have a trustworthy copy and should ask again. + pub fn warrants_requery(&self) -> bool { + !matches!(self, Self::Gone) + } +} + +/// One dropped frame and why (`docs/context-reuse.md` §4). +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct DroppedFrame { + /// The identity that was dropped. + pub frame: FrameId, + /// Why it was dropped. + pub reason: DropReason, +} + +/// The result of revalidating a held frame set (`docs/context-reuse.md` §4). +/// +/// Partitions the input into frames the host may keep reusing and frames it +/// must drop. The partition is **total and default-deny**: every input identity +/// appears in exactly one of the two lists, and it lands in `retained` only on +/// an explicit `valid`. +#[derive(Debug, Clone, Default, PartialEq, Eq)] +pub struct VerifyOutcome { + /// Frames that verified `valid` — safe to keep reusing, and the frames + /// whose byte-stable reuse §1's canonical ordering was built to protect. + pub retained: Vec, + /// Frames that must leave the composed context, each with its reason. + pub dropped: Vec, +} + +impl VerifyOutcome { + /// The dropped frames worth re-querying — everything except `gone`, which + /// is not there to re-fetch. + pub fn requery(&self) -> impl Iterator { + self.dropped + .iter() + .filter(|dropped| dropped.reason.warrants_requery()) + .map(|dropped| &dropped.frame) + } + + /// Whether an identity was dropped. + pub fn was_dropped(&self, frame: &FrameId) -> bool { + self.dropped.iter().any(|dropped| &dropped.frame == frame) + } + + /// The reason an identity was dropped, if it was. + pub fn drop_reason(&self, frame: &FrameId) -> Option<&DropReason> { + self.dropped + .iter() + .find(|dropped| &dropped.frame == frame) + .map(|dropped| &dropped.reason) + } + + fn drop_one(&mut self, frame: FrameId, reason: DropReason) { + self.dropped.push(DroppedFrame { frame, reason }); + } + + fn drop_all(&mut self, frames: impl IntoIterator, reason: DropReason) { + for frame in frames { + self.drop_one(frame, reason.clone()); + } + } +} + #[cfg(test)] mod tests { use super::*; @@ -875,4 +1083,324 @@ mod tests { Err(HostError::UnknownProvider(_)) )); } + + // ---- context/verify (§4) ---- + + use contextgraph_types::{FrameVerdict, VerifyResponse}; + use std::collections::HashMap as StdHashMap; + + /// A provider that answers `context/verify` from a scripted verdict table. + struct VerifyingProvider { + id: String, + capabilities: Capabilities, + /// frame id -> verdict. A frame absent from the table gets no verdict + /// entry at all, exercising the "silence is not validity" path. + verdicts: StdHashMap, + /// When set, `verify` fails instead of answering. + verify_error: Option, + /// Identities this provider was actually asked about. + asked: Arc>>, + } + + impl VerifyingProvider { + fn new(id: &str, supports_verify: bool, verdicts: &[(&str, Verdict)]) -> Self { + Self { + id: id.into(), + capabilities: Capabilities { + query: QueryCapability { + kinds: vec!["doc".into()], + filters: vec![], + }, + verify: supports_verify, + ..Capabilities::default() + }, + verdicts: verdicts + .iter() + .map(|(f, v)| ((*f).to_string(), v.clone())) + .collect(), + verify_error: None, + asked: Arc::new(std::sync::Mutex::new(Vec::new())), + } + } + + fn failing(id: &str) -> Self { + let mut provider = Self::new(id, true, &[]); + provider.verify_error = Some("index unavailable".into()); + provider + } + } + + #[async_trait] + impl ContextProvider for VerifyingProvider { + fn id(&self) -> &str { + &self.id + } + fn info(&self) -> &ProviderInfo { + // Local-only: nothing here is about consent. + static_info() + } + fn capabilities(&self) -> &Capabilities { + &self.capabilities + } + async fn query(&self, _query: &ContextQuery) -> Result { + Ok(ContextQueryResult { + frames: vec![], + truncated: false, + dropped_estimate: None, + }) + } + async fn verify(&self, request: &VerifyRequest) -> Result { + self.asked + .lock() + .unwrap() + .extend(request.frames.iter().cloned()); + if let Some(message) = &self.verify_error { + return Err(HostError::Provider { + id: self.id.clone(), + message: message.clone(), + }); + } + Ok(VerifyResponse::new( + request + .frames + .iter() + .filter_map(|frame| { + self.verdicts + .get(&frame.frame_id) + .map(|verdict| FrameVerdict::new(frame.clone(), verdict.clone())) + }) + .collect(), + )) + } + } + + fn static_info() -> &'static ProviderInfo { + use std::sync::OnceLock; + static INFO: OnceLock = OnceLock::new(); + INFO.get_or_init(|| ProviderInfo { + name: "verifier".into(), + version: "0.0.1".into(), + data_flow: DataFlow { + reads: true, + writes: false, + egress: false, + }, + }) + } + + fn held(provider: &str, frame: &str, digest: Option<&str>) -> FrameId { + FrameId::new(provider, frame, digest.map(String::from)) + } + + #[tokio::test] + async fn a_stale_frame_is_dropped_and_a_valid_one_is_retained() { + // The core §4 guarantee: the host demonstrably evicts a frame the + // provider says has changed, and keeps the one it vouches for. + let mut host = Host::new(); + host.register(Box::new(VerifyingProvider::new( + "docs", + true, + &[ + ("fresh", Verdict::Valid), + ( + "changed", + Verdict::Stale { + replacement_digest: Some("sha256:new".into()), + }, + ), + ], + ))); + + let fresh = held("docs", "fresh", Some("sha256:a")); + let changed = held("docs", "changed", Some("sha256:b")); + let outcome = host.verify_frames(&[fresh.clone(), changed.clone()]).await; + + assert_eq!(outcome.retained, vec![fresh]); + assert!(outcome.was_dropped(&changed)); + assert_eq!( + outcome.drop_reason(&changed), + Some(&DropReason::Stale { + replacement_digest: Some("sha256:new".into()) + }) + ); + // A stale frame is worth re-fetching; the replacement digest tells the + // host what it would be getting. + assert_eq!(outcome.requery().collect::>(), vec![&changed]); + } + + #[tokio::test] + async fn a_gone_frame_is_dropped_and_not_worth_re_querying() { + let mut host = Host::new(); + host.register(Box::new(VerifyingProvider::new( + "docs", + true, + &[("deleted", Verdict::Gone)], + ))); + let deleted = held("docs", "deleted", Some("sha256:a")); + let outcome = host.verify_frames(std::slice::from_ref(&deleted)).await; + + assert!(outcome.retained.is_empty()); + assert_eq!(outcome.drop_reason(&deleted), Some(&DropReason::Gone)); + // Nothing to re-fetch — `gone` is the one reason that doesn't warrant it. + assert_eq!(outcome.requery().count(), 0); + } + + #[tokio::test] + async fn an_unknown_verdict_and_a_missing_verdict_both_drop_the_frame() { + // Silence is not validity: a provider that omits an answer must not + // have that read as "still good". + let mut host = Host::new(); + host.register(Box::new(VerifyingProvider::new( + "docs", + true, + &[("shrugged", Verdict::Unknown)], + ))); + let shrugged = held("docs", "shrugged", Some("sha256:a")); + let unanswered = held("docs", "never-mentioned", Some("sha256:b")); + let outcome = host + .verify_frames(&[shrugged.clone(), unanswered.clone()]) + .await; + + assert!(outcome.retained.is_empty()); + assert_eq!(outcome.drop_reason(&shrugged), Some(&DropReason::Unknown)); + assert_eq!(outcome.drop_reason(&unanswered), Some(&DropReason::Unknown)); + assert_eq!(outcome.requery().count(), 2); + } + + #[tokio::test] + async fn a_provider_without_verify_support_is_never_asked_and_falls_back_to_requery() { + // The capability gate (V3): the host doesn't send a verify request at + // all, it just re-queries. + let mut host = Host::new(); + let provider = VerifyingProvider::new("docs", false, &[("anything", Verdict::Valid)]); + let asked = provider.asked.clone(); + host.register(Box::new(provider)); + + let frame = held("docs", "anything", Some("sha256:a")); + let outcome = host.verify_frames(std::slice::from_ref(&frame)).await; + + assert!(asked.lock().unwrap().is_empty(), "must not be asked"); + assert!(outcome.retained.is_empty()); + assert_eq!( + outcome.drop_reason(&frame), + Some(&DropReason::VerifyUnsupported) + ); + assert_eq!(outcome.requery().count(), 1); + } + + #[tokio::test] + async fn a_frame_without_a_digest_is_unverifiable_and_never_sent() { + // §1 D4: no digest, no revalidation — and the request only carries + // answerable identities. + let mut host = Host::new(); + let provider = VerifyingProvider::new("docs", true, &[("bare", Verdict::Valid)]); + let asked = provider.asked.clone(); + host.register(Box::new(provider)); + + let bare = held("docs", "bare", None); + let digested = held("docs", "digested", Some("sha256:a")); + let outcome = host.verify_frames(&[bare.clone(), digested.clone()]).await; + + assert_eq!(outcome.drop_reason(&bare), Some(&DropReason::NoDigest)); + let asked = asked.lock().unwrap().clone(); + assert_eq!( + asked, + vec![digested], + "only verifiable identities go on the wire" + ); + } + + #[tokio::test] + async fn a_failed_verify_drops_that_providers_frames_without_touching_another() { + // Per-provider isolation, same contract as a query fan-out leg. + let mut host = Host::new(); + host.register(Box::new(VerifyingProvider::failing("broken"))); + host.register(Box::new(VerifyingProvider::new( + "healthy", + true, + &[("good", Verdict::Valid)], + ))); + + let broken = held("broken", "any", Some("sha256:a")); + let good = held("healthy", "good", Some("sha256:b")); + let outcome = host.verify_frames(&[broken.clone(), good.clone()]).await; + + assert_eq!(outcome.retained, vec![good], "one failure must not poison"); + assert!(matches!( + outcome.drop_reason(&broken), + Some(DropReason::VerifyFailed(_)) + )); + assert!(outcome.requery().any(|frame| frame == &broken)); + } + + #[tokio::test] + async fn frames_from_an_unregistered_provider_are_dropped_not_ignored() { + let host = Host::new(); + let orphan = held("never-registered", "f", Some("sha256:a")); + let outcome = host.verify_frames(std::slice::from_ref(&orphan)).await; + assert!(outcome.retained.is_empty()); + assert_eq!( + outcome.drop_reason(&orphan), + Some(&DropReason::UnknownProvider) + ); + } + + #[tokio::test] + async fn held_frames_are_grouped_into_one_request_per_provider() { + // Verification costs bytes, not tokens — so it must not cost a round + // trip per frame either. + let mut host = Host::new(); + let docs = VerifyingProvider::new( + "docs", + true, + &[("a", Verdict::Valid), ("b", Verdict::Valid)], + ); + let asked = docs.asked.clone(); + host.register(Box::new(docs)); + + let outcome = host + .verify_frames(&[ + held("docs", "a", Some("sha256:1")), + held("docs", "b", Some("sha256:2")), + ]) + .await; + assert_eq!(outcome.retained.len(), 2); + // Both identities arrived together in a single verify call. + assert_eq!(asked.lock().unwrap().len(), 2); + } + + #[tokio::test] + async fn the_partition_is_total_so_every_held_frame_is_accounted_for() { + let mut host = Host::new(); + host.register(Box::new(VerifyingProvider::new( + "docs", + true, + &[("keep", Verdict::Valid), ("drop", Verdict::Gone)], + ))); + let input = vec![ + held("docs", "keep", Some("sha256:1")), + held("docs", "drop", Some("sha256:2")), + held("docs", "nodigest", None), + held("elsewhere", "orphan", Some("sha256:3")), + ]; + let outcome = host.verify_frames(&input).await; + assert_eq!( + outcome.retained.len() + outcome.dropped.len(), + input.len(), + "every held identity must land in exactly one bucket" + ); + for frame in &input { + assert!( + outcome.retained.contains(frame) || outcome.was_dropped(frame), + "{frame:?} was silently lost" + ); + } + } + + #[tokio::test] + async fn verifying_an_empty_held_set_is_a_no_op() { + let host = Host::new(); + let outcome = host.verify_frames(&[]).await; + assert_eq!(outcome, VerifyOutcome::default()); + } } diff --git a/contextgraph-host/src/http.rs b/contextgraph-host/src/http.rs index 003d6db..70070d1 100644 --- a/contextgraph-host/src/http.rs +++ b/contextgraph-host/src/http.rs @@ -13,7 +13,8 @@ use std::time::Duration; use async_trait::async_trait; use contextgraph_types::{ - Capabilities, ContextQuery, ContextQueryResult, PROTOCOL_VERSION, ProviderInfo, + Capabilities, ContextQuery, ContextQueryResult, PROTOCOL_VERSION, ProviderInfo, VerifyRequest, + VerifyResponse, }; use crate::error::HostError; @@ -170,6 +171,30 @@ impl ContextProvider for HttpProvider { } } + async fn verify(&self, request: &VerifyRequest) -> Result { + let reply = post_envelope( + &self.client, + &self.url, + &Envelope::Verify { + request: request.clone(), + }, + &self.id, + ) + .await?; + match reply { + Envelope::Verified { response } => Ok(response), + Envelope::Error { message } => Err(HostError::Provider { + id: self.id.clone(), + message, + }), + other => Err(HostError::UnexpectedEnvelope { + id: self.id.clone(), + expected: "verified".into(), + got: envelope_kind(&other).into(), + }), + } + } + async fn shutdown(&self) -> Result<(), HostError> { // Best-effort teardown notice; a remote endpoint is not ours to reap. let _ = post_envelope(&self.client, &self.url, &Envelope::Shutdown, &self.id).await; diff --git a/contextgraph-host/src/lib.rs b/contextgraph-host/src/lib.rs index a2f567e..d28085e 100644 --- a/contextgraph-host/src/lib.rs +++ b/contextgraph-host/src/lib.rs @@ -61,7 +61,9 @@ pub mod wire; pub use compose::compose_context; pub use consent::{ConsentDecision, ConsentRecord, ConsentStore}; pub use error::HostError; -pub use host::{FanOut, Host, ProviderOutcome, ProviderResult}; +pub use host::{ + DropReason, DroppedFrame, FanOut, Host, ProviderOutcome, ProviderResult, VerifyOutcome, +}; pub use http::HttpProvider; pub use provider::{ContextProvider, capability_matches, frame_kind_name}; pub use stdio::{RawStdioConnection, StdioProvider}; diff --git a/contextgraph-host/src/provider.rs b/contextgraph-host/src/provider.rs index 9d444d4..029e05d 100644 --- a/contextgraph-host/src/provider.rs +++ b/contextgraph-host/src/provider.rs @@ -10,7 +10,10 @@ //! support"). use async_trait::async_trait; -use contextgraph_types::{Capabilities, ContextQuery, ContextQueryResult, FrameKind, ProviderInfo}; +use contextgraph_types::{ + Capabilities, ContextQuery, ContextQueryResult, FrameKind, ProviderInfo, Verdict, + VerifyRequest, VerifyResponse, +}; use crate::error::HostError; @@ -39,6 +42,19 @@ pub trait ContextProvider: Send + Sync { /// trusted (`crate::host`). async fn query(&self, query: &ContextQuery) -> Result; + /// Revalidate frames the host already holds, without any frame body + /// travelling (`docs/context-reuse.md` §4 `context/verify`). + /// + /// Defaults to answering [`Verdict::Unknown`](contextgraph_types::Verdict::Unknown) + /// for every requested identity, so an existing provider implements + /// nothing and is simply treated as unable to vouch for its frames — the + /// host then re-queries them. A provider that overrides this **MUST** also + /// advertise [`Capabilities::verify`](contextgraph_types::Capabilities::verify), + /// since the host only asks providers that declare support. + async fn verify(&self, request: &VerifyRequest) -> Result { + Ok(VerifyResponse::uniform(request, Verdict::Unknown)) + } + /// Shut the provider down cleanly (§3.2 lifecycle). In-process providers /// default to a no-op; transport-backed providers send `shutdown` and /// reap their child. Overridable. diff --git a/contextgraph-host/src/stdio.rs b/contextgraph-host/src/stdio.rs index 5e83ed4..1666b34 100644 --- a/contextgraph-host/src/stdio.rs +++ b/contextgraph-host/src/stdio.rs @@ -27,7 +27,8 @@ use std::time::Duration; use async_trait::async_trait; use contextgraph_types::{ - Capabilities, ContextQuery, ContextQueryResult, PROTOCOL_VERSION, ProviderInfo, + Capabilities, ContextQuery, ContextQueryResult, PROTOCOL_VERSION, ProviderInfo, VerifyRequest, + VerifyResponse, }; use tokio::io::{AsyncBufReadExt, AsyncWriteExt, BufReader}; use tokio::process::{Child, ChildStdin, ChildStdout, Command}; @@ -382,6 +383,26 @@ impl ContextProvider for StdioProvider { } } + async fn verify(&self, request: &VerifyRequest) -> Result { + let mut conn = self.conn.lock().await; + conn.send(&Envelope::Verify { + request: request.clone(), + }) + .await?; + match conn.recv().await? { + Envelope::Verified { response } => Ok(response), + Envelope::Error { message } => Err(HostError::Provider { + id: self.id.clone(), + message, + }), + other => Err(HostError::UnexpectedEnvelope { + id: self.id.clone(), + expected: "verified".into(), + got: envelope_kind(&other).into(), + }), + } + } + async fn shutdown(&self) -> Result<(), HostError> { let mut conn = self.conn.lock().await; conn.shutdown().await diff --git a/contextgraph-host/src/wire.rs b/contextgraph-host/src/wire.rs index a1df2a0..e02caeb 100644 --- a/contextgraph-host/src/wire.rs +++ b/contextgraph-host/src/wire.rs @@ -18,7 +18,9 @@ //! protocol family up front (§3.2), and a mismatch is a named error, never a //! hang. -use contextgraph_types::{Capabilities, ContextQuery, ContextQueryResult, ProviderInfo}; +use contextgraph_types::{ + Capabilities, ContextQuery, ContextQueryResult, ProviderInfo, VerifyRequest, VerifyResponse, +}; use serde::{Deserialize, Serialize}; use crate::error::HostError; @@ -44,6 +46,13 @@ pub enum Envelope { Query { query: ContextQuery }, /// Provider → host budgeted, provenance-carrying frames (§3.3 response). Frames { result: ContextQueryResult }, + /// Host → provider revalidation request: are these held frames still + /// valid (`docs/context-reuse.md` §4 `context/verify`)? Carries frame + /// identities only — never bodies. Capability-gated: a host sends it only + /// to a provider advertising [`Capabilities::verify`](contextgraph_types::Capabilities::verify). + Verify { request: VerifyRequest }, + /// Provider → host per-frame verdicts (§4 response). + Verified { response: VerifyResponse }, /// Lifecycle teardown; the provider should exit cleanly (§3.2). Shutdown, /// Provider-reported failure — lets a provider report a bad request @@ -60,6 +69,8 @@ pub fn envelope_kind(env: &Envelope) -> &'static str { Envelope::HandshakeAck { .. } => "handshake_ack", Envelope::Query { .. } => "query", Envelope::Frames { .. } => "frames", + Envelope::Verify { .. } => "verify", + Envelope::Verified { .. } => "verified", Envelope::Shutdown => "shutdown", Envelope::Error { .. } => "error", } diff --git a/contextgraph-types/src/capability.rs b/contextgraph-types/src/capability.rs index 064e451..9db5d59 100644 --- a/contextgraph-types/src/capability.rs +++ b/contextgraph-types/src/capability.rs @@ -81,6 +81,16 @@ pub struct Capabilities { pub embeddings_fingerprint: Option, #[serde(default)] pub subscribe: bool, + /// Whether this provider answers `context/verify` — pull-based + /// revalidation of frames a host already holds (`docs/context-reuse.md` + /// §4). Defaults to `false`, so a provider that says nothing is treated as + /// not supporting it and the host falls back to re-querying. + /// + /// Independent of [`subscribe`](Self::subscribe): push and pull freshness + /// are complementary, and a provider may advertise either, both, or + /// neither. + #[serde(default)] + pub verify: bool, } #[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize)] @@ -96,6 +106,27 @@ mod tests { use super::*; use crate::scope::EgressScope; + #[test] + fn verify_support_defaults_off_and_is_independent_of_subscribe() { + // A provider that says nothing must not be assumed to verify — the + // host falls back to re-querying (§4). + let caps = Capabilities::default(); + assert!(!caps.verify); + assert!(!caps.subscribe); + + // Absent from the wire ⇒ still false, so pre-§4 providers keep working. + let back: Capabilities = serde_json::from_str(r#"{"upsert":false}"#).unwrap(); + assert!(!back.verify); + + // Push and pull freshness are independent axes. + let pull_only: Capabilities = + serde_json::from_str(r#"{"verify":true,"subscribe":false}"#).unwrap(); + assert!(pull_only.verify && !pull_only.subscribe); + let push_only: Capabilities = + serde_json::from_str(r#"{"verify":false,"subscribe":true}"#).unwrap(); + assert!(!push_only.verify && push_only.subscribe); + } + #[test] fn egress_provider_data_flow_roundtrips() { let flow = DataFlow { @@ -191,6 +222,7 @@ mod tests { graph: false, embeddings_fingerprint: Some("bge-small-v1".into()), subscribe: false, + verify: true, }; let json = serde_json::to_string(&caps).unwrap(); let back: Capabilities = serde_json::from_str(&json).unwrap(); diff --git a/contextgraph-types/src/lib.rs b/contextgraph-types/src/lib.rs index 3649612..a43cfd0 100644 --- a/contextgraph-types/src/lib.rs +++ b/contextgraph-types/src/lib.rs @@ -15,6 +15,7 @@ pub mod identity; pub mod query; pub mod scope; pub mod usage; +pub mod verify; pub use capability::{Capabilities, DataFlow, ProviderInfo}; pub use consent::{ConsentReceipt, Grantor}; @@ -23,6 +24,7 @@ pub use identity::{FrameId, canonical_order}; pub use query::{ContextQuery, ContextQueryResult}; pub use scope::EgressScope; pub use usage::{ProviderUsage, ServedFrame, UsageReport}; +pub use verify::{FrameVerdict, Verdict, VerifyRequest, VerifyResponse}; /// The protocol version string this crate implements. Frozen to `contextgraph/1.0` /// only at the public v1.0 release (`06-context-protocol.md` §3). diff --git a/contextgraph-types/src/verify.rs b/contextgraph-types/src/verify.rs new file mode 100644 index 0000000..bbc1e25 --- /dev/null +++ b/contextgraph-types/src/verify.rs @@ -0,0 +1,376 @@ +//! `context/verify` — pull-based revalidation of frames a host already holds +//! (`docs/context-reuse.md` §4). +//! +//! Deterministic composition (§1) makes reusing context *cheap*: an unchanged +//! frame set renders byte-identically and rides the provider's prompt cache. +//! But cheap reuse is only safe if the frames are still **true**. Without a way +//! to ask, a host faces a bad pair of choices at every turn boundary: re-query +//! everything (paying tokens and latency, and destroying the very prefix +//! stability §1 bought) or reuse silently and risk citing evidence that changed +//! underneath it. +//! +//! A verify exchange is the cheap third option. The host sends a batch of +//! [`FrameId`]s — `(provider id, frame id, content digest)` — and the provider +//! answers one [`Verdict`] per frame. **No frame body travels in either +//! direction.** That is the whole economic point: verification costs *bytes*, +//! not *tokens*, so a host can afford to do it every turn on frames it would +//! otherwise have re-queried in full. +//! +//! The digest is the ground truth. A provider compares the digest the host +//! presents against the digest its source has *now*: equal ⇒ [`Valid`](Verdict::Valid), +//! different ⇒ [`Stale`](Verdict::Stale), source gone ⇒ [`Gone`](Verdict::Gone), +//! can't tell ⇒ [`Unknown`](Verdict::Unknown). Because the digest is +//! provider-declared and opaque (§1), the provider is the only party that can +//! answer — which is exactly why §4's conformance case exists to hold it honest. +//! +//! **Verify is the pull counterpart to subscribe (#6), not a replacement.** A +//! provider that can watch its sources pushes invalidations; one that cannot — +//! a stateless HTTP endpoint, a batch-rebuilt index — can still answer a +//! question asked of it. Both are capability-gated, and a host that has neither +//! falls back to re-querying. + +use serde::{Deserialize, Serialize}; + +use crate::identity::FrameId; + +/// A host's request to revalidate frames it already holds +/// (`docs/context-reuse.md` §4). +/// +/// Carries identities only — never frame bodies. Every identity in one request +/// belongs to the provider it is sent to; a host holding frames from several +/// providers sends one request each. +#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)] +pub struct VerifyRequest { + /// The frame identities to revalidate. A host **SHOULD** only include + /// identities that carry a digest ([`FrameId::is_verifiable`]) — a + /// digest-less frame cannot be revalidated and is re-queried instead. + #[serde(default)] + pub frames: Vec, +} + +impl VerifyRequest { + /// A request for the given identities. + pub fn new(frames: Vec) -> Self { + Self { frames } + } + + /// How many identities this request asks about. + pub fn len(&self) -> usize { + self.frames.len() + } + + pub fn is_empty(&self) -> bool { + self.frames.is_empty() + } +} + +/// A provider's answer for one frame (`docs/context-reuse.md` §4). +/// +/// Serializes as an internally-tagged object keyed on `status`, so a verdict is +/// self-describing on the wire and gains variants without breaking parsers: +/// `{"status": "valid"}`, `{"status": "stale", "replacement_digest": "sha256:…"}`, +/// `{"status": "gone"}`, `{"status": "unknown"}`. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(tag = "status", rename_all = "snake_case")] +pub enum Verdict { + /// The frame's content is unchanged: the digest the host presented matches + /// the provider's current digest for that frame. The host **MAY** keep + /// reusing the body it already holds. + Valid, + /// The frame still exists but its content changed — the presented digest no + /// longer matches. The host **MUST NOT** keep serving the body it holds. + /// + /// `replacement_digest` is the provider's *current* digest for the frame, + /// offered so a host can tell "changed again since I last looked" from + /// "changed to something I already fetched" without a round trip. It is + /// optional: a provider that knows the content differs but not what it is + /// now still answers `stale` honestly. **It is a digest, never a body** — + /// the host re-queries if it wants the new content. + Stale { + #[serde(default, skip_serializing_if = "Option::is_none")] + replacement_digest: Option, + }, + /// The frame no longer exists — the source was deleted, or the provider no + /// longer serves it. The host **MUST** drop it, and re-querying *this + /// identity* is pointless. + Gone, + /// The provider cannot say. It may never have served this frame, may have + /// lost the history needed to compare digests (a rebuilt index), or may not + /// recognize the identity. The host **MUST NOT** treat this as validity — + /// an unverifiable frame is re-queried, never reused on a shrug. + Unknown, +} + +impl Verdict { + /// Whether a host may keep reusing the frame body it already holds. **Only** + /// [`Valid`](Self::Valid) — reuse requires a positive answer, never the + /// absence of a negative one (`docs/context-reuse.md` §4, requirement V2). + pub fn permits_reuse(&self) -> bool { + matches!(self, Self::Valid) + } + + /// Whether re-querying the provider could recover usable content. True for + /// [`Stale`](Self::Stale) (content exists, it changed) and + /// [`Unknown`](Self::Unknown) (the provider couldn't say, so ask properly). + /// False for [`Gone`](Self::Gone) — the frame is not there to re-fetch — and + /// for [`Valid`](Self::Valid), which needs no re-query at all. + pub fn warrants_requery(&self) -> bool { + matches!(self, Self::Stale { .. } | Self::Unknown) + } + + /// The verdict's wire name, for reports and log lines. + pub fn status(&self) -> &'static str { + match self { + Self::Valid => "valid", + Self::Stale { .. } => "stale", + Self::Gone => "gone", + Self::Unknown => "unknown", + } + } +} + +/// One frame's verdict, paired with the identity it answers +/// (`docs/context-reuse.md` §4). +/// +/// The identity is **echoed in full** rather than implied by position: a host +/// correlates verdicts by matching `frame`, so a provider that reorders, +/// omits, or duplicates entries can't silently shift a `valid` onto the wrong +/// frame. Entries a host didn't ask about are ignored; identities that come +/// back with no entry are treated as [`Unknown`](Verdict::Unknown). +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct FrameVerdict { + /// The identity being answered — echoed from the request. + pub frame: FrameId, + /// The provider's answer for it. + #[serde(flatten)] + pub verdict: Verdict, +} + +impl FrameVerdict { + /// Pair an identity with its verdict. + pub fn new(frame: FrameId, verdict: Verdict) -> Self { + Self { frame, verdict } + } +} + +/// A provider's answer to a [`VerifyRequest`] (`docs/context-reuse.md` §4). +#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)] +pub struct VerifyResponse { + /// One verdict per frame the provider is answering for. + #[serde(default)] + pub verdicts: Vec, +} + +impl VerifyResponse { + /// A response carrying the given verdicts. + pub fn new(verdicts: Vec) -> Self { + Self { verdicts } + } + + /// Answer every requested identity with the same verdict — the shape a + /// provider without real verification support returns + /// ([`Unknown`](Verdict::Unknown)), and a convenience for tests. + pub fn uniform(request: &VerifyRequest, verdict: Verdict) -> Self { + Self { + verdicts: request + .frames + .iter() + .map(|frame| FrameVerdict::new(frame.clone(), verdict.clone())) + .collect(), + } + } + + /// The verdict for one identity, matched on the **full** identity rather + /// than position. `None` when the provider returned no entry for it — which + /// a host treats as [`Unknown`](Verdict::Unknown). + pub fn verdict_for(&self, frame: &FrameId) -> Option<&Verdict> { + self.verdicts + .iter() + .find(|entry| &entry.frame == frame) + .map(|entry| &entry.verdict) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn id(frame: &str, digest: Option<&str>) -> FrameId { + FrameId::new("repo-graph", frame, digest.map(String::from)) + } + + #[test] + fn a_request_carries_identities_and_no_frame_bodies() { + let request = VerifyRequest::new(vec![ + id("retry-doc", Some("sha256:9f2c")), + id("timeout-doc", Some("sha256:aa01")), + ]); + let json = serde_json::to_string(&request).unwrap(); + // The economic guarantee of §4: verification costs bytes, not tokens. + assert!( + !json.contains("content\"") && !json.contains("title"), + "a verify request must never carry frame bodies: {json}" + ); + let back: VerifyRequest = serde_json::from_str(&json).unwrap(); + assert_eq!(back, request); + assert_eq!(back.len(), 2); + } + + #[test] + fn every_verdict_round_trips_through_its_tagged_wire_form() { + for (verdict, status) in [ + (Verdict::Valid, "valid"), + ( + Verdict::Stale { + replacement_digest: None, + }, + "stale", + ), + ( + Verdict::Stale { + replacement_digest: Some("sha256:beef".into()), + }, + "stale", + ), + (Verdict::Gone, "gone"), + (Verdict::Unknown, "unknown"), + ] { + let json = serde_json::to_string(&verdict).unwrap(); + assert!( + json.contains(&format!("\"status\":\"{status}\"")), + "verdict must be tagged on `status`: {json}" + ); + assert_eq!(verdict.status(), status); + let back: Verdict = serde_json::from_str(&json).unwrap(); + assert_eq!(back, verdict); + } + } + + #[test] + fn a_stale_verdict_without_a_replacement_omits_the_field() { + let json = serde_json::to_string(&Verdict::Stale { + replacement_digest: None, + }) + .unwrap(); + assert!( + !json.contains("replacement_digest"), + "an absent replacement must be omitted, not null: {json}" + ); + // A provider that knows content changed but not what it is now is still + // answering honestly. + let back: Verdict = serde_json::from_str(&json).unwrap(); + assert!(!back.permits_reuse()); + assert!(back.warrants_requery()); + } + + #[test] + fn a_stale_verdict_carries_a_replacement_digest_but_never_a_body() { + let verdict = Verdict::Stale { + replacement_digest: Some("sha256:beef".into()), + }; + let json = serde_json::to_string(&verdict).unwrap(); + assert!(json.contains("sha256:beef")); + let back: Verdict = serde_json::from_str(&json).unwrap(); + assert_eq!(back, verdict); + } + + #[test] + fn only_valid_permits_reuse() { + // The default-deny rule (V2): reuse needs a positive answer, never the + // mere absence of a negative one. + assert!(Verdict::Valid.permits_reuse()); + for verdict in [ + Verdict::Stale { + replacement_digest: None, + }, + Verdict::Gone, + Verdict::Unknown, + ] { + assert!( + !verdict.permits_reuse(), + "{} must not permit reuse", + verdict.status() + ); + } + } + + #[test] + fn gone_is_the_one_verdict_that_does_not_warrant_a_requery() { + // Nothing to re-fetch: the frame is not there anymore. + assert!(!Verdict::Gone.warrants_requery()); + assert!( + Verdict::Stale { + replacement_digest: None + } + .warrants_requery() + ); + assert!(Verdict::Unknown.warrants_requery()); + // A valid frame needs no re-query — that is the whole point. + assert!(!Verdict::Valid.warrants_requery()); + } + + #[test] + fn a_response_round_trips_and_flattens_the_verdict_into_each_entry() { + let response = VerifyResponse::new(vec![ + FrameVerdict::new(id("retry-doc", Some("sha256:9f2c")), Verdict::Valid), + FrameVerdict::new( + id("timeout-doc", Some("sha256:aa01")), + Verdict::Stale { + replacement_digest: Some("sha256:bb02".into()), + }, + ), + ]); + let json = serde_json::to_string(&response).unwrap(); + // `verdict` is flattened, so an entry reads as one flat object. + assert!( + !json.contains("\"verdict\""), + "verdict must flatten: {json}" + ); + let back: VerifyResponse = serde_json::from_str(&json).unwrap(); + assert_eq!(back, response); + } + + #[test] + fn a_verdict_is_correlated_by_full_identity_not_by_position() { + // A provider that reorders its answers must not shift a `valid` onto + // the wrong frame. + let valid_frame = id("retry-doc", Some("sha256:9f2c")); + let stale_frame = id("timeout-doc", Some("sha256:aa01")); + let response = VerifyResponse::new(vec![ + FrameVerdict::new(stale_frame.clone(), Verdict::Gone), + FrameVerdict::new(valid_frame.clone(), Verdict::Valid), + ]); + assert_eq!(response.verdict_for(&valid_frame), Some(&Verdict::Valid)); + assert_eq!(response.verdict_for(&stale_frame), Some(&Verdict::Gone)); + } + + #[test] + fn a_verdict_for_a_different_digest_does_not_match_the_asked_identity() { + // The digest is part of the identity: an answer about other bytes is an + // answer about a different question. + let asked = id("retry-doc", Some("sha256:9f2c")); + let response = VerifyResponse::new(vec![FrameVerdict::new( + id("retry-doc", Some("sha256:0000")), + Verdict::Valid, + )]); + assert_eq!( + response.verdict_for(&asked), + None, + "a verdict about a different digest must not answer for this one" + ); + } + + #[test] + fn a_uniform_response_answers_every_requested_identity() { + // The shape a provider without verification support returns. + let request = VerifyRequest::new(vec![ + id("retry-doc", Some("sha256:9f2c")), + id("timeout-doc", Some("sha256:aa01")), + ]); + let response = VerifyResponse::uniform(&request, Verdict::Unknown); + assert_eq!(response.verdicts.len(), 2); + for frame in &request.frames { + assert_eq!(response.verdict_for(frame), Some(&Verdict::Unknown)); + } + } +} diff --git a/docs/context-reuse.md b/docs/context-reuse.md index a22ff91..c6fadc1 100644 --- a/docs/context-reuse.md +++ b/docs/context-reuse.md @@ -263,105 +263,153 @@ auditable from the wire all the way up to the invoice line. --- -## 3. Consent scopes and receipts +## 4. Context verification ### The problem -Consent-gating is one of the seven guarantees: retrieval that transmits workspace content off-machine must be agreed to. `DataFlow.egress` already gates it — but a boolean answers the question only *in the moment it is asked*. When an auditor asks, months later, "**what** left the machine, **to whom**, and **who** agreed?", a `true` in a since-exited process is no answer. And without a shared vocabulary of egress classes, "consent" means something different to every provider. +§1 makes reusing context **cheap**: an unchanged frame set renders +byte-identically and rides the provider's prompt cache. But cheap reuse is only +safe if the frames are still **true**. A retrieved snippet is a claim about a +source, and sources move — a file is edited, a doc is rewritten, a record is +deleted. Reuse the frame anyway and the agent cites evidence that no longer +exists. -This section makes consent an **artifact** rather than an event, in two parts: a closed **scope vocabulary** that classes *where* content goes, and durable **consent receipts** that record each grant. +Today's only live mechanism for this is push invalidation ([#6](https://github.com/macanderson/context-graph-protocol/issues/6)'s +`subscribe`), which needs a long-lived channel and a provider able to watch its +sources. Plenty of providers can't: a stateless HTTP endpoint, a batch-rebuilt +index, a serverless function. And plenty of hosts don't want a subscription's +lifecycle just to answer a much simpler question at a turn boundary: *are the +frames I already hold still valid?* -### The egress-scope vocabulary +Without a way to ask, a host is left choosing between two bad options every +turn — re-query everything (paying tokens and latency, and destroying the very +prefix stability §1 bought) or reuse silently and risk citing stale evidence. -A provider declares — alongside the boolean `egress` — the [egress scopes](./protocol-surface.md#handshake--capability) its served content falls under, bound to Rust as -[`EgressScope`](https://docs.rs/contextgraph-types/latest/contextgraph_types/scope/enum.EgressScope.html) -in the new [`DataFlow.egress_scopes`](./protocol-surface.md#handshake--capability) field. Four **normative base classes** form the closed core: +### The exchange -| Scope | Wire string | Off-machine? | Meaning | -| ----- | ----------- | ------------ | ------- | -| `LocalOnly` | `local-only` | no | Nothing leaves the machine. | -| `OrgTenant` | `org-tenant` | yes | Leaves the machine, stays in the org's own infrastructure. | -| `ThirdPartyIndex` | `third-party-index` | yes | Content sent to an external index / embedding service. | -| `ThirdPartyModel` | `third-party-model` | yes | Content sent to an external model API. | - -The vocabulary is **extensible** by namespaced custom scopes — `EgressScope::Custom("vendor:scope-name")` — which **MUST** contain a `:` separator with non-empty sides, so a custom scope can never collide with or be mistaken for a base class. Everything other than `local-only` is treated as off-machine, including an unrecognized custom scope: the conservative default is that an unknown destination *leaves*, so a host never under-gates. - -**A scope is declared at the provider (data-flow) level, and it governs every frame that provider serves.** There is no per-frame scope: the serving provider's declaration *is* the egress class of each frame it returns. This keeps the consent gate — which fires once, before a query is transmitted — the single place egress is decided, rather than scattering a scope across every frame. - -The declaration must be **truthful**: an off-machine scope alongside `egress: false` is a contradiction (a provider claiming local posture while naming a destination that leaves), and a host holds a provider to this at the handshake -([`DataFlow::scopes_consistent`](https://docs.rs/contextgraph-types/latest/contextgraph_types/struct.DataFlow.html#method.scopes_consistent), -requirement C5). - -### Consent receipts - -When a host grants consent, it records a -[`ConsentReceipt`](https://docs.rs/contextgraph-types/latest/contextgraph_types/consent/struct.ConsentReceipt.html): +`context/verify` is the cheap third option. The host sends the +[frame identities](#frame-identity) it holds; the provider answers one verdict +each. ```rust -pub struct ConsentReceipt { - pub provider_id: String, // the provider this authorizes - pub scope: EgressScope, // the exact egress class consented to - pub provider_name: String, // provider identity, pinned at grant time - pub provider_version: String, - pub grantor: Grantor, // Human(id) | Policy(id) — who agreed - pub granted_at: String, // RFC 3339 - pub expires_at: Option, // RFC 3339, if consent lapses +pub struct VerifyRequest { + pub frames: Vec, // identities only — never bodies } -``` - -A receipt turns "is this allowed?" (a boolean, now) into "what left, to whom, who agreed, and when?" (a durable record). It pins the provider's identity **at grant time**, so a later rename can't retroactively rewrite what was agreed. Receipts live in an **append-only** ledger -([`ConsentStore::record_receipt`](https://docs.rs/contextgraph-host/latest/contextgraph_host/consent/struct.ConsentStore.html#method.record_receipt)): -a new grant never edits or erases an old one, so the history of consent *is* the audit trail, and it is serde-able for durable persistence across host runs. - -Like the [usage report](#2-usage-reports), a receipt is a **host-side artifact, not a wire message** — it rides no envelope variant, and a provider implements nothing to make one possible. It nonetheless lives in `contextgraph-types` rather than the host crate, for the same reason the usage report does: it is a *protocol-defined shape*. Any host in any language that claims the consent guarantee must produce this shape, and an auditor reading a persisted ledger must be able to parse it without depending on one particular host implementation. The ledger and gate that *consume* receipts are host machinery and stay in `contextgraph-host`. - -### Host behavior: reject unconsented egress -A host's pre-query consent gate -([`ConsentStore::evaluate`](https://docs.rs/contextgraph-host/latest/contextgraph_host/consent/struct.ConsentStore.html#method.evaluate)) -is scope-aware: - -- A provider declaring **off-machine egress scopes** is permitted only when *every* such scope has a recorded receipt. A scope with no matching receipt **MUST** cause the query to be refused with a **typed error** — - [`HostError::ConsentScopeRequired`](https://docs.rs/contextgraph-host/latest/contextgraph_host/error/enum.HostError.html) - naming exactly the scopes that would leave unconsented — and the payload **MUST NOT** be transmitted (requirement C6). A budget-style boolean `ConsentRecord` does **not** satisfy a scope gate; only a receipt for that scope does. -- A provider declaring only the boolean `egress` flag (no scopes) keeps the pre-scope legacy gate — a `ConsentRecord` unlocks it. This is what keeps the change additive: an existing provider that never declares scopes behaves exactly as before. - -The runtime gate is **presence-based** (does a receipt for the scope exist?) and carries no clock, so it never depends on wall-time to make a decision. **Expiry** is a first-class receipt property -([`ConsentReceipt::is_live`](https://docs.rs/contextgraph-types/latest/contextgraph_types/consent/struct.ConsentReceipt.html#method.is_live)): -a host that enforces expiry consults -[`ConsentStore::live_receipt`](https://docs.rs/contextgraph-host/latest/contextgraph_host/consent/struct.ConsentStore.html#method.live_receipt) -against its own `now`, and treats an expired receipt as absent — re-shutting the gate — while the receipt itself is never pruned from the audit ledger. - -### Worked audit scenario: "what left, where, who agreed, when?" - -Six months after the fact, an auditor asks whether a customer's repository snippets were ever sent to an external model. The host answers from its persisted consent ledger — no live process required: +pub struct FrameVerdict { + pub frame: FrameId, // the identity being answered, echoed in full + // flattened: {"status": "...", "replacement_digest": "..."} + pub verdict: Verdict, +} -```rust -for receipt in store.receipts_for("acme-cloud-model") { - println!( - "{scope} — granted by {grantor:?} at {at}{expiry}", - scope = receipt.scope, // third-party-model - grantor = receipt.grantor, // Human("ops@oxagen.sh") - at = receipt.granted_at, // 2026-01-14T09:02:00Z - expiry = receipt.expires_at // Some("2026-07-14T00:00:00Z") - .map(|e| format!(", expiring {e}")) - .unwrap_or_default(), - ); +pub enum Verdict { + Valid, // unchanged — keep reusing it + Stale { replacement_digest: Option }, // changed — drop it + Gone, // no longer exists — drop it + Unknown, // can't say — don't reuse it } ``` -Every question is answered by a field: +**No frame body travels in either direction.** That is the entire economic +point: verification costs **bytes, not tokens**, so a host can afford to run it +every turn over frames it would otherwise have re-fetched in full. Even a +`stale` verdict carries at most the provider's *current digest* — enough for a +host to tell what it would be re-fetching, never the replacement content +itself. + +The **digest is the ground truth**. A provider compares the digest the host +presents against the digest its source has now: equal ⇒ `valid`, different ⇒ +`stale`, source gone ⇒ `gone`, can't tell ⇒ `unknown`. Because the digest is +provider-declared and opaque (§1), the provider is the only party that *can* +answer — which is why the conformance case below exists to hold it honest. -- **What left?** The `scope` (`third-party-model`) — the class of destination — combined with the [usage report](#2-usage-reports)'s `served_frames`, which name the exact frames that provider served. -- **To whom?** The pinned `provider_name` / `provider_version` at grant time. -- **Who agreed?** The `grantor` — a named human or a named policy, not an anonymous "yes". -- **When?** `granted_at`, and `expires_at` if the grant was time-boxed. A query after `expires_at` would have been refused by the gate, so the window of authorized egress is itself on the record. +Each verdict **echoes the full identity it answers** rather than relying on +array position, so a provider that reorders, omits, or duplicates entries can +never shift a `valid` onto the wrong frame. -Because the ledger is append-only, a *revocation* or a lapsed expiry adds to the record rather than erasing it — the audit shows not just the current state but the full history of what was permitted and when. +### Host behavior -### Conformance (§3) +Verification is **default-deny**. A host **MUST** keep reusing a held frame +only on an explicit `valid`; every other outcome drops it (requirement V2). In +particular an identity that comes back with **no verdict at all** is treated as +`unknown` — silence is not validity. + +The reference host implements this in +[`Host::verify_frames`](https://docs.rs/contextgraph-host/latest/contextgraph_host/host/struct.Host.html#method.verify_frames), +which groups held identities by provider, asks each capable provider once, and +returns a **total partition** of the input into `retained` and `dropped` — no +held frame is ever silently lost. Each drop carries a +[`DropReason`](https://docs.rs/contextgraph-host/latest/contextgraph_host/host/enum.DropReason.html), +and `warrants_requery()` tells the host which dropped frames are worth fetching +again — false only for `gone`, which is not there to re-fetch. + +**Capability gating and fallback.** A host sends `context/verify` only to a +provider whose handshake advertised `capabilities.verify`. Against a provider +that doesn't, the host **MUST** fall back to re-querying rather than reusing +unchecked (requirement V3) — the same fallback a frame with no +`content_digest` gets (§1, D4). Since `verify` defaults to `false`, an existing +provider that implements nothing keeps working unchanged, and the reference +`ContextProvider::verify` default answers `unknown` for everything, so a +provider can never *accidentally* bless a stale frame. + +A verify failure is isolated exactly like a query fan-out leg: a provider that +errors or times out has *its own* frames dropped for re-query, and never +affects another provider's. + +**When to verify** is host policy, not protocol. A host **SHOULD** re-verify +held frames at turn boundaries once they are older than the freshness window it +is willing to tolerate. This is deliberately informative: `verify_frames` holds +no state, caches no frames, and tracks no turns — the protocol's job is to +answer the question when asked, not to decide when to ask it. + +The window is framed as *host-tolerated* rather than provider-declared on +purpose. A provider knows how often its sources *tend* to change, but only the +host knows how much staleness this particular task can absorb, and making the +window a wire field would push a scheduling decision into the protocol and give +the host state to keep. A provider that wants to share what it knows can +advertise a suggested freshness hint as a **future additive capability field**; +that would inform the host's policy without ever overriding it, and it needs no +change to the exchange specified here. + +Note the ordering payoff: because only *evicted* frames change the composed +context, a turn in which everything verifies `valid` leaves §1's canonical +rendering byte-identical, so the prompt prefix — and its cache — survives. Only +a real change breaks the prefix, which is exactly when it *should* break. + +### Verify vs subscribe (for the 1.0 freeze) + +[#6](https://github.com/macanderson/context-graph-protocol/issues/6)'s +`subscribe` and this section's `verify` answer the same question — *is it still +true?* — from opposite directions, and the freeze can keep both, either, or +one: + +| | `subscribe` (push, #6) | `verify` (pull, §4) | +| - | ---------------------- | ------------------- | +| Who initiates | Provider, when a source changes | Host, when it wants to reuse | +| Transport need | A long-lived channel | Any request/response round trip | +| Provider must | Watch its sources | Compare a digest on demand | +| Latency to detect | Immediate | At the host's next check | +| Cost shape | Idle connection + change events | One small round trip per check | +| Fits | Stateful local indexers, file watchers | Stateless HTTP, batch indexes, serverless | + +They are **complementary, not alternatives**, and the capability flags are +independent: a provider may advertise either, both, or neither. Neither +subsumes the other — push has no answer for a host that reconnects and wants to +know whether what it cached is still good, and pull has no answer for a host +that needs to know *within milliseconds*. A provider advertising both gives a +host immediate invalidation plus a way to re-establish trust after any gap in +the channel. + +Because both are capability-gated and default to `false`, the freeze can ship +`verify` without `subscribe` (this section), `subscribe` without `verify`, or +both, with no flag day either way — a host degrades to re-query whenever the +mechanism it wants is absent. + +### Conformance (§4) | # | Requirement | Enforced / verified by | | - | ----------- | ---------------------- | -| C5 | A provider **MUST** declare its egress scopes (`egress_scopes`) truthfully and consistently with `data_flow.egress`; an off-machine scope alongside `egress: false`, or a non-namespaced custom scope, is a conformance failure. | `DataFlow::scopes_consistent`; `consent-scope` conformance check | -| C6 | A host **MUST** reject a query to a provider any of whose off-machine egress scopes lacks a recorded consent receipt, with a typed error, before transmitting the payload. | `ConsentStore::evaluate` scope gate; `HostError::ConsentScopeRequired`; host scope-gate test | +| V1 | A provider advertising `verify` **MUST** answer honestly by comparing digests: `valid` for a frame whose presented digest matches what it currently serves, `stale` when it differs on a frame it still serves. It **MUST NOT** answer `valid` for content bytes it is not serving. | `verify-honesty` conformance check (asks twice about just-served frames — real digests, then mutated); `--misbehave rubber-stamp-verify` and `hollow-verify` fixture modes prove it bites both ways | +| V2 | A host **MUST** keep reusing a held frame only on an explicit `valid`; `stale`, `gone`, `unknown`, and a missing verdict all evict it. | `Verdict::permits_reuse`; `Host::verify_frames` default-deny partition; host eviction tests | +| V3 | A host **MUST NOT** send `context/verify` to a provider that does not advertise `capabilities.verify`, and **MUST** fall back to re-querying those frames. | `Host::verify_frames` capability gate; `ContextProvider::verify` default; host fallback test | +| V4 | Neither a verify request nor its response **MAY** carry frame bodies; a `stale` verdict carries at most a replacement **digest**. | `VerifyRequest`/`VerifyResponse` shapes; `verify_wire` no-bodies test asserted against the serialized envelope | diff --git a/docs/implementing-a-provider.md b/docs/implementing-a-provider.md index 8dfabc9..3a0781a 100644 --- a/docs/implementing-a-provider.md +++ b/docs/implementing-a-provider.md @@ -64,6 +64,8 @@ Both transports carry the same message vocabulary, `contextgraph-host::wire::Env | `handshake_ack` | provider → host | `{ protocol_version, provider: ProviderInfo, capabilities: Capabilities }` | | `query` | host → provider | `{ query: ContextQuery }` | | `frames` | provider → host | `{ result: ContextQueryResult }` | +| `verify` | host → provider | `{ request: VerifyRequest }` — *only if you advertise `verify`* | +| `verified` | provider → host | `{ response: VerifyResponse }` | | `shutdown` | host → provider | *(no payload)* | | `error` | provider → host | `{ message: String }` | @@ -85,7 +87,19 @@ their `serde_json` serialization, field names and all. `error` reply lets you report a problem without dying — a provider that exits on a bad request fails the `malformed-input-tolerance` conformance check). -3. **Shutdown.** The host sends `shutdown`; a well-behaved provider exits +3. **Zero or more verifications** *(optional).* If you set + `capabilities.verify`, the host may send `verify` with a batch of frame + identities and expect `verified` back with one verdict each — `valid`, + `stale` (optionally naming your current digest), `gone`, or `unknown`. + **Never send frame bodies in a `verified` reply**: the whole point is that + revalidation costs bytes rather than tokens, so the host re-queries when it + actually wants new content. Answer by comparing the digest the host presents + against the one you currently serve — a mismatch is `stale`, and answering + `valid` for bytes you are not serving fails the `verify-honesty` conformance + check. Leave `verify` unset (the default) and the host simply re-queries your + frames instead; nothing breaks. See + [context-reuse §4](./context-reuse.md#4-context-verification). +4. **Shutdown.** The host sends `shutdown`; a well-behaved provider exits cleanly (stdio: exit the process; HTTP: nothing further to do — the host doesn't expect a reply). diff --git a/docs/protocol-surface.md b/docs/protocol-surface.md index c6ad2b9..fb38c76 100644 --- a/docs/protocol-surface.md +++ b/docs/protocol-surface.md @@ -52,7 +52,8 @@ pub struct Capabilities { pub upsert: bool, pub graph: bool, pub embeddings_fingerprint: Option, - pub subscribe: bool, + pub subscribe: bool, // push invalidation (issue #6) + pub verify: bool, // answers context/verify (context-reuse §4); defaults false } // The closed egress-scope vocabulary (context-reuse §3), serialized as a flat @@ -183,6 +184,38 @@ revalidation. They surface here as the `content_digest` frame field, the `verify` / `verified` envelope variants — all additive within the `contextgraph/1` family. Their conformance requirements are consolidated below. +## Context verification + +A host revalidates frames it already holds without re-sending any body +([context-reuse §4](./context-reuse.md#4-context-verification)). Sent only to a +provider advertising `verify`; otherwise the host re-queries. + +```rust +pub struct VerifyRequest { + pub frames: Vec, // identities only — never frame bodies +} + +pub struct FrameVerdict { + pub frame: FrameId, // echoed in full, so verdicts correlate by match not position + pub verdict: Verdict, // flattened: {"status": "...", ...} +} + +pub enum Verdict { + Valid, // unchanged — may keep reusing + Stale { replacement_digest: Option }, // changed — a digest, never a body + Gone, // no longer exists + Unknown, // provider cannot say +} + +pub struct VerifyResponse { + pub verdicts: Vec, +} +``` + +A host **MUST** keep reusing a frame only on an explicit `valid` — an identity +answered `unknown`, or not answered at all, is evicted like any other. Reuse +requires a positive answer, never the absence of a negative one. + ## Wire framing (defined in `contextgraph-host`, not `contextgraph-types`) `contextgraph-types` defines the payload shapes above; `contextgraph-host::wire::Envelope` @@ -190,7 +223,7 @@ defines how they're framed on the wire — newline-delimited JSON (NDJSON), one `serde_json` value per line over stdio, or one JSON body per streamable-HTTP request/response. See [Implementing a provider](./implementing-a-provider.md) for the full envelope vocabulary (`handshake` / `handshake_ack` / `query` / -`frames` / `shutdown` / `error`) and the version-compatibility rule. See +`frames` / `verify` / `verified` / `shutdown` / `error`) and the version-compatibility rule. See [`examples/`](../examples/) for diffable wire transcripts of a complete session, or the [machine-readable JSON Schema](../schema/contextgraph-envelope.schema.json) to validate messages in any language. @@ -276,8 +309,10 @@ page; they are consolidated here because this section is the authoritative list. | U1 | A host **MUST** be able to produce a usage report for any query it executed, whose consumed total equals the summed `token_cost` of the served frames it reports. | `contextgraph-host::FanOut::usage_report`; `usage-report` conformance check | | C5 | A provider **MUST** declare its egress scopes (`egress_scopes`) truthfully and consistently with `data_flow.egress`; an off-machine scope alongside `egress: false` is a conformance failure. | `consent-scope` conformance check | | C6 | A host **MUST** reject a frame whose provider declares an egress scope with no live matching [consent receipt](./context-reuse.md#3-consent-scopes-and-receipts), with a typed error, before transmitting the query. | `ConsentStore` scope gate | -| V1 | A provider advertising the `verify` capability **MUST** answer a `context/verify` request honestly: a current identity ⇒ `valid`; a digest it no longer serves ⇒ `stale`/`gone`/`unknown`, never `valid`. | `verify` conformance check | -| V2 | A host **MUST** treat a `stale`/`gone` verdict as evicting the frame from the composed context, and **MUST** fall back to re-query for providers without `verify`. | `contextgraph-host` verify support | +| V1 | A provider advertising `verify` **MUST** answer honestly by comparing digests: `valid` when the presented digest matches what it currently serves, `stale` when it differs on a frame it still serves. It **MUST NOT** answer `valid` for content bytes it is not serving. | `verify-honesty` conformance check | +| V2 | A host **MUST** keep reusing a held frame only on an explicit `valid`; `stale`, `gone`, `unknown`, and a missing verdict all evict it. | `contextgraph-host::Host::verify_frames` default-deny partition | +| V3 | A host **MUST NOT** send `context/verify` to a provider that does not advertise `capabilities.verify`, and **MUST** fall back to re-querying those frames. | `Host::verify_frames` capability gate; `ContextProvider::verify` default | +| V4 | Neither a verify request nor its response **MAY** carry frame bodies; a `stale` verdict carries at most a replacement **digest**. | `VerifyRequest`/`VerifyResponse` shapes; `verify_wire` no-bodies test | ## Version strings diff --git a/examples/full-stdio-session.ndjson b/examples/full-stdio-session.ndjson index 1a6ab69..ec5b73a 100644 --- a/examples/full-stdio-session.ndjson +++ b/examples/full-stdio-session.ndjson @@ -1,5 +1,7 @@ -{"type":"handshake","protocol_version":"contextgraph/1.0-draft"} -{"type":"handshake_ack","protocol_version":"contextgraph/1.0-draft","provider":{"name":"repo-graph","version":"0.2.0","data_flow":{"reads":true,"writes":false,"egress":false}},"capabilities":{"query":{"kinds":["doc","symbol"],"filters":["path","lang"]},"upsert":false,"graph":true,"embeddings_fingerprint":null,"subscribe":false}} -{"type":"query","query":{"goal":"how do I configure the retry policy?","query_text":"retry policy configuration","kinds":["doc","symbol"],"anchors":["src/config.rs"],"max_frames":5,"max_tokens":1024}} -{"type":"frames","result":{"frames":[{"id":"repo-graph:retry-doc","kind":"doc","title":"Retry policy","content":"Retry behavior is set in Config::retry. max_attempts bounds the tries; backoff_ms is the initial delay, doubled each attempt.","uri":"file:///repo/docs/retry.md","score":0.92,"token_cost":41,"provenance":[{"type":"file","uri":"file:///repo/docs/retry.md","range":"L1-20","digest":"sha256:9f2c3e7a","method":"file-read","by":"repo-graph"}],"citation_label":"retry.md L1-20","relations":[]},{"id":"repo-graph:retry-sym","kind":"symbol","title":"Config::retry","content":"pub struct RetryPolicy { pub max_attempts: u32, pub backoff_ms: u64 }","uri":"file:///repo/src/config.rs","score":0.81,"token_cost":23,"provenance":[{"type":"file","uri":"file:///repo/src/config.rs","range":"L42-44","digest":"sha256:1a2b3c4d","method":"tree-sitter-symbol-extraction","by":"repo-graph"}],"citation_label":"config.rs L42-44","relations":[]}],"truncated":false}} -{"type":"shutdown"} +{"type": "handshake", "protocol_version": "contextgraph/1.0-draft"} +{"type": "handshake_ack", "protocol_version": "contextgraph/1.0-draft", "provider": {"name": "repo-graph", "version": "0.2.0", "data_flow": {"reads": true, "writes": false, "egress": false}}, "capabilities": {"query": {"kinds": ["doc", "symbol"], "filters": ["path", "lang"]}, "upsert": false, "graph": true, "embeddings_fingerprint": null, "subscribe": false, "verify": true}} +{"type": "query", "query": {"goal": "how do I configure the retry policy?", "query_text": "retry policy configuration", "kinds": ["doc", "symbol"], "anchors": ["src/config.rs"], "max_frames": 5, "max_tokens": 1024}} +{"type": "frames", "result": {"frames": [{"id": "repo-graph:retry-doc", "kind": "doc", "title": "Retry policy", "content": "Retry behavior is set in Config::retry. max_attempts bounds the tries; backoff_ms is the initial delay, doubled each attempt.", "content_digest": "sha256:9f2c3e7a1b", "uri": "file:///repo/docs/retry.md", "score": 0.92, "token_cost": 41, "provenance": [{"type": "file", "uri": "file:///repo/docs/retry.md", "range": "L1-20", "digest": "sha256:9f2c3e7a", "method": "file-read", "by": "repo-graph"}], "citation_label": "retry.md L1-20", "relations": []}, {"id": "repo-graph:retry-sym", "kind": "symbol", "title": "Config::retry", "content": "pub struct RetryPolicy { pub max_attempts: u32, pub backoff_ms: u64 }", "content_digest": "sha256:1a2b3c4d5e", "uri": "file:///repo/src/config.rs", "score": 0.81, "token_cost": 23, "provenance": [{"type": "file", "uri": "file:///repo/src/config.rs", "range": "L42-44", "digest": "sha256:1a2b3c4d", "method": "tree-sitter-symbol-extraction", "by": "repo-graph"}], "citation_label": "config.rs L42-44", "relations": []}], "truncated": false}} +{"type": "verify", "request": {"frames": [{"provider_id": "repo-graph", "frame_id": "repo-graph:retry-doc", "content_digest": "sha256:9f2c3e7a1b"}, {"provider_id": "repo-graph", "frame_id": "repo-graph:retry-sym", "content_digest": "sha256:1a2b3c4d5e"}]}} +{"type": "verified", "response": {"verdicts": [{"frame": {"provider_id": "repo-graph", "frame_id": "repo-graph:retry-doc", "content_digest": "sha256:9f2c3e7a1b"}, "status": "valid"}, {"frame": {"provider_id": "repo-graph", "frame_id": "repo-graph:retry-sym", "content_digest": "sha256:1a2b3c4d5e"}, "status": "stale", "replacement_digest": "sha256:7d8e9f0a1b"}]}} +{"type": "shutdown"} diff --git a/examples/reference-messages.json b/examples/reference-messages.json index ad78df6..cdd6bac 100644 --- a/examples/reference-messages.json +++ b/examples/reference-messages.json @@ -9,14 +9,28 @@ "provider": { "name": "repo-graph", "version": "0.2.0", - "data_flow": { "reads": true, "writes": false, "egress": false } + "data_flow": { + "reads": true, + "writes": false, + "egress": false + } }, "capabilities": { - "query": { "kinds": ["doc", "symbol"], "filters": ["path", "lang"] }, + "query": { + "kinds": [ + "doc", + "symbol" + ], + "filters": [ + "path", + "lang" + ] + }, "upsert": false, "graph": true, "embeddings_fingerprint": null, - "subscribe": false + "subscribe": false, + "verify": true } }, { @@ -33,7 +47,12 @@ } }, "capabilities": { - "query": { "kinds": ["doc"], "filters": [] }, + "query": { + "kinds": [ + "doc" + ], + "filters": [] + }, "upsert": false, "graph": false, "embeddings_fingerprint": null, @@ -45,8 +64,13 @@ "query": { "goal": "how do I configure the retry policy?", "query_text": "retry policy configuration", - "kinds": ["doc", "symbol"], - "anchors": ["src/config.rs"], + "kinds": [ + "doc", + "symbol" + ], + "anchors": [ + "src/config.rs" + ], "max_frames": 5, "max_tokens": 1024 } @@ -60,6 +84,7 @@ "kind": "doc", "title": "Retry policy", "content": "Retry behavior is set in Config::retry. max_attempts bounds the tries; backoff_ms is the initial delay, doubled each attempt.", + "content_digest": "sha256:9f2c3e7a1b", "uri": "file:///repo/docs/retry.md", "score": 0.92, "token_cost": 41, @@ -81,6 +106,7 @@ "kind": "symbol", "title": "Config::retry", "content": "pub struct RetryPolicy { pub max_attempts: u32, pub backoff_ms: u64 }", + "content_digest": "sha256:1a2b3c4d5e", "uri": "file:///repo/src/config.rs", "score": 0.81, "token_cost": 23, @@ -101,6 +127,47 @@ "truncated": false } }, + { + "type": "verify", + "request": { + "frames": [ + { + "provider_id": "repo-graph", + "frame_id": "repo-graph:retry-doc", + "content_digest": "sha256:9f2c3e7a1b" + }, + { + "provider_id": "repo-graph", + "frame_id": "repo-graph:retry-sym", + "content_digest": "sha256:1a2b3c4d5e" + } + ] + } + }, + { + "type": "verified", + "response": { + "verdicts": [ + { + "frame": { + "provider_id": "repo-graph", + "frame_id": "repo-graph:retry-doc", + "content_digest": "sha256:9f2c3e7a1b" + }, + "status": "valid" + }, + { + "frame": { + "provider_id": "repo-graph", + "frame_id": "repo-graph:retry-sym", + "content_digest": "sha256:1a2b3c4d5e" + }, + "status": "stale", + "replacement_digest": "sha256:7d8e9f0a1b" + } + ] + } + }, { "type": "error", "message": "unsupported frame kind: 'image'" diff --git a/schema/contextgraph-envelope.schema.json b/schema/contextgraph-envelope.schema.json index 08bcb68..79cffe0 100644 --- a/schema/contextgraph-envelope.schema.json +++ b/schema/contextgraph-envelope.schema.json @@ -16,6 +16,12 @@ { "$ref": "#/$defs/Frames" }, + { + "$ref": "#/$defs/Verify" + }, + { + "$ref": "#/$defs/Verified" + }, { "$ref": "#/$defs/Shutdown" }, @@ -76,6 +82,28 @@ "additionalProperties": false }, + "Verify": { + "type": "object", + "description": "host -> provider. Revalidate frames the host already holds (context/verify, docs/context-reuse.md §4). Carries frame identities only — NEVER frame bodies, so verification costs bytes rather than tokens. Capability-gated: a host sends this only to a provider whose handshake advertised capabilities.verify.", + "properties": { + "type": { "const": "verify" }, + "request": { "$ref": "#/$defs/VerifyRequest" } + }, + "required": ["type", "request"], + "additionalProperties": false + }, + + "Verified": { + "type": "object", + "description": "provider -> host. One verdict per frame identity the provider is answering for (docs/context-reuse.md §4). No frame bodies travel here either; a stale verdict may carry the replacement DIGEST, never the replacement content.", + "properties": { + "type": { "const": "verified" }, + "response": { "$ref": "#/$defs/VerifyResponse" } + }, + "required": ["type", "response"], + "additionalProperties": false + }, + "Shutdown": { "type": "object", "description": "host -> provider. No payload. A well-behaved provider exits cleanly (stdio) or expects no further requests (HTTP).", @@ -97,6 +125,68 @@ "additionalProperties": false }, + "FrameId": { + "type": "object", + "description": "The stable identity of one frame's exact content bytes: (provider id, frame id, content digest). The spine shared by deterministic composition, usage reports, and context/verify (docs/context-reuse.md §1).", + "properties": { + "provider_id": { "type": "string", "minLength": 1 }, + "frame_id": { "type": "string", "minLength": 1 }, + "content_digest": { + "type": ["string", "null"], + "$comment": "Provider-declared and opaque. Absent ⇒ the frame is not verifiable and a host re-queries it rather than reusing it unchecked (§1 D4)." + } + }, + "required": ["provider_id", "frame_id"], + "additionalProperties": false + }, + + "VerifyRequest": { + "type": "object", + "description": "The frame identities a host asks a provider to revalidate. Identities only — no frame bodies (docs/context-reuse.md §4).", + "properties": { + "frames": { + "type": "array", + "items": { "$ref": "#/$defs/FrameId" }, + "$comment": "A host SHOULD only include identities carrying a content_digest; a digest-less frame cannot be revalidated and is re-queried instead." + } + }, + "required": ["frames"], + "additionalProperties": false + }, + + "VerdictStatus": { + "enum": ["valid", "stale", "gone", "unknown"], + "description": "A provider's answer for one frame (docs/context-reuse.md §4). `valid` — content unchanged, the host MAY keep reusing the body it holds. `stale` — the frame exists but its content changed; the host MUST NOT keep serving its copy. `gone` — the frame no longer exists, and re-querying this identity is pointless. `unknown` — the provider cannot say; the host MUST NOT read this as validity. A host reuses a frame ONLY on `valid`: reuse requires a positive answer, never the absence of a negative one." + }, + + "FrameVerdict": { + "type": "object", + "description": "One frame's verdict, flattened onto the identity it answers. The identity is echoed in FULL rather than implied by position, so a host correlates by matching and a provider that reorders, omits, or duplicates entries cannot shift a `valid` onto the wrong frame. An identity that comes back with no entry is treated as `unknown` — silence is not validity.", + "properties": { + "frame": { "$ref": "#/$defs/FrameId" }, + "status": { "$ref": "#/$defs/VerdictStatus" }, + "replacement_digest": { + "type": ["string", "null"], + "$comment": "Only meaningful with status `stale`: the provider's CURRENT digest for the frame, offered so a host can tell what it would be re-fetching. A digest, never a body. Optional — a provider that knows the content changed but not what it is now still answers `stale` honestly." + } + }, + "required": ["frame", "status"], + "additionalProperties": false + }, + + "VerifyResponse": { + "type": "object", + "description": "A provider's answer to a VerifyRequest (docs/context-reuse.md §4).", + "properties": { + "verdicts": { + "type": "array", + "items": { "$ref": "#/$defs/FrameVerdict" } + } + }, + "required": ["verdicts"], + "additionalProperties": false + }, + "DataFlow": { "type": "object", "description": "Declares what a provider does with data. `egress` is the security-critical field: a conforming host gates egress providers behind recorded, named consent. `egress_scopes` (optional) classes WHERE content goes, for scope-level consent receipts.", @@ -160,7 +250,11 @@ "upsert": { "type": "boolean" }, "graph": { "type": "boolean" }, "embeddings_fingerprint": { "type": ["string", "null"] }, - "subscribe": { "type": "boolean" } + "subscribe": { "type": "boolean" }, + "verify": { + "type": "boolean", + "$comment": "Whether the provider answers context/verify (docs/context-reuse.md §4). Optional and defaults to false, so a pre-§4 provider is treated as unsupported and the host falls back to re-querying. Independent of `subscribe` — push and pull freshness are complementary." + } }, "required": ["query", "upsert", "graph", "subscribe"], "additionalProperties": false