diff --git a/crates/tracedecay-agent-hosts/src/agents/context_scout/decision_engine.rs b/crates/tracedecay-agent-hosts/src/agents/context_scout/decision_engine.rs index b8ba4c8b0e5..33b163a6f11 100644 --- a/crates/tracedecay-agent-hosts/src/agents/context_scout/decision_engine.rs +++ b/crates/tracedecay-agent-hosts/src/agents/context_scout/decision_engine.rs @@ -21,10 +21,13 @@ //! fall back to its non-model path. use std::collections::BTreeMap; +use std::io::Read as _; use std::sync::Arc; use std::sync::atomic::{AtomicUsize, Ordering}; use std::time::{Duration, Instant}; +use serde::Deserialize; +use serde::de::{self, MapAccess, SeqAccess, Visitor}; use serde_json::{Map, Value, json}; use sha2::{Digest, Sha256}; use tracedecay_application::http_agent::http_agent; @@ -33,11 +36,14 @@ use tracedecay_runtime_core::native_runtime; /// Path of the typed-question endpoint under the configured base URL. const SYSTEMONE_PATH: &str = "/v1/systemone"; +/// Bound decoded HTTP answer bytes before JSON parsing, matching qualified replay. +const MAX_DECISION_RESPONSE_BYTES: usize = 1024 * 1024; + /// Tolerance when checking a `score` against the rubric's last level index. const SCORE_EPSILON: f64 = 1e-6; /// Decision servers return a probability distribution, not independent scores. -/// Allow a small rounding error when servers serialize decimal probabilities. +/// Allow rounding error in the distribution total, not a conflicting chosen option. const CHOICE_EPSILON: f64 = 1e-3; /// What one question asks. @@ -82,6 +88,14 @@ pub enum DecisionAnswerV1 { Score(f64), } +/// Byte-exact identity of the request prepared by the configured Rust client. +/// Carries no transcript content, only the bytes and digest of the JSON body. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct DecisionRequestCommitmentV1 { + pub wire_sha256: [u8; 32], + pub wire_bytes: usize, +} + /// Every answer of one successful exchange. #[derive(Debug, Clone, PartialEq)] pub struct DecisionAnswersV1 { @@ -155,6 +169,14 @@ pub struct DecisionEngineIdentityV1 { pub trait DecisionEnginePortV1: Send + Sync { fn ask_blocking(&self, request: &DecisionRequestV1) -> DecisionEngineOutcomeV1; fn identity(&self) -> DecisionEngineIdentityV1; + + /// Test doubles cannot claim a production HTTP wire commitment. + fn request_commitment( + &self, + _request: &DecisionRequestV1, + ) -> Option { + None + } } /// HTTP client for the decision-model API. @@ -240,6 +262,31 @@ impl DecisionEngineClientV1 { self.identity.clone() } + /// Render the pretty UTF-8 JSON that is both sent over HTTP and committed. + /// Sharing the bytes prevents receipt/transport serializer drift. + fn request_wire(&self, request: &DecisionRequestV1) -> Option> { + if !request_is_well_formed(request) { + return None; + } + let body = request_body( + request, + self.identity.model.as_deref(), + self.max_state_bytes, + ); + serde_json::to_vec_pretty(&body).ok() + } + + pub fn request_commitment( + &self, + request: &DecisionRequestV1, + ) -> Option { + let wire = self.request_wire(request)?; + Some(DecisionRequestCommitmentV1 { + wire_sha256: Sha256::digest(&wire).into(), + wire_bytes: wire.len(), + }) + } + fn try_acquire(&self) -> Option { let mut current = self.in_flight.load(Ordering::Acquire); loop { @@ -272,12 +319,16 @@ impl DecisionEngineClientV1 { return DecisionEngineOutcomeV1::Backpressured; }; let started = Instant::now(); - let body = request_body( - request, - self.identity.model.as_deref(), - self.max_state_bytes, - ); - let mut response = match self.agent.post(&self.endpoint).send_json(&body) { + let Some(wire) = self.request_wire(request) else { + tracing::debug!("decision engine request could not be serialized"); + return DecisionEngineOutcomeV1::Malformed; + }; + let mut response = match self + .agent + .post(&self.endpoint) + .header("content-type", "application/json") + .send(wire.as_slice()) + { Ok(response) => response, Err(error) => return transport_failure(&error), }; @@ -286,20 +337,37 @@ impl DecisionEngineClientV1 { tracing::debug!(status, "decision engine refused"); return DecisionEngineOutcomeV1::Refused { status }; } - let value = match response.body_mut().read_json::() { + // The server is independently configured and may return an unlimited + // response, including one without Content-Length. Never deserialize + // unbounded bytes from a transcript-bearing decision exchange. + let mut answer_bytes = Vec::new(); + if let Err(error) = response + .body_mut() + .as_reader() + .take((MAX_DECISION_RESPONSE_BYTES + 1) as u64) + .read_to_end(&mut answer_bytes) + { + return if is_io_timeout(&error) { + tracing::debug!(%error, "decision engine timed out reading its answer"); + DecisionEngineOutcomeV1::TimedOut + } else { + tracing::debug!(%error, "decision engine answer unreadable"); + DecisionEngineOutcomeV1::Malformed + }; + } + if answer_bytes.len() > MAX_DECISION_RESPONSE_BYTES { + tracing::debug!("decision engine answer exceeds the byte limit"); + return DecisionEngineOutcomeV1::Malformed; + } + let value = match serde_json::from_slice::(&answer_bytes) { Ok(value) => value, Err(error) => { - return if is_timeout(&error) { - tracing::debug!(%error, "decision engine timed out reading its answer"); - DecisionEngineOutcomeV1::TimedOut - } else { - tracing::debug!(%error, "decision engine answer unreadable"); - DecisionEngineOutcomeV1::Malformed - }; + tracing::debug!(%error, "decision engine answer is not JSON"); + return DecisionEngineOutcomeV1::Malformed; } }; let latency_micros = u64::try_from(started.elapsed().as_micros()).unwrap_or(u64::MAX); - match parse_answers(&value, request) { + match parse_answers(&value.0, request) { Some((answers, model)) => DecisionEngineOutcomeV1::Answered(DecisionAnswersV1 { answers, model, @@ -334,6 +402,13 @@ impl DecisionEnginePortV1 for DecisionEngineClientV1 { fn identity(&self) -> DecisionEngineIdentityV1 { Self::identity(self) } + + fn request_commitment( + &self, + request: &DecisionRequestV1, + ) -> Option { + Self::request_commitment(self, request) + } } fn request_is_well_formed(request: &DecisionRequestV1) -> bool { @@ -411,22 +486,101 @@ fn transport_failure(error: &ureq::Error) -> DecisionEngineOutcomeV1 { fn is_timeout(error: &ureq::Error) -> bool { match error { ureq::Error::Timeout(_) => true, - ureq::Error::Io(io) => { - matches!( - io.kind(), - std::io::ErrorKind::TimedOut | std::io::ErrorKind::WouldBlock - ) || io.to_string().to_ascii_lowercase().contains("timed out") - } + ureq::Error::Io(io) => is_io_timeout(io), other => other.to_string().to_ascii_lowercase().contains("timeout"), } } +fn is_io_timeout(error: &std::io::Error) -> bool { + matches!( + error.kind(), + std::io::ErrorKind::TimedOut | std::io::ErrorKind::WouldBlock + ) || error.to_string().to_ascii_lowercase().contains("timed out") +} + +/// Refuse ambiguous response objects before interpreting model decisions. +/// serde_json::Value alone silently keeps the last occurrence of a JSON key, +/// while the qualified HTTP evaluator rejects all duplicate keys. +struct UnambiguousValue(Value); + +impl<'de> Deserialize<'de> for UnambiguousValue { + fn deserialize(deserializer: D) -> Result + where + D: serde::Deserializer<'de>, + { + deserializer.deserialize_any(UnambiguousValueVisitor) + } +} + +struct UnambiguousValueVisitor; + +impl<'de> Visitor<'de> for UnambiguousValueVisitor { + type Value = UnambiguousValue; + + fn expecting(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + formatter.write_str("JSON without duplicate object keys") + } + + fn visit_unit(self) -> Result { + Ok(UnambiguousValue(Value::Null)) + } + + fn visit_bool(self, value: bool) -> Result { + Ok(UnambiguousValue(Value::Bool(value))) + } + + fn visit_i64(self, value: i64) -> Result { + Ok(UnambiguousValue(Value::Number(value.into()))) + } + + fn visit_u64(self, value: u64) -> Result { + Ok(UnambiguousValue(Value::Number(value.into()))) + } + + fn visit_f64(self, value: f64) -> Result { + let number = serde_json::Number::from_f64(value) + .ok_or_else(|| E::custom("nonfinite decision JSON number"))?; + Ok(UnambiguousValue(Value::Number(number))) + } + + fn visit_str(self, value: &str) -> Result { + Ok(UnambiguousValue(Value::String(value.to_owned()))) + } + + fn visit_string(self, value: String) -> Result { + Ok(UnambiguousValue(Value::String(value))) + } + + fn visit_seq>(self, mut sequence: A) -> Result { + let mut values = Vec::new(); + while let Some(value) = sequence.next_element::()? { + values.push(value.0); + } + Ok(UnambiguousValue(Value::Array(values))) + } + + fn visit_map>(self, mut entries: A) -> Result { + let mut values = Map::new(); + while let Some((key, value)) = entries.next_entry::()? { + if values.contains_key(&key) { + return Err(de::Error::custom(format!("duplicate decision JSON key: {key}"))); + } + values.insert(key, value.0); + } + Ok(UnambiguousValue(Value::Object(values))) + } +} + type ParsedAnswers = (BTreeMap, Option); -/// Every requested question must be answered with the requested kind; extra -/// answers are ignored. +/// Every requested question must be answered with the requested kind. +/// Unexpected answers are malformed, matching the qualified HTTP evaluator's +/// exact-request contract rather than silently accepting a different batch. fn parse_answers(value: &Value, request: &DecisionRequestV1) -> Option { let answers = value.get("answers")?.as_object()?; + if answers.len() != request.questions.len() { + return None; + } let mut parsed = BTreeMap::new(); for question in &request.questions { let answer = answers.get(&question.key)?.as_object()?; @@ -463,7 +617,7 @@ fn parse_answer( } let chosen = match answer.get("choice") { None | Some(Value::Null) => None, - Some(value) => Some(label_text(value)?), + Some(value) => Some(value.as_str()?.to_owned()), }; if chosen .as_ref() @@ -486,7 +640,7 @@ fn parse_answer( let selected_probability = *probabilities.get(chosen)?; if probabilities .values() - .any(|probability| *probability > selected_probability + CHOICE_EPSILON) + .any(|probability| *probability > selected_probability) { return None; } @@ -541,15 +695,6 @@ fn unit_interval(value: &Value) -> Option { .filter(|probability| (0.0..=1.0).contains(probability)) } -fn label_text(value: &Value) -> Option { - match value { - Value::String(text) => Some(text.clone()), - Value::Number(number) => Some(number.to_string()), - Value::Bool(flag) => Some(flag.to_string()), - _ => None, - } -} - #[cfg(test)] #[allow(clippy::unwrap_used)] pub mod test_server { @@ -660,7 +805,7 @@ pub mod test_server { #[cfg(test)] #[allow(clippy::unwrap_used)] mod tests { - use std::io::Read as _; + use std::io::{BufRead as _, BufReader, Write as _}; use std::time::{Duration, Instant}; use super::test_server::{ @@ -786,6 +931,96 @@ mod tests { ); } + #[test] + fn the_prepared_request_commitment_matches_the_actual_utf8_wire() { + let server = fake_decision_engine(200, &single_noul_body("q", 0.5), Duration::ZERO); + let mut configured = config(&server.url); + configured.model = Some("kev-4b".to_owned()); + configured.max_state_bytes = 9; + let client = DecisionEngineClientV1::new(configured.clone()).unwrap(); + let original = request("old private context éé latest", vec![noul("q")]); + let commitment = client.request_commitment(&original).unwrap(); + let _ = answered(client.ask_blocking(&original)); + let actual = server.last_body.lock().unwrap().clone(); + assert_eq!(server.requests.load(Ordering::SeqCst), 1); + assert_eq!(commitment.wire_bytes, actual.len()); + assert_eq!( + commitment.wire_sha256, + <[u8; 32]>::from(Sha256::digest(actual.as_bytes())) + ); + let sent: Value = serde_json::from_str(&actual).unwrap(); + assert_eq!(actual, serde_json::to_string_pretty(&sent).unwrap()); + assert_eq!(sent["model"], "kev-4b"); + assert_eq!(sent["state"], tail_on_char_boundary(&original.state, 9)); + assert!(!actual.contains("old private context")); + + configured.model = Some("different-model".to_owned()); + let changed_model = DecisionEngineClientV1::new(configured).unwrap(); + assert_ne!( + changed_model.request_commitment(&original), + Some(commitment) + ); + let mut changed_question = original.clone(); + changed_question.questions[0] + .instructions + .push_str(" Be precise."); + assert_ne!( + client.request_commitment(&changed_question), + Some(commitment) + ); + // The prefix intentionally discarded by the configured byte cap + // does not affect the sent request, but changed visible text does. + let alternate_prefix = request("different private context éé latest", vec![noul("q")]); + assert_eq!( + client.request_commitment(&alternate_prefix), + Some(commitment) + ); + let changed_visible = request("old private context éé latest!", vec![noul("q")]); + assert_ne!( + client.request_commitment(&changed_visible), + Some(commitment) + ); + assert!( + client + .request_commitment(&request("state", vec![])) + .is_none() + ); + } + + #[test] + fn duplicate_json_keys_cannot_change_the_model_decision() { + // Every response below is otherwise a valid typed answer. The ordinary + // serde_json::Value decoder accepts the last duplicate key; it must + // not let an ambiguous server response qualify as Scout evidence. + for (body, questions) in [ + ( + r#"{"answers":{"q":{"type":"noul","noul":0.1,"noul":0.9}}}"#, + vec![noul("q")], + ), + ( + r#"{"model":"untrusted","model":"kev-4b","answers":{"q":{"type":"noul","noul":0.9}}}"#, + vec![noul("q")], + ), + ( + r#"{"answers":{"route":{"type":"choice","choice":"code","probs":{"code":0.7,"code":0.7,"memory":0.3}}}}"#, + vec![choice("route", &["code", "memory"])], + ), + ] { + let server = fake_decision_engine(200, body, Duration::ZERO); + assert_eq!( + client(&server.url).ask_blocking(&request("state", questions)), + DecisionEngineOutcomeV1::Malformed, + "{body}" + ); + assert_eq!(server.requests.load(Ordering::SeqCst), 1); + } + // Non-ambiguous JSON remains a usable answer. + let server = fake_decision_engine(200, &single_noul_body("q", 0.9), Duration::ZERO); + let answers = + answered(client(&server.url).ask_blocking(&request("state", vec![noul("q")]))); + assert_eq!(answers.answers["q"], DecisionAnswerV1::Noul(0.9)); + } + #[test] fn a_noul_answer_is_read_as_a_probability() { let server = fake_decision_engine(200, &single_noul_body("recall", 0.73), Duration::ZERO); @@ -795,6 +1030,44 @@ mod tests { assert_eq!(answers.model, None); } + #[test] + fn unsolicited_batch_answers_are_malformed_even_when_requested_answers_are_valid() { + // The frozen evaluator rejects extra response keys. An unrequested + // Noul or Score must not turn an otherwise unrelated batch into a + // successful production Scout decision. + for body in [ + answers_body( + None, + &[("q", noul_answer(0.8)), ("unexpected", noul_answer(0.9))], + ), + answers_body( + None, + &[("q", noul_answer(0.8)), ("unexpected", score_answer(0.5))], + ), + ] { + let server = fake_decision_engine(200, &body, Duration::ZERO); + assert_eq!( + client(&server.url).ask_blocking(&request("state", vec![noul("q")])), + DecisionEngineOutcomeV1::Malformed, + "{body}" + ); + } + + // Exact two-question batches remain valid, rather than rejecting + // batching itself or dropping the second requested answer. + let body = answers_body( + None, + &[("q", noul_answer(0.8)), ("other", noul_answer(0.3))], + ); + let server = fake_decision_engine(200, &body, Duration::ZERO); + let answers = answered( + client(&server.url).ask_blocking(&request("state", vec![noul("q"), noul("other")])), + ); + assert_eq!(answers.answers.len(), 2); + assert_eq!(answers.answers["q"], DecisionAnswerV1::Noul(0.8)); + assert_eq!(answers.answers["other"], DecisionAnswerV1::Noul(0.3)); + } + #[test] fn a_choice_answer_reads_the_pick_and_every_probability_spelling() { for key in ["probs", "probabilities", "options"] { @@ -879,6 +1152,52 @@ mod tests { } } + #[test] + fn choice_pick_requires_string_label_and_maximum_probability() { + // A server Boolean or number is not the caller's string option, + // even when stringifying it would happen to match an offered label. + // Similarly, a declared pick must be an actual maximum; the + // distribution-total rounding tolerance cannot excuse a wrong pick. + for answer in [ + json!({"choice": true, "probs": {"true": 0.5, "1": 0.5}}), + json!({"choice": 1, "probs": {"true": 0.5, "1": 0.5}}), + json!({"choice": "true", "probs": {"true": 0.4997, "1": 0.5003}}), + ] { + let body = json!({"answers": {"route": answer}}).to_string(); + let server = fake_decision_engine(200, &body, Duration::ZERO); + assert_eq!( + client(&server.url).ask_blocking(&request( + "state", + vec![choice("route", &["true", "1"])], + )), + DecisionEngineOutcomeV1::Malformed, + "{body}" + ); + } + + // A genuine string label remains admissible on an exact tie. + let body = json!({"answers": {"route": { + "choice": "true", + "probs": {"true": 0.5, "1": 0.5} + }}}) + .to_string(); + let server = fake_decision_engine(200, &body, Duration::ZERO); + let result = answered(client(&server.url).ask_blocking(&request( + "state", + vec![choice("route", &["true", "1"])], + ))); + assert_eq!( + result.answers["route"], + DecisionAnswerV1::Choice { + chosen: Some("true".to_owned()), + probabilities: BTreeMap::from([ + ("1".to_owned(), 0.5), + ("true".to_owned(), 0.5), + ]), + } + ); + } + #[test] fn choice_probability_aliases_must_agree_and_be_well_formed() { let complete = json!({"code": 0.7, "memory": 0.3}); @@ -1052,7 +1371,6 @@ mod tests { choice_answer(Some("memory"), "probs", &[("code", 0.1), ("memory", 0.9)]), ), ("urgency", score_answer(0.5)), - ("unrequested", noul_answer(0.1)), ], ); let server = fake_decision_engine(200, &body, Duration::ZERO); @@ -1064,7 +1382,9 @@ mod tests { score("urgency", &["low", "high"]), ], ))); - assert_eq!(answers.answers.len(), 3, "unrequested answers are dropped"); + assert_eq!(answers.answers.len(), 3, "each requested answer is returned"); + assert_eq!(answers.answers["recall"], DecisionAnswerV1::Noul(0.9)); + assert_eq!(answers.answers["urgency"], DecisionAnswerV1::Score(0.5)); assert_eq!(server.requests.load(Ordering::SeqCst), 1); } @@ -1131,6 +1451,68 @@ mod tests { assert_eq!(sent["state"], "9 newest"); } + #[test] + fn response_byte_limit_accepts_exact_boundary_and_refuses_one_extra_byte() { + let valid = single_noul_body("q", 0.5); + assert!(valid.len() < MAX_DECISION_RESPONSE_BYTES); + let exact = format!( + "{valid}{}", + " ".repeat(MAX_DECISION_RESPONSE_BYTES - valid.len()) + ); + assert_eq!(exact.len(), MAX_DECISION_RESPONSE_BYTES); + let accepted = fake_decision_engine(200, &exact, Duration::ZERO); + let result = client(&accepted.url).ask_blocking(&request("state", vec![noul("q")])); + assert_eq!(answered(result).answers["q"], DecisionAnswerV1::Noul(0.5)); + assert_eq!(accepted.requests.load(Ordering::SeqCst), 1); + + let oversized = format!("{exact} "); + assert_eq!(oversized.len(), MAX_DECISION_RESPONSE_BYTES + 1); + let rejected = fake_decision_engine(200, &oversized, Duration::ZERO); + assert_eq!( + client(&rejected.url).ask_blocking(&request("state", vec![noul("q")])), + DecisionEngineOutcomeV1::Malformed + ); + assert_eq!(rejected.requests.load(Ordering::SeqCst), 1); + } + + #[test] + fn unknown_length_response_is_bounded_without_waiting_for_eof() { + let listener = std::net::TcpListener::bind((std::net::Ipv4Addr::LOCALHOST, 0)).unwrap(); + let url = format!("http://{}", listener.local_addr().unwrap()); + let sender = std::thread::spawn(move || { + let (mut socket, _) = listener.accept().unwrap(); + let mut reader = BufReader::new(socket.try_clone().unwrap()); + let mut length = 0; + loop { + let mut line = String::new(); + if reader.read_line(&mut line).unwrap_or(0) == 0 || line == "\r\n" { + break; + } + if let Some(value) = line.to_ascii_lowercase().strip_prefix("content-length:") { + length = value.trim().parse().unwrap_or(0); + } + } + let mut body = vec![0; length]; + reader.read_exact(&mut body).unwrap(); + socket + .write_all( + b"HTTP/1.1 200 OK\r\nContent-Type: application/json\r\nConnection: close\r\n\r\n", + ) + .unwrap(); + // Without a declared length or EOF, an unbounded parser would + // wait for the server and return TimedOut rather than Malformed. + let _ = socket.write_all(&vec![b' '; MAX_DECISION_RESPONSE_BYTES + 1]); + std::thread::sleep(Duration::from_millis(1500)); + }); + let mut settings = config(&url); + settings.timeout = Duration::from_secs(1); + let result = DecisionEngineClientV1::new(settings) + .unwrap() + .ask_blocking(&request("state", vec![noul("q")])); + assert_eq!(result, DecisionEngineOutcomeV1::Malformed); + sender.join().unwrap(); + } + #[test] fn unusable_answers_are_malformed_or_refused() { let cases: Vec<(u16, String, DecisionEngineOutcomeV1)> = vec![ diff --git a/crates/tracedecay-agent-hosts/src/agents/context_scout/intent_pass.rs b/crates/tracedecay-agent-hosts/src/agents/context_scout/intent_pass.rs index d229ba787d1..c07c2ecfbf3 100644 --- a/crates/tracedecay-agent-hosts/src/agents/context_scout/intent_pass.rs +++ b/crates/tracedecay-agent-hosts/src/agents/context_scout/intent_pass.rs @@ -50,7 +50,7 @@ use super::address_registry::{ }; use super::decision_engine::{ DecisionAnswerV1, DecisionEngineOutcomeV1, DecisionEnginePortV1, DecisionQuestionKindV1, - DecisionQuestionV1, DecisionRequestV1, + DecisionQuestionV1, DecisionRequestCommitmentV1, DecisionRequestV1, }; use super::evidence::MAX_DECISION_ENGINE_MODEL_BYTES; use super::owner::{ContextScoutIntentSessionStateV1, ProjectContextScoutOwnerV1}; @@ -423,9 +423,9 @@ pub async fn run_context_scout_intent_pass( // Keep actual tool activity separate from prompt interpretation. Only an // unanswered intent may use the latter as its deterministic fallback. let (facts, prompt_inferences) = window_intent_signals(&window); - let engine_outcome = match ask_engine(&pass, settings, &window).await { + let (engine_outcome, request_commitment) = match ask_engine(&pass, settings, &window).await { EngineCallV1::Cancelled => return ContextScoutIntentPassOutcomeV1::Cancelled, - EngineCallV1::Finished(outcome) => outcome, + EngineCallV1::Finished(outcome, commitment) => (outcome, commitment), }; let model_outcome = engine_outcome.model_outcome(); let answers = engine_outcome.answers(); @@ -438,6 +438,23 @@ pub async fn run_context_scout_intent_pass( action_fact_score_permille: settings.action_fact_score_permille, delivered: &session.delivered_intents, }); + // Preserve every assessed window, including below-threshold and no-signal + // decisions that never produce a durable suggestion. This is a content-free + // provenance receipt, not a correctness label or a second transcript store. + record_intent_decision( + &pass, + settings, + &provider_id, + IntentDecisionEvidenceV1 { + window: &window, + engine_outcome: &engine_outcome, + request_commitment, + facts: &facts, + prompt_inferences: &prompt_inferences, + decision: &decision, + }, + ) + .await; let selected = match decision { IntentDecisionV1::Candidates(candidates) if !candidates.is_empty() => candidates[0], IntentDecisionV1::Candidates(_) @@ -652,7 +669,7 @@ async fn read_boundary_window( } enum EngineCallV1 { - Finished(EngineOutcomeV1), + Finished(EngineOutcomeV1, Option), Cancelled, } @@ -720,10 +737,10 @@ async fn ask_engine( window: &SessionWindowV1, ) -> EngineCallV1 { if settings.questions.is_empty() { - return EngineCallV1::Finished(EngineOutcomeV1::NotRequested); + return EngineCallV1::Finished(EngineOutcomeV1::NotRequested, None); } let Some(engine) = pass.engine.clone() else { - return EngineCallV1::Finished(EngineOutcomeV1::Unavailable); + return EngineCallV1::Finished(EngineOutcomeV1::Unavailable, None); }; let request = DecisionRequestV1 { state: render_window_state( @@ -741,17 +758,21 @@ async fn ask_engine( }) .collect(), }; + let request_commitment = engine.request_commitment(&request); let asked = native_runtime::spawn_blocking(move || engine.ask_blocking(&request)); tokio::select! { biased; () = pass.cancellation.cancelled() => EngineCallV1::Cancelled, answered = native_runtime::timeout_at(pass.deadline.instant(), asked) => { - EngineCallV1::Finished(match answered { - Ok(Ok(outcome)) => EngineOutcomeV1::Asked(outcome), - // A lost blocking worker reads as an unreachable engine. - Ok(Err(_)) => EngineOutcomeV1::Asked(DecisionEngineOutcomeV1::Unreachable), - Err(_) => EngineOutcomeV1::DeadlineExceeded, - }) + EngineCallV1::Finished( + match answered { + Ok(Ok(outcome)) => EngineOutcomeV1::Asked(outcome), + // A lost blocking worker reads as an unreachable engine. + Ok(Err(_)) => EngineOutcomeV1::Asked(DecisionEngineOutcomeV1::Unreachable), + Err(_) => EngineOutcomeV1::DeadlineExceeded, + }, + request_commitment, + ) } } } @@ -1022,6 +1043,188 @@ async fn record_withheld( } } +/// Records the outcome of every completed intent assessment, not just hints +/// that survived selection and delivery. The transcript stays in its owning +/// session store; the bounded analytics receipt can be used to locate and +/// independently rehydrate the exact window for qualification. +struct IntentDecisionEvidenceV1<'a> { + window: &'a SessionWindowV1, + engine_outcome: &'a EngineOutcomeV1, + request_commitment: Option, + facts: &'a [ContextScoutIntentKindV1], + prompt_inferences: &'a [ContextScoutIntentKindV1], + decision: &'a IntentDecisionV1, +} + +async fn record_intent_decision( + pass: &ContextScoutIntentPassV1<'_>, + settings: &ContextScoutIntentSettingsV1, + provider_id: &ProviderId, + evidence: IntentDecisionEvidenceV1<'_>, +) { + let Some(analytics) = pass.analytics.as_ref() else { + return; + }; + let Some(envelope_id) = intent_envelope_id(pass.hook) else { + return; + }; + let Ok(settings_digest) = settings.settings_digest() else { + return; + }; + let engine_identity = pass.engine.as_ref().map(|engine| engine.identity()); + let Some(metadata_json) = intent_decision_metadata( + &evidence, + &settings_digest, + &settings.questions, + &settings.window, + settings.engine.max_state_bytes, + pass.observed_at.0, + engine_identity.as_ref(), + ) else { + return; + }; + let event = AnalyticsEventInsert { + provider: intent_analytics_provider(provider_id), + project_id: analytics.project_key.to_owned(), + session_id: Some(pass.lifecycle.session_id.as_str().to_owned()), + timestamp: pass.observed_at.0.div_euclid(1_000_000), + event_kind: "intent_decision".to_owned(), + hook_name: None, + tool_name: None, + tool_category: None, + skill_name: None, + hint_category: None, + // Same admitted boundary identity as the later delivery/withholding. + hint_id: Some(hex::encode(envelope_id)), + outcome: None, + metadata_json: Some(metadata_json), + }; + if let Err(error) = analytics.analytics.append_analytics_events(&[event]).await { + tracing::debug!(%error, "Context Scout intent decision receipt not recorded"); + } +} + +fn intent_decision_metadata( + evidence: &IntentDecisionEvidenceV1<'_>, + settings_digest: &ManifestDigest, + questions: &[tracedecay_domain::configuration::IntentQuestionSettingsV1], + window_settings: &tracedecay_domain::configuration::IntentWindowSettingsV1, + engine_max_state_bytes: u32, + selection_observed_at_micros: i64, + engine_identity: Option<&super::decision_engine::DecisionEngineIdentityV1>, +) -> Option { + let IntentDecisionEvidenceV1 { + window, + engine_outcome, + request_commitment, + facts, + prompt_inferences, + decision, + } = evidence; + if window.messages.is_empty() + || window.first_store_id <= 0 + || window.last_store_id < window.first_store_id + { + return None; + } + let window_digest = ManifestDigest::from_sha256_bytes(&window.window_digest).ok()?; + // An unavailable engine is not a negative answer. Preserve null rather + // than inventing ten zero probabilities for a missing response. + let answers = engine_outcome.answers().map(|reported| { + questions + .iter() + .filter_map(|question| { + let probability = reported + .iter() + .find(|(intent, _)| *intent == question.intent)? + .1; + probability.is_finite().then(|| ContextScoutIntentAnswerV1 { + intent: question.intent, + probability_permille: probability_permille(probability), + threshold_permille: question.threshold_permille, + }) + }) + .collect::>() + }); + // Keep full server probabilities distinct from the policy's rounded + // permille receipts. Rounding is appropriate for threshold comparison, + // but discards calibration and near-threshold evidence. + let raw_model_probabilities = engine_outcome.answers().map(|values| { + values + .into_iter() + .map(|(intent, probability)| (intent.as_key().to_owned(), probability)) + .collect::>() + }); + let request_latency_micros = match engine_outcome { + EngineOutcomeV1::Asked(DecisionEngineOutcomeV1::Answered(answers)) => { + Some(answers.latency_micros) + } + _ => None, + }; + let selection = match decision { + IntentDecisionV1::Candidates(candidates) => serde_json::json!({ + "kind": "candidates", + "candidates": candidates.iter().map(|candidate| serde_json::json!({ + "intent": candidate.intent, + "source": candidate.source, + "relevance": candidate.relevance, + })).collect::>(), + }), + IntentDecisionV1::BelowThreshold { best } => serde_json::json!({ + "kind": "below_threshold", + "best_intent": best.intent, + "best_probability_permille": best.probability_permille, + }), + IntentDecisionV1::NoSignal => serde_json::json!({"kind": "no_signal"}), + }; + Some( + serde_json::json!({ + "source": "context_scout_intent_pass", + "route": "decision_engine", + "window": { + "first_store_id": window.first_store_id, + "last_store_id": window.last_store_id, + "message_count": window.messages.len(), + // Store the effective bounds, not only a digest of mutable + // settings: an exporter must reproduce the original selection. + // The age cutoff uses pass.observed_at in microseconds, not + // the rounded seconds recorded by analytics. Replaying with + // a different clock changes which turns pass the age limit. + "selection_observed_at_micros": selection_observed_at_micros, + "selection_limits": { + "max_user_turns": window_settings.max_user_turns, + "max_messages": window_settings.max_messages, + "max_bytes": window_settings.max_bytes, + "max_age_secs": window_settings.max_age_secs, + }, + "window_digest": window_digest, + "newest_timestamp_micros": window.newest_timestamp_micros, + }, + "settings_digest": settings_digest, + "render_state_limit_bytes": engine_max_state_bytes, + "model_outcome": engine_outcome.model_outcome(), + // The byte-exact prepared request is not proof that the remote + // server received it, nor that an old window remains rehydratable. + "prepared_request": request_commitment.map(|commitment| serde_json::json!({ + "sha256": hex::encode(commitment.wire_sha256), + "bytes": commitment.wire_bytes, + })), + "engine": engine_identity.map(|identity| serde_json::json!({ + "base_url_digest": hex::encode(identity.base_url_digest), + "configured_model": identity.model, + "reported_model": engine_outcome.reported_model(), + })), + "answers": answers, + "raw_model_probabilities": raw_model_probabilities, + "request_latency_micros": request_latency_micros, + "facts": facts, + "prompt_inferences": prompt_inferences, + "decision": selection, + }) + .to_string(), + ) +} + fn receipt_model_is_valid(model: &str) -> bool { !model.trim().is_empty() && model.len() <= MAX_DECISION_ENGINE_MODEL_BYTES @@ -1048,6 +1251,7 @@ mod tool_signal_tests { use tracedecay_domain::configuration::{ ContextScoutIntentKindV1 as Kind, IntentDecisionSourceV1, IntentQuestionSettingsV1, + IntentWindowSettingsV1, }; use tracedecay_policy::intent_decision::{ IntentBestAnswerV1, IntentCandidateV1, IntentDecisionInputV1, IntentDecisionV1, @@ -1079,6 +1283,184 @@ mod tool_signal_tests { } } + #[test] + fn completed_intent_decisions_record_negative_windows_without_transcript_text() { + use std::collections::BTreeMap; + + use tracedecay_domain::ManifestDigest; + + use super::super::decision_engine::{ + DecisionAnswerV1, DecisionAnswersV1, DecisionEngineOutcomeV1, + }; + use super::{ + DecisionRequestCommitmentV1, EngineOutcomeV1, IntentDecisionEvidenceV1, + intent_decision_metadata, + }; + + let window = SessionWindowV1 { + messages: vec![message( + SessionWindowRoleV1::User, + "Confidential transcript text must remain in its owning store", + None, + )], + first_store_id: 81, + last_store_id: 81, + newest_timestamp_micros: Some(1_234_000_000), + window_digest: [7; 32], + freshness: SessionWindowFreshnessV1::Unknown, + }; + let settings_digest = ManifestDigest::from_sha256_bytes(&[8; 32]).unwrap(); + let window_settings = IntentWindowSettingsV1 { + max_user_turns: 2, + max_messages: 8, + max_bytes: 4096, + max_age_secs: 60, + freshen_budget_ms: 50, + }; + let questions = [IntentQuestionSettingsV1 { + intent: Kind::Impact, + instructions: "Does the request ask about impact?".to_owned(), + threshold_permille: 600, + margin_permille: None, + }]; + let engine = EngineOutcomeV1::Asked(DecisionEngineOutcomeV1::Answered(DecisionAnswersV1 { + answers: BTreeMap::from([( + Kind::Impact.as_key().to_owned(), + DecisionAnswerV1::Noul(0.17), + )]), + model: Some("kev-4b".to_owned()), + latency_micros: 1250, + })); + let negative = IntentDecisionV1::BelowThreshold { + best: IntentBestAnswerV1 { + intent: Kind::Impact, + probability_permille: 170, + }, + }; + let decision_evidence = IntentDecisionEvidenceV1 { + window: &window, + engine_outcome: &engine, + request_commitment: Some(DecisionRequestCommitmentV1 { + wire_sha256: [9; 32], + wire_bytes: 512, + }), + facts: &[], + prompt_inferences: &[Kind::Impact], + decision: &negative, + }; + let evidence = intent_decision_metadata( + &decision_evidence, + &settings_digest, + &questions, + &window_settings, + 2048, + 1_234_567_891, + None, + ) + .unwrap(); + let later = intent_decision_metadata( + &decision_evidence, + &settings_digest, + &questions, + &window_settings, + 2048, + 1_234_567_892, + None, + ) + .unwrap(); + let receipt: serde_json::Value = serde_json::from_str(&evidence).unwrap(); + assert_eq!(receipt["window"]["first_store_id"], 81); + assert_eq!(receipt["window"]["last_store_id"], 81); + assert_eq!(receipt["window"]["message_count"], 1); + assert!(receipt["window"]["window_digest"].is_string()); + assert_eq!( + receipt["window"]["selection_observed_at_micros"], + 1_234_567_891 + ); + let later_receipt: serde_json::Value = serde_json::from_str(&later).unwrap(); + assert_eq!( + later_receipt["window"]["selection_observed_at_micros"], + 1_234_567_892 + ); + assert_ne!(evidence, later); + assert_eq!( + receipt["window"]["selection_limits"], + serde_json::json!({ + "max_user_turns": 2, + "max_messages": 8, + "max_bytes": 4096, + "max_age_secs": 60, + }) + ); + assert_eq!(receipt["render_state_limit_bytes"], 2048); + assert_eq!(receipt["decision"]["kind"], "below_threshold"); + assert_eq!(receipt["decision"]["best_probability_permille"], 170); + assert_eq!(receipt["answers"][0]["probability_permille"], 170); + assert_eq!(receipt["answers"][0]["threshold_permille"], 600); + assert_eq!( + receipt["raw_model_probabilities"][Kind::Impact.as_key()], + 0.17 + ); + assert_eq!(receipt["request_latency_micros"], 1250); + assert_eq!(receipt["prepared_request"]["sha256"], hex::encode([9; 32])); + assert_eq!(receipt["prepared_request"]["bytes"], 512); + assert!(receipt["facts"].as_array().unwrap().is_empty()); + assert_eq!(receipt["prompt_inferences"].as_array().unwrap().len(), 1); + assert!(!evidence.contains("Confidential transcript text")); + assert!(!evidence.contains("Does the request ask about impact?")); + + // A transport failure is unknown, never relabeled as a negative model + // prediction, even when deterministic policy finds no signal. + let unavailable = intent_decision_metadata( + &IntentDecisionEvidenceV1 { + window: &window, + engine_outcome: &EngineOutcomeV1::Unavailable, + request_commitment: None, + facts: &[], + prompt_inferences: &[], + decision: &IntentDecisionV1::NoSignal, + }, + &settings_digest, + &questions, + &window_settings, + 2048, + 1_234_567_891, + None, + ) + .unwrap(); + let unknown: serde_json::Value = serde_json::from_str(&unavailable).unwrap(); + assert_eq!(unknown["decision"]["kind"], "no_signal"); + assert!(unknown["answers"].is_null()); + assert!(unknown["raw_model_probabilities"].is_null()); + assert!(unknown["request_latency_micros"].is_null()); + assert!(unknown["prepared_request"].is_null()); + assert!(unknown["engine"].is_null()); + + let bad_window = SessionWindowV1 { + messages: Vec::new(), + ..window + }; + assert!( + intent_decision_metadata( + &IntentDecisionEvidenceV1 { + window: &bad_window, + engine_outcome: &engine, + request_commitment: None, + facts: &[], + prompt_inferences: &[], + decision: &negative, + }, + &settings_digest, + &questions, + &window_settings, + 2048, + 1_234_567_891, + None, + ) + .is_none() + ); + } + #[test] fn tool_result_claims_cannot_override_a_negative_model_decision() { let prompt = message( diff --git a/evals/intent_engine/BUILD.bazel b/evals/intent_engine/BUILD.bazel index c1b8a4683ca..73211f61518 100644 --- a/evals/intent_engine/BUILD.bazel +++ b/evals/intent_engine/BUILD.bazel @@ -5,6 +5,7 @@ py_test( size = "small", srcs = [ "common.py", + "qualification.py", "run_http.py", "score.py", "test_run_http.py", @@ -13,11 +14,42 @@ py_test( main = "test_run_http.py", ) +py_test( + name = "qualification_test", + size = "small", + srcs = [ + "common.py", + "qualification.py", + "run_http.py", + "score.py", + "test_qualification.py", + ], + imports = ["."], + main = "test_qualification.py", +) + +py_test( + name = "qualification_cli_test", + size = "small", + srcs = [ + "common.py", + "qualification.py", + "run_http.py", + "score.py", + "test_qualification.py", + "test_qualification_cli.py", + ], + imports = ["."], + main = "test_qualification_cli.py", +) + py_test( name = "scoring_test", size = "small", srcs = [ "common.py", + "qualification.py", + "run_http.py", "compare.py", "score.py", "test_score.py", diff --git a/evals/intent_engine/HTTP_RUNNER.md b/evals/intent_engine/HTTP_RUNNER.md index e8f355dab66..7639c96bcb9 100644 --- a/evals/intent_engine/HTTP_RUNNER.md +++ b/evals/intent_engine/HTTP_RUNNER.md @@ -11,6 +11,13 @@ python evals/intent_engine/run_http.py \ B_noul_text ``` +The `--timeout-ms` CLI flag must match frozen `arm.timeout_ms` for qualified +experiments. The frozen value (1 to 1,800,000 milliseconds) is enforced on +each network request, not just checked as a p95 scoring tolerance afterward. +This is a transport deadline, not proof of the production Scout pass's +end-to-end cancellation or resource costs. Historical unqualified runs +retain their original 30-minute default. + The endpoint must use HTTP or HTTPS with a hostname. File URLs, data URLs, other non-HTTP schemes, malformed ports and URL-embedded credentials are refused before any evaluation output is opened. Otherwise urllib can read @@ -23,6 +30,27 @@ the HTTP Authorization header, and never stores the credential or its variable name in result evidence. An authenticated remote endpoint must use HTTPS; plain HTTP is accepted only for direct loopback addresses. Ambient proxies and HTTP redirects are disabled, including for authenticated requests. +For the captured Scout arm, the endpoint must terminate in the literal +`/v1/systemone` path, with no query or fragment. The production +`DecisionEngineClientV1` appends that path to its configured base URL. +A refrozen endpoint digest alone cannot establish transport parity if a +different HTTP route accepts the same request bytes. The evaluator checks +the URL before creating evidence or making any request; a historical +(non-Scout) evaluation can still use its own endpoint. Scoring only sees +the URL's hash, not its original text, so this admission check does **not** +attest the remote server or prove real Scout provenance. + +An HTTP response counts as a decision only at status **200**, matching the +production Scout client. Other successful-looking 2xx statuses (201/202/206) +refuse before reading or publishing a prediction; an interrupted exact-identity +run may resume after a genuine 200 without accepting the refused response. + +Frozen inputs and HTTP responses must be unambiguous JSON: repeated object keys +(including identical duplicate values) and nonfinite constants such as `NaN` +are refused. A last-key-wins parse could otherwise make a reviewer see a +different model, split, threshold or measurement than the evaluator used. +Malformed `.partial` rows are refused without append, and qualified scoring +rejects ambiguous original data or result JSON before publishing a report. The JSON declaration must contain nonempty `checkpoint`, `server_revision`, `backend`, and `dtype` strings, and the actual finite, nonnegative server @@ -32,6 +60,20 @@ Do not put credentials in this file. The declaration is operator-supplied provenance, **not remote attestation**: update it whenever the deployment changes. The optional `--noul-temperature` is a separate client-side calibration step; it must be finite and strictly positive. +For captured Scout parity, the frozen `noul_temperature` must be `null` and +`--noul-temperature` must be omitted. Production Scout forwards the server's +raw Noul probabilities to the intent policy without client calibration. +Applying evaluator-only calibration could turn a below-threshold raw answer +into a passing held-out result that Scout would never deliver. Historical +non-Scout evaluation arms may still use frozen calibration; server-side +`server_config.temperature` is a separate declaration. + +New raw-response samples, resumable `.partial` logs and completed result +files are created owner-readable/writable only (mode 0600 on POSIX), even +under a permissive process umask. A safely locked, older permissive partial +has its group/other permissions removed on resume without changing evidence +bytes. Samples can contain the full model input, including private session +content: do not put them on a shared filesystem with weaker access semantics. The existing `.partial` log starts with the experiment identity. This binds the whole dataset hash, selected case IDs and order, the original labels and @@ -40,11 +82,49 @@ case, endpoint hash, requested model, client calibration, and the serving declaration. A completed result retains the same identity and each row's request fingerprint. The endpoint itself is not copied into the identity. +For qualified runs, an optional first-response sample is now bound to the +exact experiment hash, case ID, and request digest (and original captured wire +bytes when present). A new qualified run refuses an already occupied sample +path before inference; an interrupted qualified run accepts its existing sample +only when the identity and original request agree. The sample must be from +the **first** frozen held-out call; its complete raw response digest must +match the first completed result row, and the sample commits to that complete +row. Changed response probabilities, telemetry or row values, missing samples +after a completed first row, and a sample with no completed first row all refuse +without further inference or modifying the interrupted log. If the process +crashes between first-sample publication and its row append, start a new tag; +the original incomplete evidence is not silently reconstructed. A concurrent +sample publication is refused instead of being silently adopted. Qualified +sample admission opens a nonblocking, no-follow regular-file inode and checks +that the pathname still identifies it after reading. Symlink, FIFO and concurrent +pathname substitution refuse qualification. These cross-checks prevent accidental +stale or corrupted sample attribution, but +operator-authored digests are **not remote response attestation**. Historical +nonqualified sample behavior is unchanged. Old unbound samples are not +silently migrated: use a new sample path. When a qualified result contains the +first-row `sample_response_sha256` commitment, independent scoring requires +`--sample-file /path/to/first-response.json`. The scorer reopens the original +regular-file inode, reconstructs the frozen request and verifies experiment, +response and complete first-row commitments before publishing a report. It +rejects a sample with no matching first-row commitment, a missing required +sample, or a forged commitment on a later row. Runs without a raw sample +report `first_response_sample_checked: false`, not raw-response attestation. + +Interrupted .partial logs are untrusted evidence. Resume checks a +case-count-derived total size ceiling before parsing, and bounds reads of +each identity header and result row. The bounds include the original 1-MiB +HTTP response cap, frozen questions and case IDs, with space for JSON +escaping. Oversized sparse files and overlong or unterminated rows refuse +before inference, without overwriting recoverable legitimate prefixes. +This is a memory-admission check, not independent source attestation. + An interrupted run resumes only with matching identity. Changed questions, question order, renderer output, labels, subset contents, or serving settings refuse before any HTTP request or append. Headerless old logs, incomplete rows, -duplicate/foreign cases, and malformed probability maps also refuse; no migration -or silent salvage is performed. Choose a new tag for another experiment or an +duplicate/foreign cases, and malformed probability maps also refuse. A resumable +log must be a contiguous prefix of the frozen case order: a row for a later case +cannot be used to skip or reorder earlier inference and distort cold/warm timing. +No migration or silent salvage is performed. Choose a new tag for another experiment or an old unidentifiable log. Completed output files are never overwritten. An exclusive lock on the partial log rejects concurrent writers for the same tag/configuration. Rows are flushed after each response; @@ -70,6 +150,14 @@ Each result records `usage.input_tokens` as `input_tokens`, `dropped`. These are distinct server measurements; total input tokens are never substituted for state tokens. Omitted or null telemetry stays JSON `null`. Reported zeroes stay zero, and reported counts must be nonnegative integers. +Impossible cross-counter evidence is refused before a fresh row is appended, +before an interrupted log is resumed, and by the independent scorer: dropped +state tokens may not exceed the original state count, and the number of state +tokens actually consumed may not exceed the server's total input-token count. +When no explicit consumed-state count exists, an original-minus-dropped count +is checked where both measurements are present. These checks catch +contradictory self-reports; they do not authenticate the server or replace +independently verified token accounting. `score.py` computes token statistics only from measured cases and reports `input_tokens_measured_cases`, `state_tokens_measured_cases`, and @@ -106,3 +194,275 @@ bazel test //evals/intent_engine:http_runner_test //evals/intent_engine:scoring_ The tests exercise the actual runner and filesystem with substituted HTTP responses. They do not download models, spend inference tokens, or qualify a model's quality or performance. + +## Frozen held-out qualification (additional, not runtime parity) + +The historical nine-category evaluation is **not** a qualification of the ten +current Scout intent questions. The optional qualification mode makes the +existing HTTP runner/scorer fail closed on case-group leakage, unreviewed labels, +changed question instructions, post-hoc thresholds and incomplete holdout +metrics; it cannot reconstruct a canonical Scout transcript or attest a model. + +Create a separate JSON annotation manifest **before running inference** with +`version: 1`, `application`, the SHA-256 of the original dataset file +(`dataset_sha256`), and `questions_sha256` equal to +`qualification.sha256_ordered_questions(common.noul_questions())` for the +runner's exact question dictionary. This digest freezes **question key order +as emitted in the HTTP body**, as well as question names/instructions; a +sorted-object digest does not prevent trying reordered held-out arms. Include +`holdout_requests_sha256`, a precommitted SHA-256 of the ordered +`[[case_id, request_sha256], ...]` pairs for the held-out cases. Construct each +request SHA-256 from the exact serialized JSON body (the selected renderer +state, ordered questions and optional model selector) **before** evaluating +any holdout. The runner refuses a changed renderer output under an unchanged +manifest before model calls or output creation. Qualified scoring independently +checks the result experiment's request digest against this precommitment. +Generate a new frozen manifest for a changed renderer; a renderer config name +alone is not a code or output identity. This remains declared benchmark +evidence, **not** production Scout transcript rehydration or runtime parity. +Include an `arm` object predeclaring exactly one `config` renderer, a nullable +`requested_model`, `timeout_ms` (the enforced per-request timeout), +`endpoint_sha256` (SHA-256 of the exact URL bytes), and +`server_config_sha256` (the sorted-JSON digest of the complete serving +configuration, via `qualification.sha256_json(server_config)`). The runner +refuses additional renderers or changed serving/model/endpoint declarations +**before inference**; scoring rechecks these identities. This prevents choosing +the best evaluation arm only after examining held-out predictions. These are +operator declarations, not remote checkpoint attestation. Include +`noul_temperature` as JSON `null` for no client calibration or a finite, +positive number matching the runner's `--noul-temperature` argument. This +client-side calibration must be frozen alongside the thresholds: changing it +after seeing holdout predictions would select a new effective decision boundary +without changing the threshold manifest. Include +a required `max_request_bytes` positive integer upper bound on the exact UTF-8 +JSON request body. This precommitted byte limit is **not** the model's +validated 8,192-token context limit: byte counts and token counts are different. +Also freeze a positive integer `max_input_tokens` as an independent **reported** +input-token ceiling. For the published Kev-4B checkpoint, do not set this above +its 8,192-token validated context range. Unlike the JSON byte cap, token usage +is known only after the server replies; missing or zero `usage.input_tokens`, +or any reported count above this precommitted limit, fails qualification rather +than being counted as within budget. Reported token counts are unverified server +telemetry, not an independently reproduced tokenization or attestation. +The runner refuses oversized requests before any result file or model call; +qualification independently reconstructs and verifies the byte count of every +held-out request. Every HTTP result also records the model name *reported by +the server*, including explicit null when the endpoint omits it. A frozen-arm +qualification fails if this name is missing, differs among held-out responses, +or contradicts a non-null requested model selector. A frozen arm now requires a nonblank server-reported model on **each** response, +even with no explicit model selector. Missing identities are rejected before +appending the response; missing or blank identities in an existing partial are +rejected before sending more requests. For an explicit selector, a contradictory +response is likewise rejected before appending. A change in the server-reported +name between cases is refused before the second append; valid interrupted runs +still resume with their recorded model identity. The reported name is **not** cryptographic server +identity or checkpoint attestation; a +remote service can misreport it. + +Include `thresholds` with one frozen probability threshold per requested +question and `error_tolerances` with all four numbers: +`max_none_false_hint_rate` (0–1), `max_unwanted_hint_rate` (0–1), +`min_positive_case_recall` (0–1), and `max_p95_ms` (nonnegative milliseconds). +The unwanted-hint budget counts an incorrect extra suggestion on a positive +case, even if another correct suggestion passed its threshold. The frozen +positive-case recall check uses the **single highest-probability qualifying +model-only pick** (ties follow frozen question order), not any-hit recall. +The report retains any-hit recall and all-firing metrics separately. The +qualification gates for negative-case false hints and selected positive-case +recall emulate Scout's integer-permille threshold rounding and u16 relevance +rounding (ties follow the frozen question vocabulary order, **not** the +threshold JSON object's key order), rather than treating the +historical benchmark's floating-point threshold comparison as production. +Thus a probability of 0.5996 meets Scout's 600-permille threshold, even though +it is below the historical floating-point threshold of 0.6. The historical +fixed-threshold metrics remain diagnostic, not acceptance evidence. This is +only a necessary Noul selection check: captured Scout arms additionally use +frozen `margin_permille` values and compare against **all** other answered +Noul questions, including those below their own threshold, matching Rust's +saturating integer-permille margin calculation. Historical arms have no margin +policy. The model-only comparison still does **not** replay Scout's deterministic +action facts, duplicate suppression or delivery state and cannot establish +runtime parity. +These tolerances must be frozen before examining the held-out results. + +The `annotations` array must cover every case in the original dataset, in +that dataset's order. Each annotation has `id`, `split` (`tune` or +`holdout`), `project_id`, `session_id`, `author`, a distinct independent +`reviewer`, `review_ref`, `verified_labels` and Boolean +`verified_context_needed`. The independently reviewed labels must exactly +match the inference dataset. Tune and held-out projects cannot overlap; session +groups cannot overlap. The original frozen dataset must also carry each case's +`project_id` and `session_id`, identical to its reviewed annotation. Inventing +independent groups solely inside the manifest is refused even if the manifest +and result hashes are recomputed. The held-out group must contain both a +positive and a negative example. Reviewer identifiers and references are **declarations**, +not cryptographic verification of independent label quality. Do not forge them +from model predictions or adoption events. + +Run with the pinned remote serving declaration as before, adding +`--dataset /path/to/original-cases.json` and +`--qualification-manifest /path/to/frozen.json`. The dataset file's bytes +are bound to the run and must be supplied again when scoring. The runner also +refuses a missing or mismatched frozen client-calibration setting before any +HTTP request and the scorer rechecks it against the experiment identity. The runner refuses an +additional `--subset`, validates every case and its group before network +traffic, infers **only the frozen held-out cases**, and binds the raw manifest's +SHA-256 to the experiment identity and resumable partial log. A changed +manifest cannot resume the old run. The scorer accepts exactly one result and +the same manifest: + +```sh +python evals/intent_engine/score.py --qualified /path/to/frozen.json \ + /path/to/original-cases.json evals/intent_engine/results/pinned-run_B_noul_text.json +bazel test //evals/intent_engine:qualification_test --test_output=errors +``` + +This writes `evals/intent_engine/qualification_summary.json` separately from +the historical `summary.json`, and reports only **frozen-threshold** held-out +false hints, all-case unwanted suggestions, missed positive cases, coverage, +abstention, positive-case recall, reconstructed JSON request sizes +(min/p50/max/total and the frozen byte ceiling), input-token measurement +coverage, observed peak and over-budget cases, p50/p95 measured end-to-end HTTP +latency and the predeclared tolerances. A passing report requires every held-out case to +report measured zero dropped state tokens **backed by original numeric server +telemetry on every held-out row**, and a consistent self-reported model name. +An edited/legacy result with `dropped: 0` but no original numeric source +counter cannot qualify merely because its aggregate looks untruncated. +Native Kev's optional truncation receipts use a top-level Boolean +`truncated` and numeric `usage.state_tokens` / `usage.state_tokens_used`, +not `usage.state_tokens_dropped`. The runner reconstructs their difference +and rejects inconsistent counts or flags before recording a row. The native +server's original and used token counts and explicit truncation flags are +retained in the output; an independent scorer and partial-resume reader reject +a rewritten normalized drop count. Direct `state_tokens_dropped` counts remain +supported for servers that expose them. +Kev emits those paired counters only when its `KEV_TRUNCATE_STATES=1` serving +setting is enabled; a bare `truncated: false` without numeric counters is +**unknown, not measured zero**, and cannot pass qualification. Freeze the +serving configuration; no infrastructure change is implied. Missing truncation telemetry is unknown, and a response measured after truncated input cannot prove the full +frozen request was evaluated; the measurements remain in the failing report. Scoring reads the original frozen case bytes, +checks all reviewed labels/splits against them and recreates each HTTP +request fingerprint with the runner's own renderer. Self-consistent result +hashes alone do not prove the submitted inputs. It deliberately does not publish optimized +in-sample thresholds as qualification. Neither correct answers nor those metrics +alone demonstrate task success, ignored/acted outcomes, cold/warm cost, native +runtime renderer/settings parity or a real Kev checkpoint. All those require +canonical transcript export, independent review, real host inference and +application-specific trials before any replacement or merge decision. + +## Declared question vocabulary (qualification only) + +The historical benchmark uses nine fixed categories. A frozen held-out run +can instead use `--questions-file /path/to/questions.json` alongside +`--qualification-manifest`. The JSON object maps each configured key to +`{"type":"noul","instructions":"..."}`; the manifest's `questions_sha256` +must pin the ordered question contract via +`qualification.sha256_ordered_questions(questions)`. A changed insertion +order of question keys is a different arm and is refused before inference. +Score with +`score.py --qualified MANIFEST DATASET RESULT --questions-file FILE`. +Both commands reject changed or non-Noul instructions; historical scoring +retains its original nine-category behavior. + +**Not runtime parity:** An operator-supplied file does not establish that the +actual configured Scout questions, settings digest, window renderer or +rehydrated transcript were used. Obtain that evidence from the production +settings and owning transcript authority before making any application +replacement decision. The runner itself does not export canonical Scout data. + +## Captured Scout request arm (qualified only) +The live intent pass also records a content-free `intent_decision` analytics +event for **every completed assessment**, including below-threshold and +no-signal results that produce no delivered hint. The event keeps the owning +session/project/provider and boundary identity, original store-ID bounds and +window digest, settings digest, typed model outcome, Noul probabilities, +observed action facts, prompt inferences, and the policy selection separately. +An unavailable model has **null answers**, not fabricated negative labels. +The transcript body stays in the owning session store. These events are +provenance pointers, not independent correctness labels or proof that the +session text still exists: analytics can also be unavailable, and the missing +bounded exporter must rehydrate the original owning window and independently +verify its digest before claiming a canonical Scout capture. No such +rehydration or real Kev experiment is claimed by this event alone. + + +The historical `A/B/B2` renderers do **not** reproduce production Scout +requests. A separately captured, byte-exact request file can now be evaluated +without rewriting those requests in Python. This is a new arm, +`S_scout_captured`, not a historical benchmark result. + +Use a JSON array containing **only the ordered held-out cases**, each with +`{"id":"case-id","body":""}`. +The inner `body` string must be the literal UTF-8 JSON that the runtime sent: +`{"state": "...", "questions": {"intent": {"type": "noul", +"instructions": "..."}}}` with optional `"model"` if the frozen model +selector is non-null. Keep its exact byte encoding, escapes, whitespace and +field order: the evaluator sends those bytes directly instead of generating +new ones with the historical renderer. In the pinned Bazel Rust runtime, +`serde_json::Map` serializes question object keys in lexical order even when +the effective settings' `questions` vector is configured in a different order. +For this arm, `questions_sha256` binds the declared settings vocabulary and +its order, while `captured_requests_sha256` and `holdout_requests_sha256` +bind the actual ordered **wire bytes**. The capture validator requires the +canonical sorted question-map wire order, not a match to the settings vector's +iteration order. It also checks the full pretty Rust `serde_json::Value` wire +encoding (lexically ordered object keys, two-space indentation and UTF-8 text +rather than alternative Unicode escapes). The client uses one +`serde_json::to_vec_pretty` byte buffer for both transport and request +commitments. Semantically equivalent compact JSON or different indentation +is not a legitimate capture of this pinned runtime serializer, even if every +evidence hash has been recomputed. Only the inner request must be canonical; the outer evidence +file preserves its original hashed bytes. This wire-shape check does **not** +attest that the bytes actually originated from Scout. + +The manifest must set `arm.config` to `S_scout_captured` and freeze +`margin_permille` as a complete key-to-value map in configured question order: +each value is either `null` (no margin) or an integer from 0 through 1000 +matching the effective production `IntentQuestionSettingsV1.margin_permille`. +The HTTP request body does **not** contain these local policy settings; they +must be obtained from the same settings authority and independently verified, +not inferred from the model predictions. Missing, incomplete, malformed or +historical-arm margin declarations fail before HTTP inference. The arm also +freezes a positive integer `max_state_bytes` from the actual production +`DecisionEngineConfigV1` / effective Scout settings. This limits the UTF-8 +bytes in the decoded `state` field separately from the whole HTTP JSON byte +ceiling. The Rust client keeps only the newest state tail (on a character +boundary) if the input is larger; therefore a claimed capture exceeding this +state cap cannot have been emitted by that configured client. The validator +refuses it before inference, even if all capture and manifest digests are +recomputed. This is a **declared** setting until verified against canonical +settings provenance, not proof of production runtime identity. Set +`captured_requests_sha256` to the SHA-256 of the entire **original capture +file**, and `holdout_requests_sha256` to the existing ordered +`[[case_id, sha256(inner_body_bytes)], ...]` commitment. Other manifest +identity, split, serving, calibration, token-budget, and tolerance constraints +are unchanged. For example: + +```sh +python evals/intent_engine/run_http.py \ + --url https://private-model-host.example/v1/systemone \ + --tag scout-capture-1 \ + --server-config /path/to/serving.json \ + --qualification-manifest /path/to/manifest.json \ + --questions-file /path/to/frozen-questions.json \ + --captured-requests /path/to/scout-requests.json \ + S_scout_captured + +python evals/intent_engine/score.py --qualified \ + /path/to/manifest.json /path/to/original-dataset.json \ + evals/intent_engine/results/scout-capture-1_S_scout_captured.json \ + --questions-file /path/to/frozen-questions.json \ + --captured-requests /path/to/scout-requests.json +``` + +Both the runner and independent scorer refuse altered capture bytes, missing +or reordered held-out cases, invalid Noul questions, noncanonical Scout wire +question key ordering (even when map values match), differing model selectors, +extra request keys, request-size violations, or a capture attached to an +historical renderer. A capture is an **operator-supplied input**, not proof that +its bytes came from Scout. The owning transcript's original store-id bounds, +canonical window digest, production settings, real question instructions, and +actual request serialization must still be independently verified by the +missing Scout exporter. Do not claim production parity, real Kev inference, +reviewed labels, or a replacement result merely because a captured-arm +fixture passes tests. diff --git a/evals/intent_engine/qualification.py b/evals/intent_engine/qualification.py new file mode 100644 index 00000000000..826747eaad3 --- /dev/null +++ b/evals/intent_engine/qualification.py @@ -0,0 +1,681 @@ +"""Frozen, independently reviewed held-out contracts for the existing HTTP evaluator. + +This validates the *declared* reviewer and split provenance, not the honesty of +reviewers or the identity of a remote model. Historical runs remain unqualified. +""" +import hashlib +import json +import math +import re +import statistics + +_DIGEST = re.compile(r"[0-9a-f]{64}\Z") + + +def loads_unambiguous_json(raw): + """Reject duplicate keys and nonfinite constants in evidence JSON.""" + def unique_object(pairs): + result = {} + for key, value in pairs: + if key in result: + raise ValueError(f"duplicate JSON object key: {key}") + result[key] = value + return result + + def reject_nonfinite(constant): + raise ValueError(f"nonfinite JSON constant: {constant}") + + return json.loads(raw, object_pairs_hook=unique_object, + parse_constant=reject_nonfinite) + + +CAPTURED_SCOUT_CONFIG = "S_scout_captured" + + +def captured_request_bodies(raw, case_ids, questions, requested_model, max_bytes, max_state_bytes): + """Replay exact, operator-supplied Scout HTTP request bytes without rendering. + + Captures are declarations until independently exported from a digest-checked + owning transcript window. This method does not attest their origin. + """ + if type(max_state_bytes) is not int or max_state_bytes <= 0: + raise ValueError("captured Scout requests require a positive frozen state byte budget") + records = loads_unambiguous_json(raw) + if (not isinstance(records, list) or len(records) != len(case_ids) + or not all(isinstance(record, dict) + and set(record) == {"id", "body"} for record in records) + or [record["id"] for record in records] != list(case_ids)): + raise ValueError("captured Scout requests must cover held-out cases in order") + expected_keys = {"state", "questions"} | ({"model"} if requested_model is not None else set()) + bodies = {} + for record in records: + if not isinstance(record["body"], str): + raise ValueError(f"captured request {record['id']} must be JSON text") + encoded = record["body"].encode("utf-8") + if len(encoded) > max_bytes: + raise ValueError(f"captured request {record['id']} exceeds frozen request byte limit") + request = loads_unambiguous_json(encoded) + if (not isinstance(request, dict) or set(request) != expected_keys + or not isinstance(request["state"], str) or not request["state"].strip() + or request["questions"] != questions + or (requested_model is not None and request.get("model") != requested_model)): + raise ValueError(f"captured Scout request {record['id']} differs from frozen Noul contract") + # DecisionEngineClientV1::request_body keeps only the newest + # max_state_bytes of UTF-8 state on a character boundary. A longer + # captured state could never have been sent by the frozen client. + if len(request["state"].encode("utf-8")) > max_state_bytes: + raise ValueError( + f"captured Scout request {record['id']} exceeds frozen state byte budget" + ) + # The production Rust serde_json::Map sorts keys on the Bazel + # build. The settings Vec order is not the serialized wire order. + # Capture bytes are pinned independently; reject any question-map + # order other than the production encoder's canonical sorted order. + if list(request["questions"]) != sorted(questions): + raise ValueError(f"captured Scout request {record['id']} question order differs from canonical Scout wire order") + # The Rust client sends serde_json::to_vec_pretty(Value) directly + # and hashes those same bytes. Refuse compact, reordered or otherwise + # re-encoded captures even when their digests have been recomputed. + canonical = json.dumps( + request, sort_keys=True, ensure_ascii=False, + indent=2, allow_nan=False, + ).encode("utf-8") + if encoded != canonical: + raise ValueError( + f"captured Scout request {record['id']} is not canonical Rust JSON wire bytes" + ) + bodies[record["id"]] = encoded + return bodies + + +def load_questions(path): + """Read a frozen Noul question vocabulary supplied by the settings owner. + + A question file is only a declared contract. Its provenance and the Scout + renderer/window digest must be established separately before runtime parity. + """ + with open(path, "rb") as source: + questions = loads_unambiguous_json(source.read()) + return validate_questions(questions) + + +def validate_questions(questions): + if not isinstance(questions, dict) or not questions: + raise ValueError("qualification requires a nonempty Noul question vocabulary") + for key, question in questions.items(): + if not isinstance(key, str) or not key.strip(): + raise ValueError("qualification question keys must be nonempty strings") + if (not isinstance(question, dict) + or set(question) != {"type", "instructions"} + or question["type"] != "noul" + or not isinstance(question["instructions"], str) + or not question["instructions"].strip() + or len(question["instructions"].encode("utf-8")) > 512): + raise ValueError(f"qualification question {key!r} must be a bounded Noul instruction") + return questions + + +def sha256_bytes(data): + return hashlib.sha256(data).hexdigest() + + +def sha256_json(value): + return sha256_bytes(json.dumps(value, sort_keys=True, allow_nan=False).encode("utf-8")) + + +def sha256_ordered_questions(questions): + """Freeze the actual question order sent to the HTTP model. + + Sorted-object digests cannot bind order: two question files with identical + keys/instructions but different insertion orders produce different HTTP + request bodies and can produce different predictions on the same holdout. + Normalize only each question's own field order, not the sequence of keys. + """ + validate_questions(questions) + return sha256_json([[key, question] for key, question in questions.items()]) + + +def load_manifest(path): + with open(path, "rb") as source: + raw = source.read() + return loads_unambiguous_json(raw), sha256_bytes(raw) + + +def validate_manifest(manifest, dataset_sha, questions, cases=None): + """Fail closed before an HTTP request or a qualified result is published. + + `cases` is the complete original dataset, not a filtered subset. Its + source-owned grouping identities must match the declared split metadata. + """ + if not isinstance(manifest, dict) or manifest.get("version") != 1: + raise ValueError("unsupported qualification manifest version") + if not isinstance(dataset_sha, str) or not _DIGEST.fullmatch(dataset_sha): + raise ValueError("invalid qualification dataset digest") + if manifest.get("dataset_sha256") != dataset_sha: + raise ValueError("qualification dataset digest mismatch") + validate_questions(questions) + if manifest.get("questions_sha256") != sha256_ordered_questions(questions): + raise ValueError("qualification question vocabulary or instruction mismatch") + # Renderer names are not immutable code identities. Freeze actual held-out + # HTTP request bytes before seeing any model predictions. + frozen_requests = manifest.get("holdout_requests_sha256") + if not isinstance(frozen_requests, str) or not _DIGEST.fullmatch(frozen_requests): + raise ValueError("qualification requires a frozen held-out request digest") + + # A held-out split is not independent if the evaluator can try multiple + # renderers, model selectors or serving declarations and pick the winner. + arm = manifest.get("arm") + required_arm = {"config", "requested_model", "endpoint_sha256", "server_config_sha256", "timeout_ms"} + if not isinstance(arm, dict) or set(arm) != required_arm: + raise ValueError("qualification requires one frozen HTTP arm") + if not isinstance(arm["config"], str) or not arm["config"].strip(): + raise ValueError("qualification HTTP arm requires a renderer config") + timeout_ms = arm["timeout_ms"] + if type(timeout_ms) is not int or not 0 < timeout_ms <= 1_800_000: + raise ValueError("qualification HTTP arm requires timeout_ms in (0, 1800000]") + model = arm["requested_model"] + if model is not None and (not isinstance(model, str) or not model.strip()): + raise ValueError("qualification HTTP arm requires a valid requested model or null") + for field in ("endpoint_sha256", "server_config_sha256"): + value = arm[field] + if not isinstance(value, str) or not _DIGEST.fullmatch(value): + raise ValueError(f"qualification HTTP arm has invalid {field}") + + captured_sha = manifest.get("captured_requests_sha256") + if arm["config"] == CAPTURED_SCOUT_CONFIG: + if not isinstance(captured_sha, str) or not _DIGEST.fullmatch(captured_sha): + raise ValueError("qualification requires a frozen captured Scout request file") + elif captured_sha is not None: + raise ValueError("only captured Scout arms may declare captured request evidence") + + # Freeze the production DecisionEngineConfigV1 state cap independently + # from the whole serialized HTTP request limit. Without this, an invented + # capture with a larger model-visible transcript could be called parity. + state_limit = manifest.get("max_state_bytes") + if arm["config"] == CAPTURED_SCOUT_CONFIG: + if type(state_limit) is not int or state_limit <= 0: + raise ValueError("captured Scout arm requires frozen positive max_state_bytes") + elif state_limit is not None: + raise ValueError("only captured Scout arms may declare max_state_bytes") + + # Margins belong to the local intent policy, not the Noul HTTP body. + # Without freezing them a captured arm can pass on hints that Scout + # would suppress when a close competing answer is present. + margins = manifest.get("margin_permille") + if arm["config"] == CAPTURED_SCOUT_CONFIG: + if not isinstance(margins, dict) or set(margins) != set(questions): + raise ValueError("captured Scout arm requires a complete frozen margin_permille map") + for key, margin in margins.items(): + if margin is not None and (type(margin) is not int or not 0 <= margin <= 1_000): + raise ValueError(f"captured Scout margin_permille for {key} must be null or 0..1000") + elif margins is not None: + raise ValueError("only captured Scout arms may declare margin_permille") + + request_limit = manifest.get("max_request_bytes") + if (isinstance(request_limit, bool) or not isinstance(request_limit, int) + or request_limit <= 0): + raise ValueError("qualification requires a positive integer max_request_bytes") + token_limit = manifest.get("max_input_tokens") + if (isinstance(token_limit, bool) or not isinstance(token_limit, int) + or token_limit <= 0): + raise ValueError("qualification requires a positive integer max_input_tokens") + + thresholds = manifest.get("thresholds") + if not isinstance(thresholds, dict) or set(thresholds) != set(questions): + raise ValueError("qualification thresholds must cover every requested question") + for value in thresholds.values(): + if (isinstance(value, bool) or not isinstance(value, (int, float)) + or not math.isfinite(value) or not 0 <= value <= 1): + raise ValueError("qualification thresholds must be finite probabilities") + + # Freeze client-side probability calibration before any held-out inference. + # Otherwise an operator can pick a post-hoc temperature while preserving + # the same question, threshold, and holdout identities. + if "noul_temperature" not in manifest: + raise ValueError("qualification requires frozen client calibration") + client_temperature = manifest["noul_temperature"] + if client_temperature is not None and ( + isinstance(client_temperature, bool) + or not isinstance(client_temperature, (int, float)) + or not math.isfinite(client_temperature) + or client_temperature <= 0 + ): + raise ValueError("qualification client calibration must be null or positive finite") + + # The production Scout client forwards the server's Noul probability + # unchanged. Historical evaluator calibration is not a runtime feature: + # applying it here can turn a below-threshold raw answer into a passing + # model-only held-out suggestion that Scout would never produce. + if arm["config"] == CAPTURED_SCOUT_CONFIG and client_temperature is not None: + raise ValueError("captured Scout parity forbids client-side Noul calibration") + + tolerances = manifest.get("error_tolerances") + required = ("max_none_false_hint_rate", "max_unwanted_hint_rate", "min_positive_case_recall", "max_p95_ms") + if not isinstance(tolerances, dict) or set(tolerances) != set(required): + raise ValueError("qualification tolerances must be frozen before inference") + for key in required: + value = tolerances[key] + if (isinstance(value, bool) or not isinstance(value, (int, float)) + or not math.isfinite(value) or value < 0 + or (key != "max_p95_ms" and value > 1)): + raise ValueError(f"invalid qualification tolerance {key}") + + application = manifest.get("application") + if not isinstance(application, str) or not application.strip(): + raise ValueError("qualification requires an application") + annotations = manifest.get("annotations") + if not isinstance(annotations, list) or not annotations: + raise ValueError("qualification requires reviewer-verified annotations") + seen = set() + projects = {"tune": set(), "holdout": set()} + sessions = {"tune": set(), "holdout": set()} + heldout = [] + tune = [] + negative = positive = 0 + case_map = None + if cases is not None: + if not isinstance(cases, list) or not all(isinstance(c, dict) for c in cases): + raise ValueError("qualification dataset must contain cases") + case_map = {c.get("id"): c for c in cases} + if len(case_map) != len(cases): + raise ValueError("qualification dataset has duplicate case IDs") + for annotation in annotations: + if not isinstance(annotation, dict): + raise ValueError("qualification annotation must be an object") + case_id = annotation.get("id") + if not isinstance(case_id, str) or not case_id or case_id in seen: + raise ValueError("duplicate or missing qualification case ID") + seen.add(case_id) + split = annotation.get("split") + if split not in projects: + raise ValueError("qualification split must be tune or holdout") + for key in ("project_id", "session_id", "author", "reviewer", "review_ref"): + value = annotation.get(key) + if not isinstance(value, str) or not value.strip(): + raise ValueError(f"qualification {case_id} lacks {key}") + if annotation["author"] == annotation["reviewer"]: + raise ValueError(f"qualification {case_id} is self-labelled") + labels = annotation.get("verified_labels") + if (not isinstance(labels, list) + or any(not isinstance(label, str) or label not in questions for label in labels) + or len(set(labels)) != len(labels)): + raise ValueError(f"qualification {case_id} has invalid verified labels") + if not isinstance(annotation.get("verified_context_needed"), bool): + raise ValueError(f"qualification {case_id} lacks reviewed context-needed label") + if case_map is not None: + recorded = case_map.get(case_id) + if (recorded is None or recorded.get("labels") != labels + or recorded.get("context_needed") != annotation["verified_context_needed"]): + raise ValueError(f"qualification {case_id} disagrees with the dataset") + projects[split].add(annotation["project_id"]) + sessions[split].add(annotation["session_id"]) + if split == "holdout": + heldout.append(case_id) + positive += bool(labels) + negative += not labels + else: + tune.append(case_id) + if case_map is not None: + if seen != set(case_map): + raise ValueError("qualification must account for every dataset case") + if [a["id"] for a in annotations] != [c["id"] for c in cases]: + raise ValueError("qualification case ordering differs from the frozen dataset") + if not tune or not heldout or not positive or not negative: + raise ValueError("qualification needs tune and positive/negative holdout cases") + if projects["tune"] & projects["holdout"] or sessions["tune"] & sessions["holdout"]: + raise ValueError("qualification leaks project/session groups between splits") + if case_map is not None: + for annotation in annotations: + recorded = case_map[annotation["id"]] + if (recorded.get("project_id") != annotation["project_id"] + or recorded.get("session_id") != annotation["session_id"]): + raise ValueError( + f"qualification {annotation['id']} group identity disagrees with frozen dataset" + ) + return { + "heldout_ids": heldout, + "holdout_requests_sha256": frozen_requests, + "arm": arm, + "captured_requests_sha256": captured_sha, + "max_request_bytes": request_limit, + "max_state_bytes": state_limit, + "max_input_tokens": token_limit, + "thresholds": thresholds, + "margin_permille": margins, + "noul_temperature": client_temperature, + "error_tolerances": tolerances, + "annotations": {a["id"]: a for a in annotations}, + "application": application, + } + + +def runtime_probability_score(probability, scale): + """Match Rust's nonnegative f64::round at its fixed-point decision scale.""" + return math.floor(probability * scale + 0.5) + + +def holdout_decision_metrics(rows, thresholds, question_order, margin_permille=None): + """Measure all threshold hits separately from the single model-only pick. + + Scout delivers only the first candidate, not every above-threshold answer. + Emulate its permille threshold and u16 relevance rounding, not the historical + floating-point benchmark comparison. Stable relevance ties must use the + frozen question sequence, never the independently serialized threshold map + order. Captured arms also apply frozen per-question margins against every + usable model answer, even when that competitor is below its own threshold. + Action facts and delivery suppression are still omitted; this cannot + establish production parity. + """ + if len(question_order) != len(thresholds) or set(question_order) != set(thresholds): + raise ValueError("model-only selection requires the complete frozen question order") + if margin_permille is not None and ( + not isinstance(margin_permille, dict) + or set(margin_permille) != set(question_order) + or any(value is not None and (type(value) is not int or not 0 <= value <= 1_000) + for value in margin_permille.values()) + ): + raise ValueError("model-only margins must cover frozen questions in permille") + hinted = unwanted = missed_positive = positive = 0 + selected_correct_positive = selected_unwanted = none_negative = none_false_hint = 0 + for row in rows: + labels = set(row["labels"]) + scores = { + key: runtime_probability_score(row["probs"][key], 1_000) + for key in question_order + } + fired = set() + for key in question_order: + if scores[key] < runtime_probability_score(thresholds[key], 1_000): + continue + margin = margin_permille[key] if margin_permille is not None else None + if margin is not None: + # Rust's runner-up is any other answered question, not only + # a competitor that passed its own threshold. Saturate at zero. + runner_up = max((value for other, value in scores.items() + if other != key), default=0) + if max(0, scores[key] - runner_up) < margin: + continue + fired.add(key) + none_negative += not labels + none_false_hint += bool(not labels and fired) + hinted += bool(fired) + unwanted += bool(fired - labels) + positive += bool(labels) + missed_positive += bool(labels and not (fired & labels)) + # Scout sorts on u16 relevance, not the raw f64. Distinct close + # probabilities can quantize to a tie; retain frozen question order. + selected = max( + (key for key in question_order if key in fired), + key=lambda key: runtime_probability_score(row["probs"][key], 65_535), + default=None, + ) + selected_correct_positive += bool(labels and selected in labels) + selected_unwanted += bool(selected is not None and selected not in labels) + return { + "hinted_cases": hinted, + "abstained_cases": len(rows) - hinted, + "coverage_rate": hinted / len(rows), + "abstention_rate": (len(rows) - hinted) / len(rows), + "unwanted_hint_cases": unwanted, + "unwanted_hint_rate": unwanted / len(rows), + "none_false_hint_cases": none_false_hint, + "none_false_hint_rate": none_false_hint / none_negative if none_negative else None, + "missed_positive_cases": missed_positive, + "missed_positive_rate": missed_positive / positive if positive else None, + "model_only_selected_correct_positive_cases": selected_correct_positive, + "model_only_selected_positive_case_recall": ( + selected_correct_positive / positive if positive else None + ), + "model_only_selected_unwanted_hint_cases": selected_unwanted, + "model_only_selected_unwanted_hint_rate": selected_unwanted / len(rows), + } + +def qualified_report(result, manifest, manifest_sha, questions, dataset_bytes, captured_requests_bytes=None, sample_path=None): + """Reproduce qualified predictions from the frozen original dataset.""" + from score import score_config + from run_http import ( + STATES, has_numeric_truncation_receipt, request_bytes, validate_dataset, + validate_row_truncation, validate_qualified_sample, + ) + + if not isinstance(dataset_bytes, bytes): + raise ValueError("qualification requires the original dataset bytes") + dataset_sha = sha256_bytes(dataset_bytes) + if not isinstance(result, dict): + raise ValueError("qualification result must be an object") + if result.get("dataset_sha256") != dataset_sha: + raise ValueError("qualification result dataset digest differs from supplied evidence") + cases = loads_unambiguous_json(dataset_bytes) + validate_dataset(cases, questions) + + experiment = result.get("experiment") + if not isinstance(experiment, dict): + raise ValueError("qualification requires an identity-bound HTTP experiment") + server = experiment.get("server_config") + if not isinstance(server, dict): + raise ValueError("qualification requires a pinned serving declaration") + for field in ("checkpoint", "server_revision", "backend", "dtype"): + if not isinstance(server.get(field), str) or not server[field].strip(): + raise ValueError(f"qualification serving declaration lacks {field}") + temperature = server.get("temperature") + if (isinstance(temperature, bool) or not isinstance(temperature, (int, float)) + or not math.isfinite(temperature) or temperature < 0): + raise ValueError("qualification requires a finite serving temperature") + if experiment.get("qualification_manifest_sha256") != manifest_sha: + raise ValueError("qualification experiment is not bound to this frozen manifest") + validated = validate_manifest(manifest, dataset_sha, questions, cases) + arm = validated["arm"] + if (experiment.get("config") != arm["config"] + or experiment.get("requested_model") != arm["requested_model"] + or experiment.get("timeout_ms") != arm["timeout_ms"] + or experiment.get("endpoint_sha256") != arm["endpoint_sha256"] + or sha256_json(server) != arm["server_config_sha256"]): + raise ValueError("qualification HTTP arm differs from frozen manifest") + if experiment.get("requests_sha256") != validated["holdout_requests_sha256"]: + raise ValueError("qualification rendered held-out requests differ from frozen manifest") + rows = result.get("rows") + if (not isinstance(rows, list) or not all(isinstance(row, dict) for row in rows) + or [row.get("id") for row in rows] != validated["heldout_ids"]): + raise ValueError("qualification result differs from frozen held-out case order") + for row in rows: + annotation = validated["annotations"][row["id"]] + if (row.get("labels") != annotation["verified_labels"] + or row.get("context_needed") != annotation["verified_context_needed"]): + raise ValueError(f"qualification result {row['id']} does not match reviewed labels") + # Historical scoring accepts Choice results and optional "none" scores. + # Qualified Scout runs are Noul-only: the original HTTP runner cannot emit + # a Choice pick or an extra probability outside its frozen question set. + # Reject mixed-type edited evidence before computing qualification checks. + expected_questions = set(questions) + for row in rows: + probabilities = row.get("probs") + raw_probabilities = row.get("probs_raw") + if ( + "choice" in row + or not isinstance(probabilities, dict) + or set(probabilities) != expected_questions + or ( + raw_probabilities is not None + and ( + not isinstance(raw_probabilities, dict) + or set(raw_probabilities) != expected_questions + ) + ) + ): + raise ValueError( + f"qualification row {row['id']} must contain only frozen Noul probabilities" + ) + # Native truncation counters and the normalized drop count must remain + # consistent even if the result JSON was edited after the HTTP response. + for row in rows: + validate_row_truncation(row) + # Normalized dropped=0 can be forged or survive an older partial log with + # no original numeric server counters. Such a log may be scored, but it + # cannot establish that the model evaluated an untruncated input. + source_truncation_cases = sum(has_numeric_truncation_receipt(row) for row in rows) + # Reconstruct real HTTP request bytes: self-consistent metadata hashes + # alone cannot prove the frozen prompts were used. + captured = None + if arm["config"] == CAPTURED_SCOUT_CONFIG: + if (not isinstance(captured_requests_bytes, bytes) + or sha256_bytes(captured_requests_bytes) != validated["captured_requests_sha256"]): + raise ValueError("qualification requires the original frozen Scout request capture") + captured = captured_request_bodies( + captured_requests_bytes, validated["heldout_ids"], questions, + arm["requested_model"], validated["max_request_bytes"], + validated["max_state_bytes"], + ) + renderer = None + else: + if captured_requests_bytes is not None: + raise ValueError("historical renderers cannot import Scout request captures") + renderer = STATES.get(experiment.get("config")) + if renderer is None: + raise ValueError("qualification has an unsupported request renderer") + case_map = {case["id"]: case for case in cases} + request_sizes = [] + reconstructed_requests = {} + for row in rows: + if captured is not None: + rendered = captured[row["id"]] + else: + body = {"state": renderer(case_map[row["id"]]), "questions": questions} + if experiment.get("requested_model"): + body["model"] = experiment["requested_model"] + rendered = request_bytes(body) + if row.get("request_sha256") != sha256_bytes(rendered): + raise ValueError( + f"qualification request fingerprint disagrees with frozen renderer for {row['id']}" + ) + size = len(rendered) + if size > validated["max_request_bytes"]: + raise ValueError(f"qualification case {row['id']} exceeds frozen request byte limit") + recorded_size = row.get("request_size_bytes") + if (isinstance(recorded_size, bool) or not isinstance(recorded_size, int) + or recorded_size != size): + raise ValueError(f"qualification request byte count disagrees for {row['id']}") + request_sizes.append(size) + reconstructed_requests[row["id"]] = ( + rendered if captured is not None else body, + sha256_bytes(rendered), + ) + + # The runner checks raw first-response evidence on resume. Repeat that + # check independently before publishing a qualified scoring report. + if any("sample_response_sha256" in row for row in rows[1:]): + raise ValueError("qualification has a sample commitment outside the first held-out row") + first_row = rows[0] + has_sample_commitment = "sample_response_sha256" in first_row + if has_sample_commitment != (sample_path is not None): + raise ValueError("qualification first-response sample and row commitment must be provided together") + if has_sample_commitment: + response_sha, row_sha = validate_qualified_sample( + sample_path, result["experiment_sha256"], reconstructed_requests, + expected_row=first_row, + ) + if (first_row["sample_response_sha256"] != response_sha + or sha256_json(first_row) != row_sha): + raise ValueError("qualification first-response sample response or row disagrees with completed evidence") + + if ("noul_temperature" not in experiment + or experiment["noul_temperature"] != validated["noul_temperature"]): + raise ValueError("qualification client calibration differs from frozen manifest") + + # The serving declaration is operator-supplied. Compare it to the model + # reported by each actual HTTP response so a changed serving default or + # a mismatched explicit selector cannot silently qualify. + observed_models = [row.get("observed_model") for row in rows] + reported_model = observed_models[0] if ( + all(isinstance(value, str) and value.strip() + and value == observed_models[0] for value in observed_models) + ) else None + reported_model_consistent = ( + reported_model is not None + and (arm["requested_model"] is None + or reported_model == arm["requested_model"]) + ) + + scores = score_config(result, fixed_t=validated["thresholds"], cats=tuple(questions)) + fixed = scores["fixed"]["all"] + latency = scores["latency"] + decisions = holdout_decision_metrics( + rows, validated["thresholds"], tuple(questions), + validated["margin_permille"], + ) + tolerances = validated["error_tolerances"] + # A JSON byte ceiling does not bound tokenizer-specific context usage. + # An unknown or zero server-reported count is not proof of fitting inside + # the model's predeclared validated input-token budget. + reported_tokens = [row.get("input_tokens") for row in rows] + measured_tokens = [ + value for value in reported_tokens if type(value) is int and value > 0 + ] + token_evidence_complete = len(measured_tokens) == len(rows) + tokens_within_budget = ( + token_evidence_complete + and all(value <= validated["max_input_tokens"] for value in measured_tokens) + ) + rates = { + "none_false_hint_rate": decisions["none_false_hint_rate"], + "unwanted_hint_rate": decisions["unwanted_hint_rate"], + # An any-hit recall can pass even when the highest-ranked suggestion + # is wrong. Never qualify a single-delivery application on that number. + "positive_case_recall": decisions["model_only_selected_positive_case_recall"], + "p95_ms": latency["p95_ms"], + } + if any(value is None for value in rates.values()): + raise ValueError("qualification cannot pass with unmeasured error or latency evidence") + # A successful HTTP call can conceal server-side truncation. Such a score + # cannot prove that the model saw the frozen full input. Unknown telemetry + # is not a measured zero, even when every prediction happens to be correct. + complete_input_evidence = ( + latency["truncation_measured_cases"] == len(rows) + and latency["cases_truncated"] == 0 + and source_truncation_cases == len(rows) + ) + checks = { + "none_false_hint_rate": rates["none_false_hint_rate"] <= tolerances["max_none_false_hint_rate"], + "unwanted_hint_rate": rates["unwanted_hint_rate"] <= tolerances["max_unwanted_hint_rate"], + "positive_case_recall": rates["positive_case_recall"] >= tolerances["min_positive_case_recall"], + "p95_ms": rates["p95_ms"] <= tolerances["max_p95_ms"], + "untruncated_input_evidence": complete_input_evidence, + "reported_model_consistency": reported_model_consistent, + "reported_input_token_budget": tokens_within_budget, + } + return { + "application": validated["application"], + "kind": "declared_independent_labels_heldout_not_runtime_parity", + "experiment_sha256": result["experiment_sha256"], + "qualification_manifest_sha256": manifest_sha, + "reported_model": reported_model, + "first_response_sample_checked": has_sample_commitment, + "cases": len(rows), + "thresholds": validated["thresholds"], + "tolerances": tolerances, + "fixed_threshold_metrics": fixed, + "holdout_decisions": decisions, + "request_size_bytes": { + "min": min(request_sizes), + "p50": statistics.median(request_sizes), + "max": max(request_sizes), + "total": sum(request_sizes), + "frozen_limit": validated["max_request_bytes"], + }, + "input_token_evidence": { + "measured_cases": len(measured_tokens), + "missing_or_zero_cases": len(rows) - len(measured_tokens), + "cases_over_frozen_limit": sum( + value > validated["max_input_tokens"] for value in measured_tokens + ), + "max_reported_tokens": max(measured_tokens, default=None), + "frozen_limit": validated["max_input_tokens"], + }, + "latency": latency, + "numeric_truncation_source_evidence": { + "measured_cases": source_truncation_cases, + "missing_cases": len(rows) - source_truncation_cases, + }, + "checks": checks, + "qualified_against_frozen_tolerances": all(checks.values()), + "limits": "Reviewer and serving/model identities and token counts are self-reported, not authenticated or checkpoint-attested; task effectiveness and canonical Scout transcript parity are not proven.", + } diff --git a/evals/intent_engine/run_http.py b/evals/intent_engine/run_http.py index 07f9340f7f8..3bdbd26e2ca 100644 --- a/evals/intent_engine/run_http.py +++ b/evals/intent_engine/run_http.py @@ -23,6 +23,14 @@ HERE = os.path.dirname(os.path.abspath(__file__)) sys.path.insert(0, HERE) from common import calibrated, noul_questions, prompt_only, text_window +from qualification import ( + CAPTURED_SCOUT_CONFIG, captured_request_bodies, load_manifest, load_questions, + loads_unambiguous_json, validate_manifest, +) + +# A decision response contains only a bounded set of typed answers and usage. +# Refuse unexpectedly large remote bodies before materializing or parsing them. +MAX_HTTP_RESPONSE_BYTES = 1024 * 1024 STATES = { "A_noul_prompt": prompt_only, @@ -38,31 +46,151 @@ def digest(value): def request_bytes(body): - # Preserve question and request-field order in both HTTP and resume evidence. + # Never reserialize a Rust-produced Scout request through Python's encoder. + if isinstance(body, bytes): + return body return json.dumps(body, allow_nan=False).encode("utf-8") -def publish_sample(path, serialized): +def validate_qualified_sample(path, experiment_sha, requests, expected_row=None): + """Refuse a stale first-response sample during an exact qualified resume. + + Raw samples are debugging evidence, not independent model attestation. + Verify the exact original bytes through publication: a replacement can + preserve every scored value while rewriting the evidence artifact. + Never normalize the original Scout request bytes. + """ + # The path is untrusted resumable evidence. A separate islink/isfile + # preflight followed by builtins.open can follow a swapped-in symlink. + # Open the inode itself without following links or waiting on a FIFO, + # then verify it is still the named, regular sample after reading. + # A sample contains two representations of the frozen first request and + # a JSON-escaped copy of the at-most-1-MiB response. Allow generous + # serialization expansion, not unlimited attacker-controlled file bytes. + first_request = next(iter(requests.values()))[0] + sample_limit = 16 * (len(request_bytes(first_request)) + MAX_HTTP_RESPONSE_BYTES) + 4096 + if not hasattr(os, "O_NOFOLLOW") and os.path.islink(path): + raise ValueError("qualified sample path must be a regular file") + flags = os.O_RDONLY | getattr(os, "O_NONBLOCK", 0) | getattr(os, "O_NOFOLLOW", 0) + try: + descriptor = os.open(path, flags) + except OSError as error: + if error.errno in (errno.ELOOP, errno.ENOTDIR): + raise ValueError("qualified sample path must be a regular file") from error + raise + try: + opened = os.fstat(descriptor) + if not stat.S_ISREG(opened.st_mode): + raise ValueError("qualified sample path must be a regular file") + if opened.st_size > sample_limit: + raise ValueError("qualified sample exceeds bounded evidence byte limit") + with os.fdopen(descriptor, "rb", closefd=False) as source: + sample_bytes = source.read(sample_limit + 1) + # The inode can grow between fstat and read. A bounded read must also + # refuse growth rather than treating a truncated prefix as evidence. + if len(sample_bytes) > sample_limit: + raise ValueError("qualified sample exceeds bounded evidence byte limit") + finally: + os.close(descriptor) + try: + named = os.stat(path, follow_symlinks=False) + except FileNotFoundError as error: + raise ValueError("qualified sample path disappeared during validation") from error + if (not stat.S_ISREG(named.st_mode) + or (opened.st_dev, opened.st_ino) != (named.st_dev, named.st_ino)): + raise ValueError("qualified sample path was replaced or redirected") + sample = loads_unambiguous_json(sample_bytes) + if not isinstance(sample, dict): + raise ValueError("qualified sample must be a JSON object") + case_id = sample.get("case_id") + # The first-response sample must be from the first frozen HTTP call, + # not merely from an arbitrary request in the same experiment. + if case_id != next(iter(requests)): + raise ValueError("qualified sample is not the first held-out response") + original = requests.get(case_id) if isinstance(case_id, str) else None + if (original is None or sample.get("experiment_sha256") != experiment_sha + or sample.get("request_sha256") != original[1]): + raise ValueError("qualified sample disagrees with frozen experiment identity") + body = original[0] + if isinstance(body, bytes): + matches_request = ( + sample.get("request_json") == body.decode("utf-8") + and sample.get("request") == loads_unambiguous_json(body) + ) + else: + matches_request = ( + "request_json" not in sample and sample.get("request") == body + ) + if not matches_request: + raise ValueError("qualified sample request differs from frozen HTTP bytes") + raw_response = sample.get("response") + raw_probabilities = noul_probabilities(raw_response, sample["request"]["questions"]) + if expected_row is not None: + # A response digest in the row and a row digest in the sample can both + # be rewritten together. Independently replay the raw response against + # the scored first row so such self-consistent edits cannot certify + # invented probabilities or favorable usage/cost telemetry. + usage = raw_response.get("usage") or {} + if not isinstance(usage, dict): + raise ValueError("qualified sample has invalid raw usage") + server_ms = raw_response.get("latency_ms") + if server_ms is None: + timing = raw_response.get("timing") or {} + if not isinstance(timing, dict): + raise ValueError("qualified sample has invalid raw timing") + server_ms = timing.get("inference_ms") + recorded_raw = expected_row.get("probs_raw", expected_row.get("probs")) + reconstructed = { + "observed_model": raw_response.get("model"), + "input_tokens": usage.get("input_tokens"), + "state_tokens": usage.get("state_tokens"), + "dropped": dropped_state_tokens(raw_response, usage), + "server_ms": server_ms, + } + optional = ( + ("state_tokens_used", "state_tokens_used"), + ("state_tokens_dropped", "reported_state_tokens_dropped"), + ("truncated", "usage_truncated"), + ) + if ( + recorded_raw != raw_probabilities + or any(expected_row.get(key) != value for key, value in reconstructed.items()) + or any((name in usage) != (stored in expected_row) + or (name in usage and usage[name] != expected_row[stored]) + for name, stored in optional) + or ("truncated" in raw_response) != ("truncated" in expected_row) + or ("truncated" in raw_response + and raw_response["truncated"] != expected_row["truncated"]) + ): + raise ValueError("qualified sample raw response disagrees with first scored row") + row_sha = sample.get("row_sha256") + if not isinstance(row_sha, str) or not re.fullmatch(r"[0-9a-f]{64}", row_sha): + raise ValueError("qualified sample lacks the completed first-row commitment") + return digest(sample["response"]), row_sha, hashlib.sha256(sample_bytes).hexdigest() + + +def publish_sample(path, serialized, refuse_existing=False): """Publish a completely rendered sample without clobbering an existing one. Same-directory staging avoids exposing a partial sample after a write error; linking the completed file keeps the original exclusive-create behavior. """ directory = os.path.dirname(os.path.abspath(path)) - # mkstemp forces mode 0600. Match open(path, "x") by applying the - # process umask at creation without changing the process-wide umask. + # Samples include the raw request, potentially containing Scout session + # transcripts. The caller's umask must not make that evidence public. staged = os.path.join(directory, f".noul-sample-{secrets.token_hex(16)}") - descriptor = os.open(staged, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o666) + descriptor = os.open(staged, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600) try: - with os.fdopen(descriptor, "w", encoding="utf-8") as output: - output.write(serialized) + with os.fdopen(descriptor, "wb") as output: + output.write(serialized.encode("utf-8")) try: os.link(staged, path) except FileExistsError: - # A dangling symlink occupies the pathname but exists() says - # false. Do not treat that unusable capture as already published. - if not os.path.isfile(path): - raise ValueError(f"sample path exists but is not a regular file: {path}") + # Qualified samples must never silently adopt an existing file + # that appeared after the preflight. Unqualified historical + # samples retain their original no-clobber behavior. + if refuse_existing or not os.path.isfile(path): + raise ValueError(f"sample path exists or was replaced during publication: {path}") finally: os.unlink(staged) @@ -110,7 +238,7 @@ def publish_final_result(path, document): serialized = json.dumps(document, indent=1, allow_nan=False) directory = os.path.dirname(os.path.abspath(path)) staged = os.path.join(directory, f".noul-result-{secrets.token_hex(16)}") - descriptor = os.open(staged, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o666) + descriptor = os.open(staged, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600) try: with os.fdopen(descriptor, "w", encoding="utf-8") as output: output.write(serialized) @@ -141,6 +269,22 @@ def require_http_endpoint(url): return endpoint +def require_scout_endpoint(url): + """The captured arm must use the same endpoint shape as Scout's client. + + DecisionEngineClientV1 appends /v1/systemone to its base URL; allowing + an arbitrary route, query or fragment turns byte-exact request replay + into a different inference contract despite a refrozen endpoint digest. + """ + endpoint = require_http_endpoint(url) + if (not endpoint.path.endswith("/v1/systemone") + or endpoint.query or endpoint.fragment): + raise ValueError( + "captured Scout inference requires canonical /v1/systemone " + "endpoint without query or fragment" + ) + + def load_bearer_token(name, url): """Read an optional bearer credential without persisting it in evidence.""" if not re.fullmatch(r"[A-Za-z_][A-Za-z0-9_]*", name): @@ -170,13 +314,20 @@ def post(url, body, timeout=1800, bearer_token=None): ) started = time.perf_counter() with opener.open(req, timeout=timeout) as response: - out = json.loads(response.read()) + # urllib treats every 2xx status as successful. The Rust Scout client + # accepts exactly 200, so a 201/202/206 must not become model evidence. + if response.status != 200: + raise ValueError(f"decision HTTP status {response.status}; Scout requires 200") + raw = response.read(MAX_HTTP_RESPONSE_BYTES + 1) + if len(raw) > MAX_HTTP_RESPONSE_BYTES: + raise ValueError("decision HTTP response exceeds bounded byte limit") + out = loads_unambiguous_json(raw) return out, (time.perf_counter() - started) * 1000 def load_server_config(path): with open(path, encoding="utf-8") as source: - config = json.load(source) + config = loads_unambiguous_json(source.read()) if not isinstance(config, dict): raise ValueError("--server-config must contain a JSON object") for field in ("checkpoint", "server_revision", "backend", "dtype"): @@ -223,6 +374,122 @@ def noul_probabilities(response, questions): return probabilities(values, questions) +def dropped_state_tokens(response, usage): + """Derive measurable truncation from the native HTTP response contract. + + Kev reports the original and used state-token counts with a top-level + truncated flag. Other servers may report state_tokens_dropped directly. + A bare false flag is not numeric evidence of zero dropped tokens. + """ + direct = usage.get("state_tokens_dropped") + if direct is not None and (type(direct) is not int or direct < 0): + raise ValueError("invalid truncation dropped-token count") + + original = usage.get("state_tokens") + used = usage.get("state_tokens_used") + for value in (original, used): + if value is not None and (type(value) is not int or value < 0): + raise ValueError("invalid truncation state-token count") + + # A directly reported drop count cannot exceed the original state. + # Without this check, a malformed server can claim more dropped tokens + # than were ever sent and still leave a self-consistent evidence row. + if direct is not None and original is not None and direct > original: + raise ValueError("invalid truncation: dropped state tokens exceed original") + + derived = None + if original is not None and used is not None: + if used > original: + raise ValueError("invalid truncation: used state tokens exceed original") + derived = original - used + if direct is not None and direct != derived: + raise ValueError("inconsistent truncation token counts") + + dropped = derived if derived is not None else direct + flags = [] + for envelope, field in ((response, "truncated"), (usage, "truncated")): + if field in envelope: + flag = envelope[field] + if type(flag) is not bool: + raise ValueError("invalid truncation flag") + flags.append(flag) + if dropped is not None and flag != (dropped > 0): + raise ValueError("inconsistent truncation flag and token counts") + if len(set(flags)) > 1: + raise ValueError("inconsistent truncation flags") + return dropped + + +def has_numeric_truncation_receipt(row): + """A normalized drop count alone is not original server telemetry. + + Native Kev supplies original/used token counts; other servers may emit + an explicit dropped-token count. Neither a bare Boolean flag nor a + self-authored normalized `dropped` field proves a zero-drop input. + """ + return type(row.get("reported_state_tokens_dropped")) is int or ( + type(row.get("state_tokens")) is int + and type(row.get("state_tokens_used")) is int + ) + + +def validate_row_truncation(row): + """Recheck persisted server counters; normalized dropped alone is mutable. + + Older rows without raw counter fields still have unknown original source + semantics. New rows retain the supplied counts/flags for independent + qualification and exact partial-resume validation. + """ + # Kev's total input count includes every state token the model actually + # consumed, even when some original state tokens were truncated. A low + # reported input count cannot establish context fit if the used-state + # counter already exceeds it. Check fresh, resumed and scored rows alike. + original = row.get("state_tokens") + dropped = row.get("dropped") + used = row.get("state_tokens_used") + # Even historical HTTP rows lacking native counters must not claim that + # more state tokens were dropped than existed. Otherwise an edited partial + # can pass resume validation with a negative inferred used-token count. + if type(original) is int and type(dropped) is int and dropped > original: + raise ValueError("inconsistent usage: normalized dropped tokens exceed original state") + if used is None and type(original) is int and type(dropped) is int: + used = original - dropped + input_tokens = row.get("input_tokens") + if type(input_tokens) is int and type(used) is int and used > input_tokens: + raise ValueError("inconsistent usage: used state tokens exceed total input tokens") + + native_fields = ("state_tokens_used", "reported_state_tokens_dropped", + "truncated", "usage_truncated") + if not any(field in row for field in native_fields): + return + usage = {"state_tokens": row.get("state_tokens")} + if "state_tokens_used" in row: + usage["state_tokens_used"] = row["state_tokens_used"] + if "reported_state_tokens_dropped" in row: + usage["state_tokens_dropped"] = row["reported_state_tokens_dropped"] + if "usage_truncated" in row: + usage["truncated"] = row["usage_truncated"] + envelope = {"truncated": row["truncated"]} if "truncated" in row else {} + expected = dropped_state_tokens(envelope, usage) + if row.get("dropped") != expected: + raise ValueError("truncation row disagrees with recorded server counters") + + +def checked_qualified_model(previous, reported, requested): + """Refuse a changed reported model within one frozen HTTP arm. + + A server name is not checkpoint attestation, but neither a resumed row + nor a new response may contradict the selected or previously seen name. + """ + if not isinstance(reported, str) or not reported.strip(): + raise ValueError("qualified HTTP response requires a server-reported model") + if requested is not None and reported != requested: + raise ValueError("qualified HTTP reported model differs from frozen selector") + if previous is not None and reported != previous: + raise ValueError("qualified HTTP responses mix reported models") + return reported + + def serialize_result(row, questions, temperature): """Admit fresh and resumed rows before any durable output is changed.""" probabilities(row.get("probs"), questions) @@ -238,6 +505,12 @@ def serialize_result(row, questions, temperature): elif (isinstance(value, bool) or not isinstance(value, (int, float)) or value < 0 or (isinstance(value, float) and not math.isfinite(value))): raise ValueError(f"{field} must be a finite nonnegative number") + if "observed_model" in row: + observed_model = row["observed_model"] + if observed_model is not None and ( + not isinstance(observed_model, str) or not observed_model.strip() + ): + raise ValueError("reported HTTP model must be a nonempty string or null") if temperature is None: if "probs_raw" in row: raise ValueError("uncalibrated result must not contain raw probabilities") @@ -245,6 +518,12 @@ def serialize_result(row, questions, temperature): raw = probabilities(row.get("probs_raw"), questions) if row["probs"] != {key: calibrated(value, temperature) for key, value in raw.items()}: raise ValueError("calibrated probabilities disagree with raw probabilities") + validate_row_truncation(row) + if "sample_response_sha256" in row and ( + not isinstance(row["sample_response_sha256"], str) + or not re.fullmatch(r"[0-9a-f]{64}", row["sample_response_sha256"]) + ): + raise ValueError("invalid first-response sample commitment") # json.loads accepts NaN/Infinity, including in extra nested fields. Reject # them while validating the partial log, not after opening the final file. return json.dumps(row, allow_nan=False) @@ -276,6 +555,10 @@ def lock_partial_writer(partial, path): opened = os.fstat(partial.fileno()) if (current.st_dev, current.st_ino) != (opened.st_dev, opened.st_ino): raise ValueError(f"partial output was replaced or redirected: {path}") + # Older interrupted logs may predate private evidence permissions. + # Tighten the opened, locked inode before reading or continuing inference. + if os.name == "posix" and stat.S_IMODE(opened.st_mode) & 0o077: + os.fchmod(partial.fileno(), stat.S_IMODE(opened.st_mode) & 0o700) partial.seek(0) @@ -305,22 +588,54 @@ def open_partial_for_resume(path): def resume_rows(partial, header, cases, requests, questions, temperature): """Validate the whole existing log before appending or issuing HTTP work.""" - first = partial.readline() - if not first.endswith("\n") or json.loads(first) != header: + # .partial is untrusted input. Limit total evidence and individual JSON + # lines using the original response cap and frozen case/question identities. + # The writer uses ASCII-escaped JSON; these character caps also bound bytes. + header_limit = len(json.dumps(header, allow_nan=False)) + 4096 + row_limit = 16 * ( + MAX_HTTP_RESPONSE_BYTES + + len(json.dumps(questions, allow_nan=False)) + + max(len(case_id) for case_id in cases) + ) + 4096 + if os.fstat(partial.fileno()).st_size > header_limit + 1 + len(cases) * (row_limit + 1): + raise ValueError("partial output exceeds bounded evidence byte limit") + first = partial.readline(header_limit + 1) + if len(first) > header_limit: + raise ValueError("partial header exceeds bounded evidence byte limit") + if not first.endswith("\n") or loads_unambiguous_json(first) != header: raise ValueError("partial output belongs to another or unrecorded experiment; use a new tag") done = {} - for line_number, line in enumerate(partial, 2): + ordered_ids = tuple(cases) + qualified = "qualification_manifest_sha256" in header["experiment"] + previous_model = None + for line_number, line in enumerate(iter(lambda: partial.readline(row_limit + 1), ""), 2): + if len(line) > row_limit: + raise ValueError(f"partial row at line {line_number} exceeds bounded evidence byte limit") if not line.endswith("\n"): raise ValueError(f"partial output has an incomplete row at line {line_number}") - row = json.loads(line) + row = loads_unambiguous_json(line) case_id = row.get("id") if isinstance(row, dict) else None if not isinstance(case_id, str) or case_id not in cases or case_id in done: raise ValueError(f"partial output has an unknown or duplicate case at line {line_number}") + # A real interrupted run appends only a prefix in request order. + # Skipped or reordered rows can misrepresent cold/warm inference order. + if case_id != ordered_ids[len(done)]: + raise ValueError( + f"partial output is not a contiguous prefix of frozen case order at line {line_number}" + ) case = cases[case_id] if (row.get("request_sha256") != requests[case_id][1] or row.get("labels") != case["labels"] or row.get("context_needed") != case["context_needed"]): raise ValueError(f"partial output has mismatched case evidence for {case_id}") + if qualified: + recorded_size = row.get("request_size_bytes") + if type(recorded_size) is not int or recorded_size != len(request_bytes(requests[case_id][0])): + raise ValueError(f"partial output has mismatched request byte count for {case_id}") + previous_model = checked_qualified_model( + previous_model, row.get("observed_model"), + header["experiment"].get("requested_model"), + ) serialize_result(row, questions, temperature) done[case_id] = row return done @@ -365,33 +680,79 @@ def main(argv=None): ap.add_argument("--tag", required=True) ap.add_argument("--server-config", required=True, help="JSON file with immutable serving identity; omit credentials") ap.add_argument("--subset") + ap.add_argument("--dataset", help="original JSON cases; frozen digest recorded in each run") + ap.add_argument("--qualification-manifest", help="freeze reviewed held-out labels, groups, thresholds and tolerances") + ap.add_argument("--questions-file", help="declared typed Noul questions; qualified runs only, requires separate runtime provenance") + ap.add_argument("--captured-requests", help="frozen verbatim Scout HTTP requests, captured qualification arm only") ap.add_argument("--send-model") + ap.add_argument("--timeout-ms", type=int, default=1_800_000, + help="per-request HTTP timeout in milliseconds (1..1800000); frozen for qualified runs") ap.add_argument("--bearer-token-env", metavar="NAME", help="name of an environment variable holding a bearer token (never recorded)") ap.add_argument("--sample-out", help="write the first raw response here") ap.add_argument("--noul-temperature", type=float, help="client calibration temperature; raw probabilities are retained") - ap.add_argument("configs", nargs="+", choices=tuple(STATES)) + ap.add_argument("configs", nargs="+", choices=(*STATES, CAPTURED_SCOUT_CONFIG)) args = ap.parse_args(argv) if not re.fullmatch(r"[A-Za-z0-9][A-Za-z0-9_.-]*", args.tag): raise ValueError("--tag must be a filename component starting with a letter or digit") if (args.noul_temperature is not None and (not math.isfinite(args.noul_temperature) or args.noul_temperature <= 0)): raise ValueError("--noul-temperature must be finite and greater than zero") + if not 0 < args.timeout_ms <= 1_800_000: + raise ValueError("--timeout-ms must be in (0, 1800000]") require_http_endpoint(args.url) server_config = load_server_config(args.server_config) bearer_token = (load_bearer_token(args.bearer_token_env, args.url) if args.bearer_token_env is not None else None) if args.sample_out and os.path.lexists(args.sample_out) and not os.path.isfile(args.sample_out): raise ValueError(f"sample path exists but is not a regular file: {args.sample_out}") - with open(os.path.join(HERE, "dataset.json"), "rb") as source: + with open(args.dataset or os.path.join(HERE, "dataset.json"), "rb") as source: dataset_bytes = source.read() - dataset = json.loads(dataset_bytes) + dataset = loads_unambiguous_json(dataset_bytes) dataset_sha = hashlib.sha256(dataset_bytes).hexdigest() - questions = noul_questions() + if args.questions_file and not args.qualification_manifest: + raise ValueError("--questions-file requires --qualification-manifest") + if args.captured_requests and not args.qualification_manifest: + raise ValueError("--captured-requests requires --qualification-manifest") + if CAPTURED_SCOUT_CONFIG in args.configs and not args.qualification_manifest: + raise ValueError("captured Scout requests require frozen qualification") + questions = load_questions(args.questions_file) if args.questions_file else noul_questions() case_ids = validate_dataset(dataset, questions) + qualification_sha = None + captured = None + if args.qualification_manifest: + if args.subset: + raise ValueError("qualification selects its own frozen holdout; --subset is not allowed") + manifest, qualification_sha = load_manifest(args.qualification_manifest) + qualified = validate_manifest(manifest, dataset_sha, questions, dataset) + if args.noul_temperature != qualified["noul_temperature"]: + raise ValueError("qualification client calibration differs from frozen manifest") + arm = qualified["arm"] + if (args.configs != [arm["config"]] + or args.send_model != arm["requested_model"] + or args.timeout_ms != arm["timeout_ms"] + or hashlib.sha256(args.url.encode("utf-8")).hexdigest() != arm["endpoint_sha256"] + or digest(server_config) != arm["server_config_sha256"]): + raise ValueError("qualification HTTP arm differs from frozen manifest") + if arm["config"] == CAPTURED_SCOUT_CONFIG: + require_scout_endpoint(args.url) + if (arm["config"] == CAPTURED_SCOUT_CONFIG) != bool(args.captured_requests): + raise ValueError("captured Scout arms require --captured-requests and no historical renderer") + if args.captured_requests: + with open(args.captured_requests, "rb") as source: + capture_bytes = source.read() + if hashlib.sha256(capture_bytes).hexdigest() != qualified["captured_requests_sha256"]: + raise ValueError("captured Scout request evidence differs from frozen manifest") + captured = captured_request_bodies( + capture_bytes, qualified["heldout_ids"], questions, + args.send_model, qualified["max_request_bytes"], + qualified["max_state_bytes"], + ) + keep = set(qualified["heldout_ids"]) + dataset = [case for case in dataset if case["id"] in keep] if args.subset: with open(args.subset, encoding="utf-8") as source: - ids = json.load(source)["ids"] + ids = loads_unambiguous_json(source.read())["ids"] if (not isinstance(ids, list) or not ids or any(not isinstance(case_id, str) for case_id in ids) or len(set(ids)) != len(ids) or not set(ids).issubset(case_ids)): @@ -401,6 +762,24 @@ def main(argv=None): if not dataset: raise ValueError("the experiment must contain at least one case") cases = {case["id"]: case for case in dataset} + if qualification_sha is not None: + # Bound and verify precommitted request bodies before opening any + # output. A stable config name does not imply a stable renderer. + renderer = STATES.get(qualified["arm"]["config"]) + rendered_requests = [] + for case in dataset: + if captured is not None: + encoded = captured[case["id"]] + else: + body = {"state": renderer(case), "questions": questions} + if args.send_model: + body["model"] = args.send_model + encoded = request_bytes(body) + if len(encoded) > qualified["max_request_bytes"]: + raise ValueError(f"qualification case {case['id']} exceeds frozen request byte limit") + rendered_requests.append([case["id"], hashlib.sha256(encoded).hexdigest()]) + if digest(rendered_requests) != qualified["holdout_requests_sha256"]: + raise ValueError("qualification rendered held-out requests differ from frozen manifest") os.makedirs(os.path.join(HERE, "results"), exist_ok=True) for cfg in args.configs: name = f"{args.tag}_{cfg}" @@ -412,9 +791,12 @@ def main(argv=None): require_exclusive_publication(os.path.dirname(os.path.abspath(args.sample_out))) requests = {} for case in dataset: - body = {"state": STATES[cfg](case), "questions": questions} - if args.send_model: - body["model"] = args.send_model + if captured is not None: + body = captured[case["id"]] + else: + body = {"state": STATES[cfg](case), "questions": questions} + if args.send_model: + body["model"] = args.send_model requests[case["id"]] = (body, hashlib.sha256(request_bytes(body)).hexdigest()) identity = { "version": 2, @@ -433,10 +815,31 @@ def main(argv=None): "noul_temperature": args.noul_temperature, "server_config": server_config, } + if qualification_sha is not None: + identity["qualification_manifest_sha256"] = qualification_sha + identity["timeout_ms"] = args.timeout_ms header = {"experiment": identity, "experiment_sha256": digest(identity)} partial_path = path + ".partial" + admitted_sample = None + sample_sha256 = None + if qualification_sha is not None and args.sample_out and os.path.lexists(args.sample_out): + # A new experiment cannot inherit a previous tag's first-response + # evidence. An exact resume may retain only its own sample. + if not os.path.lexists(partial_path): + raise ValueError("qualified sample already exists for a new experiment; choose a new --sample-out") + admitted_sample = validate_qualified_sample( + args.sample_out, header["experiment_sha256"], requests, + ) + sample_sha256 = admitted_sample[2] try: - partial = open(partial_path, "x+", encoding="utf-8") + # Default umask 022 (or an explicit umask 000) must not expose + # result labels, cases or response telemetry to other users. + descriptor = os.open(partial_path, os.O_RDWR | os.O_CREAT | os.O_EXCL, 0o600) + try: + partial = os.fdopen(descriptor, "r+", encoding="utf-8") + except BaseException: + os.close(descriptor) + raise created = True except FileExistsError: partial = open_partial_for_resume(partial_path) @@ -452,15 +855,52 @@ def main(argv=None): done = {} else: done = resume_rows(partial, header, cases, requests, questions, args.noul_temperature) + if qualification_sha is not None and args.sample_out: + first_id = next(iter(cases)) + if done and admitted_sample is None: + raise ValueError("qualified first-response sample is missing for resumed evidence") + if admitted_sample is not None: + if first_id not in done: + # A crash between publishing the sample and appending + # its row cannot be reconstructed as a completed call. + raise ValueError("qualified sample has no completed first-response row; use a new tag") + response_sha, row_sha, _ = admitted_sample + first_row = done[first_id] + current_sample = validate_qualified_sample( + args.sample_out, header["experiment_sha256"], requests, + expected_row=first_row, + ) + if current_sample[2] != sample_sha256: + raise ValueError("qualified first-response sample changed during resume admission") + if (first_row.get("sample_response_sha256") != response_sha + or digest(first_row) != row_sha): + raise ValueError("qualified sample response or row disagrees with completed evidence") + previous_model = next( + (row["observed_model"] for row in done.values() + if row.get("observed_model") is not None), None + ) for index, case in enumerate(dataset): if case["id"] in done: continue body, request_sha = requests[case["id"]] + # Match the precommitted model-serving deadline; a post-hoc + # p95 threshold cannot substitute for a real request timeout. + timeout_arg = ({"timeout": args.timeout_ms / 1000} + if qualification_sha is not None or args.timeout_ms != 1_800_000 + else {}) if bearer_token is None: - response, ms = post(args.url, body) + response, ms = post(args.url, body, **timeout_arg) else: - response, ms = post(args.url, body, bearer_token=bearer_token) + response, ms = post(args.url, body, bearer_token=bearer_token, **timeout_arg) probs = noul_probabilities(response, questions) + observed_model = response.get("model") + if (args.send_model is not None and observed_model is not None + and observed_model != args.send_model): + raise ValueError("reported model differs from requested model") + if qualification_sha is not None: + previous_model = checked_qualified_model( + previous_model, observed_model, args.send_model, + ) server_ms = response.get("latency_ms") if server_ms is None: server_ms = (response.get("timing") or {}).get("inference_ms") @@ -471,24 +911,58 @@ def main(argv=None): raise ValueError("response usage must be an object or null") row = { "id": case["id"], "request_sha256": request_sha, + "request_size_bytes": len(request_bytes(body)), "labels": case["labels"], "context_needed": case["context_needed"], "ms": ms, "server_ms": server_ms, "input_tokens": usage.get("input_tokens"), "state_tokens": usage.get("state_tokens"), - "dropped": usage.get("state_tokens_dropped"), "probs": probs, + "dropped": dropped_state_tokens(response, usage), "probs": probs, + "observed_model": observed_model, } + # Retain the original typed truncation measurements so a + # resume and an independent scorer can reconstruct dropped. + for field, persisted in ( + ("state_tokens_used", "state_tokens_used"), + ("state_tokens_dropped", "reported_state_tokens_dropped"), + ("truncated", "usage_truncated"), + ): + if field in usage: + row[persisted] = usage[field] + if "truncated" in response: + row["truncated"] = response["truncated"] if args.noul_temperature is not None: row["probs_raw"] = probs row["probs"] = {key: calibrated(value, args.noul_temperature) for key, value in probs.items()} + if qualification_sha is not None and args.sample_out and index == 0: + row["sample_response_sha256"] = digest(response) # Refuse malformed row and raw-sample data before publishing # either. A non-finite response must not leave a sample file # that a later valid retry mistakes for a completed capture. serialized_row = serialize_result(row, questions, args.noul_temperature) - if args.sample_out and not os.path.exists(args.sample_out): - serialized_sample = json.dumps( - {"request": body, "response": response}, indent=1, allow_nan=False + if args.sample_out and ( + (qualification_sha is not None and index == 0) + or (qualification_sha is None and not os.path.exists(args.sample_out)) + ): + sample = { + "request": loads_unambiguous_json(body) if isinstance(body, bytes) else body, + "response": response, + } + if isinstance(body, bytes): + sample["request_json"] = body.decode("utf-8") + if qualification_sha is not None: + sample.update({ + "experiment_sha256": header["experiment_sha256"], + "case_id": case["id"], + "request_sha256": request_sha, + "row_sha256": digest(row), + }) + serialized_sample = json.dumps(sample, indent=1, allow_nan=False) + publish_sample( + args.sample_out, serialized_sample, + refuse_existing=qualification_sha is not None, ) - publish_sample(args.sample_out, serialized_sample) + if qualification_sha is not None: + sample_sha256 = hashlib.sha256(serialized_sample.encode("utf-8")).hexdigest() partial.write(serialized_row + "\n") partial.flush() done[case["id"]] = row @@ -496,6 +970,25 @@ def main(argv=None): # Hold the partial writer lock through exclusive publication. # A second evaluator must not publish the same complete log first. # Failed publication retains the partial for an exact resume. + if qualification_sha is not None and args.sample_out: + # Preflight admission alone does not bind the sample for the + # duration of a resumed run: the sample path can be replaced + # while subsequent held-out HTTP calls are in flight. Recheck + # the original first response against the durable first row + # immediately before exposing a completed qualification. + first_row = done[next(iter(cases))] + response_sha, row_sha, final_sample_sha256 = validate_qualified_sample( + args.sample_out, header["experiment_sha256"], requests, + expected_row=first_row, + ) + if (first_row.get("sample_response_sha256") != response_sha + or digest(first_row) != row_sha + or sample_sha256 is None + or final_sample_sha256 != sample_sha256): + raise ValueError( + "qualified first-response sample changed during inference; " + "refusing completed result" + ) publish_final_result(path, { "config": name, "dataset_sha256": dataset_sha, "noul_temperature": args.noul_temperature, diff --git a/evals/intent_engine/score.py b/evals/intent_engine/score.py index da9375d9624..b4984c0aa51 100644 --- a/evals/intent_engine/score.py +++ b/evals/intent_engine/score.py @@ -26,11 +26,11 @@ def publish_summary(path, summary): os.unlink(staged) -def validate_rows(rows, config): +def validate_rows(rows, config, cats=CATS): """Refuse invalid measured evidence before any threshold or AUC arithmetic.""" if not isinstance(rows, list) or not rows: raise ValueError("configuration %r has no rows to score" % config) - categories = set(CATS) + categories = set(cats) seen = set() for row in rows: if not isinstance(row, dict): @@ -172,7 +172,7 @@ def case_metrics(rows, ths): none_rows = [r for r in rows if not r["labels"]] none_fh = 0 for r in rows: - fired = {c for c in CATS if r["probs"][c] >= ths[c]} + fired = {c for c in ths if r["probs"][c] >= ths[c]} exact += fired == set(r["labels"]) passing = [(r["probs"][c] - ths[c], c) for c in fired] pick = max(passing)[1] if passing else None @@ -210,6 +210,8 @@ def lat(rows): observed_truncated = sum(value > 0 for value in dropped) result = { "median_ms": round(statistics.median(ms)), + "p50_ms": statistics.median(ms), + "p95_ms": ms[math.ceil(0.95 * len(ms)) - 1], "p90_ms": round(ms[int(0.9 * (len(ms) - 1))]), "total_cases": len(rows), "cases_truncated": observed_truncated if len(dropped) == len(rows) else None, @@ -223,13 +225,13 @@ def lat(rows): result[f"{field}_measured_cases"] = len(values) return result -def score_config(res, fixed_t=None): +def score_config(res, fixed_t=None, cats=CATS): if "experiment" in res: experiment = res["experiment"] if not isinstance(experiment, dict) or experiment.get("version") != 2: raise ValueError("HTTP result telemetry version is unsupported; rerun with a new tag") rows = res["rows"] - validate_rows(rows, res.get("config")) + validate_rows(rows, res.get("config"), cats) validate_calibration(res, rows) validate_http_provenance(res, rows) choices = ["choice" in row for row in rows] @@ -237,7 +239,7 @@ def score_config(res, fixed_t=None): raise ValueError("configuration %r has incomplete choice coverage" % res.get("config")) ctx = [r for r in rows if r["context_needed"]]; nctx = [r for r in rows if not r["context_needed"]] per = {} - for c in CATS: + for c in cats: pos = [r["probs"][c] for r in rows if c in r["labels"]] neg = [r["probs"][c] for r in rows if c not in r["labels"]] sweep = {str(t): prf(rows, c, t) for t in TH} @@ -247,7 +249,7 @@ def score_config(res, fixed_t=None): "sweep": sweep, "best_t": best, "best_f1": sweep[str(best)]["f1"]} # A singleton has no training rows after holding out its only case. loo_cases = len(rows) if len(rows) > 1 else 0 - for c in CATS: + for c in cats: if not loo_cases: per[c].update(loo_f1=None, loo_p=None, loo_r=None) continue @@ -276,19 +278,19 @@ def training_f1(t): per[c]["loo_p"] = tp / (tp + fp) if tp + fp else None per[c]["loo_r"] = tp / (tp + fn) if tp + fn else None out = {"per_cat": per, "loo_evaluated_cases": loo_cases, - "macro_loo_f1": statistics.mean(per[c]["loo_f1"] or 0 for c in CATS) if loo_cases else None} - for name, t in [("t0.5", {c: 0.5 for c in CATS}), ("t0.6", {c: 0.6 for c in CATS}), - ("t0.7", {c: 0.7 for c in CATS}), - ("best_t_in_sample", {c: per[c]["best_t"] for c in CATS})] + ([("fixed", fixed_t)] if fixed_t else []): - mf = statistics.mean(prf(rows, c, t[c])["f1"] or 0 for c in CATS) + "macro_loo_f1": statistics.mean(per[c]["loo_f1"] or 0 for c in cats) if loo_cases else None} + for name, t in [("t0.5", {c: 0.5 for c in cats}), ("t0.6", {c: 0.6 for c in cats}), + ("t0.7", {c: 0.7 for c in cats}), + ("best_t_in_sample", {c: per[c]["best_t"] for c in cats})] + ([("fixed", fixed_t)] if fixed_t else []): + mf = statistics.mean(prf(rows, c, t[c])["f1"] or 0 for c in cats) out[name] = {"macro_f1": mf, - "macro_f1_ctx": statistics.mean(prf(ctx, c, t[c])["f1"] or 0 for c in CATS) if ctx else None, + "macro_f1_ctx": statistics.mean(prf(ctx, c, t[c])["f1"] or 0 for c in cats) if ctx else None, "all": case_metrics(rows, t), "ctx": case_metrics(ctx, t), "no_ctx": case_metrics(nctx, t)} - out["macro_auc"] = mean_defined(per[c]["auc"] for c in CATS) - out["macro_auc_ctx"] = mean_defined(per[c]["auc_ctx"] for c in CATS) - out["macro_auc_n_categories"] = sum(per[c]["auc"] is not None for c in CATS) - out["macro_auc_ctx_n_categories"] = sum(per[c]["auc_ctx"] is not None for c in CATS) - out["reliability"] = reliability(rows) + out["macro_auc"] = mean_defined(per[c]["auc"] for c in cats) + out["macro_auc_ctx"] = mean_defined(per[c]["auc_ctx"] for c in cats) + out["macro_auc_n_categories"] = sum(per[c]["auc"] is not None for c in cats) + out["macro_auc_ctx_n_categories"] = sum(per[c]["auc_ctx"] is not None for c in cats) + out["reliability"] = reliability(rows, cats) out["latency"] = lat(rows) if all(choices): def ch(rs): @@ -299,15 +301,54 @@ def ch(rs): out["choice_argmax_per_cat"] = {c: { "tp": sum(r["choice"] == c and c in r["labels"] for r in rows), "fp": sum(r["choice"] == c and c not in r["labels"] for r in rows), - "fn": sum(r["choice"] != c and c in r["labels"] for r in rows)} for c in CATS} + "fn": sum(r["choice"] != c and c in r["labels"] for r in rows)} for c in cats} return out if __name__ == "__main__": here = os.path.dirname(os.path.abspath(__file__)) + if len(sys.argv) > 1: + usage = ("usage: score.py [--qualified MANIFEST DATASET RESULT " + "[--questions-file FILE] [--captured-requests FILE] [--sample-file FILE]]") + if sys.argv[1] != "--qualified" or len(sys.argv) < 5 or len(sys.argv[5:]) % 2: + raise ValueError(usage) + options = {} + for index in range(5, len(sys.argv), 2): + flag = sys.argv[index] + if flag not in ("--questions-file", "--captured-requests", "--sample-file") or flag in options: + raise ValueError(usage) + options[flag] = sys.argv[index + 1] + from qualification import load_manifest, load_questions, loads_unambiguous_json, qualified_report + from common import noul_questions + manifest, manifest_sha = load_manifest(sys.argv[2]) + with open(sys.argv[3], "rb") as source: + dataset_bytes = source.read() + with open(sys.argv[4], encoding="utf-8") as source: + result = loads_unambiguous_json(source.read()) + questions = load_questions(options["--questions-file"]) if "--questions-file" in options else noul_questions() + captured_requests_bytes = None + if "--captured-requests" in options: + with open(options["--captured-requests"], "rb") as source: + captured_requests_bytes = source.read() + report = qualified_report( + result, manifest, manifest_sha, questions, dataset_bytes, + captured_requests_bytes, options.get("--sample-file"), + ) + publish_summary(os.path.join(here, "qualification_summary.json"), report) + print(json.dumps(report, indent=2, allow_nan=False)) + # A completed evaluation is not necessarily a qualified model. Keep + # the measurements available while failing the calling automation. + if not report["qualified_against_frozen_tolerances"]: + failed = ", ".join(name for name, passed in report["checks"].items() if not passed) + print(f"qualification failed frozen checks: {failed}", file=sys.stderr) + sys.exit(1) + sys.exit(0) fixed = json.load(open(os.path.join(here, "fixed_thresholds.json"))) if os.path.exists(os.path.join(here, "fixed_thresholds.json")) else None summary = {}; allres = {} for f in sorted(glob.glob(os.path.join(here, "results", "*.json"))): res = json.load(open(f)) + if (isinstance(res.get("experiment"), dict) + and "qualification_manifest_sha256" in res["experiment"]): + raise ValueError("held-out results require --qualified MANIFEST DATASET RESULT; refuse in-sample scoring") if res["config"] in allres: raise ValueError("duplicate configuration %r in result files" % res["config"]) allres[res["config"]] = res diff --git a/evals/intent_engine/test_qualification.py b/evals/intent_engine/test_qualification.py new file mode 100644 index 00000000000..d2959a09a4c --- /dev/null +++ b/evals/intent_engine/test_qualification.py @@ -0,0 +1,2932 @@ +"""Falsifiable held-out qualification tests; no synthetic model is scored as Kev.""" +import copy +import hashlib +import json +import os +import shutil +import subprocess +import sys +import tempfile +import unittest +from pathlib import Path +from unittest.mock import patch + +import qualification +import run_http +import score +from common import CATS, calibrated, noul_questions + + +class QualificationTests(unittest.TestCase): + def setUp(self): + self.questions = noul_questions() + self.dataset = [ + {"id": "tune-1", "labels": ["impact"], "context_needed": True, + "current_prompt": "What breaks?", "turns": []}, + {"id": "tune-2", "labels": [], "context_needed": False, + "current_prompt": "Hello", "turns": []}, + {"id": "holdout-positive", "labels": ["impact"], "context_needed": True, + "current_prompt": "What changes?", "turns": []}, + {"id": "holdout-negative", "labels": [], "context_needed": False, + "current_prompt": "Show a greeting", "turns": []}, + ] + for case in self.dataset: + holdout = case["id"].startswith("holdout") + case["project_id"] = "project-b" if holdout else "project-a" + case["session_id"] = "session-b" if holdout else "session-a" + self.dataset_bytes = json.dumps(self.dataset).encode("utf-8") + self.dataset_sha = qualification.sha256_bytes(self.dataset_bytes) + annotations = [] + for case in self.dataset: + holdout = case["id"].startswith("holdout") + annotations.append({ + "id": case["id"], "split": "holdout" if holdout else "tune", + "project_id": "project-b" if holdout else "project-a", + "session_id": "session-b" if holdout else "session-a", + "author": "case-creator", "reviewer": "separate-reviewer", + "review_ref": "https://review.example/evidence/" + case["id"], + "verified_labels": case["labels"], + "verified_context_needed": case["context_needed"], + }) + self.endpoint = "http://127.0.0.1:8009/v1/systemone" + self.server_config = { + "checkpoint": "pinned", "server_revision": "r1", + "backend": "cpu", "dtype": "float32", "temperature": 2.41, + } + self.manifest = { + "version": 1, "application": "historical-intent-evaluation", + "dataset_sha256": self.dataset_sha, + "questions_sha256": qualification.sha256_ordered_questions(self.questions), + "holdout_requests_sha256": self.holdout_request_digest(), + "arm": { + "config": "B_noul_text", "requested_model": None, + "endpoint_sha256": qualification.sha256_bytes(self.endpoint.encode("utf-8")), + "server_config_sha256": qualification.sha256_json(self.server_config), + "timeout_ms": 1_800_000, + }, + "noul_temperature": None, + "max_request_bytes": 65536, + "max_input_tokens": 64, + "thresholds": {key: 0.6 for key in self.questions}, + "error_tolerances": { + "max_none_false_hint_rate": 0.0, + "max_unwanted_hint_rate": 0.0, + "min_positive_case_recall": 1.0, + "max_p95_ms": 20.0, + }, + "annotations": annotations, + } + self.manifest_bytes = json.dumps(self.manifest).encode("utf-8") + self.manifest_sha = qualification.sha256_bytes(self.manifest_bytes) + + def holdout_request_digest(self, questions=None, model=None, config="B_noul_text"): + questions = self.questions if questions is None else questions + fingerprints = [] + for case in self.dataset[2:]: + body = {"state": run_http.STATES[config](case), "questions": questions} + if model is not None: + body["model"] = model + fingerprints.append([ + case["id"], + hashlib.sha256(run_http.request_bytes(body)).hexdigest(), + ]) + return score.evidence_digest(fingerprints) + + def valid(self, manifest=None, cases=None): + return qualification.validate_manifest( + self.manifest if manifest is None else manifest, + self.dataset_sha, self.questions, + self.dataset if cases is None else cases, + ) + + def test_ambiguous_frozen_input_json_refuses_without_model_calls(self): + # Same-value duplicates are deliberately non-vacuous: the old + # last-key-wins parser accepted all three frozen artifacts. + raw_manifest = self.manifest_bytes.decode("utf-8") + raw_dataset = self.dataset_bytes.decode("utf-8") + raw_server = json.dumps(self.server_config) + variants = ( + ("manifest", raw_manifest.replace( + '"max_input_tokens": 64', + '"max_input_tokens": 64, "max_input_tokens": 64', 1)), + ("dataset", raw_dataset.replace( + '"project_id": "project-b"', + '"project_id": "project-b", "project_id": "project-b"', 1)), + ("server", raw_server.replace( + '"checkpoint": "pinned"', + '"checkpoint": "pinned", "checkpoint": "pinned"', 1)), + ) + for changed_file, ambiguous in variants: + with self.subTest(changed_file=changed_file): + with tempfile.TemporaryDirectory() as temp: + here = Path(temp) + dataset_bytes = (ambiguous.encode("utf-8") + if changed_file == "dataset" else self.dataset_bytes) + manifest = copy.deepcopy(self.manifest) + manifest["dataset_sha256"] = qualification.sha256_bytes(dataset_bytes) + (here / "dataset.json").write_bytes(dataset_bytes) + (here / "manifest.json").write_text( + ambiguous if changed_file == "manifest" else json.dumps(manifest) + ) + (here / "server.json").write_text( + ambiguous if changed_file == "server" else raw_server + ) + with patch.object(run_http, "HERE", str(here)), patch.object( + run_http, "post" + ) as post: + with self.assertRaisesRegex(ValueError, "duplicate JSON object key"): + run_http.main([ + "--url", self.endpoint, "--tag", "frozen", + "--server-config", str(here / "server.json"), + "--qualification-manifest", str(here / "manifest.json"), + "B_noul_text", + ]) + post.assert_not_called() + self.assertFalse((here / "results").exists()) + + def test_nonfinite_unused_manifest_field_is_not_reproducible_json(self): + with tempfile.TemporaryDirectory() as temp: + path = Path(temp) / "manifest.json" + path.write_bytes(self.manifest_bytes[:-1] + b', "unreviewed": NaN}') + with self.assertRaisesRegex(ValueError, "nonfinite JSON constant"): + qualification.load_manifest(path) + + def test_positive_and_negative_holdout_is_separate_from_tuning(self): + verified = self.valid() + self.assertEqual(verified["heldout_ids"], ["holdout-positive", "holdout-negative"]) + self.assertEqual(verified["thresholds"]["impact"], 0.6) + self.assertEqual(len(verified["annotations"]), 4) + + def test_group_identity_cannot_be_invented_by_the_annotation_manifest(self): + for field in ("project_id", "session_id"): + with self.subTest(field=field): + forged = copy.deepcopy(self.manifest) + forged["annotations"][2][field] = "invented-" + field + with self.assertRaisesRegex(ValueError, "group identity disagrees with frozen dataset"): + self.valid(forged) + missing_source = copy.deepcopy(self.dataset) + del missing_source[2][field] + with self.assertRaisesRegex(ValueError, "group identity disagrees with frozen dataset"): + self.valid(cases=missing_source) + + def test_self_consistent_rehashed_metadata_cannot_hide_group_forgery(self): + forged = copy.deepcopy(self.manifest) + forged["annotations"][2]["project_id"] = "invented-independent-project" + forged_sha = qualification.sha256_json(forged) + result = self.signed_result() + result["experiment"]["qualification_manifest_sha256"] = forged_sha + result["experiment_sha256"] = score.evidence_digest(result["experiment"]) + with self.assertRaisesRegex(ValueError, "group identity disagrees with frozen dataset"): + qualification.qualified_report( + result, forged, forged_sha, self.questions, self.dataset_bytes + ) + + def test_near_neighbor_session_or_project_leak_refuses_entire_split(self): + for key, value in (("project_id", "project-a"), ("session_id", "session-a")): + with self.subTest(key=key): + changed = copy.deepcopy(self.manifest) + changed["annotations"][2][key] = value + if key == "session_id": + changed["annotations"][2]["project_id"] = "project-a" + with self.assertRaisesRegex(ValueError, "leaks project/session"): + self.valid(changed) + + def test_reused_session_id_under_different_project_is_not_independent(self): + changed = copy.deepcopy(self.manifest) + changed["annotations"][2]["session_id"] = "session-a" + # Changing the project label must not conceal a shared session. + self.assertNotEqual(changed["annotations"][2]["project_id"], "project-a") + with self.assertRaisesRegex(ValueError, "leaks project/session"): + self.valid(changed) + + def test_unwanted_hints_on_positive_cases_are_not_hidden_by_recall(self): + result = self.signed_result() + result["rows"][0]["probs"]["call_graph"] = 0.99 + report = qualification.qualified_report( + result, self.manifest, self.manifest_sha, self.questions, self.dataset_bytes + ) + self.assertEqual(report["fixed_threshold_metrics"]["none_false_hint_rate"], 0) + self.assertEqual(report["fixed_threshold_metrics"]["pos_cases_with_a_correct_hint"], 1) + self.assertEqual(report["holdout_decisions"]["unwanted_hint_cases"], 1) + self.assertEqual(report["holdout_decisions"]["unwanted_hint_rate"], 0.5) + self.assertEqual(report["holdout_decisions"]["abstention_rate"], 0.5) + self.assertFalse(report["checks"]["unwanted_hint_rate"]) + self.assertFalse(report["qualified_against_frozen_tolerances"]) + + + def test_single_selected_wrong_hint_blocks_qualification_despite_any_hit_recall(self): + # The original all-hits recall is 1.0: impact is over threshold. + # Scout-like single selection would deliver the higher wrong category. + # Even a precommitted permissive unwanted-hint budget cannot allow the + # frozen positive-case recall check to accept that wrong first pick. + result = self.signed_result() + result["rows"][0]["probs"]["call_graph"] = 0.99 + frozen = copy.deepcopy(self.manifest) + frozen["error_tolerances"]["max_unwanted_hint_rate"] = 1.0 + manifest_sha = qualification.sha256_json(frozen) + result["experiment"]["qualification_manifest_sha256"] = manifest_sha + result["experiment_sha256"] = score.evidence_digest(result["experiment"]) + report = qualification.qualified_report( + result, frozen, manifest_sha, self.questions, self.dataset_bytes + ) + self.assertEqual(report["fixed_threshold_metrics"]["pos_cases_with_a_correct_hint"], 1) + self.assertEqual(report["holdout_decisions"]["unwanted_hint_rate"], 0.5) + self.assertEqual(report["holdout_decisions"]["model_only_selected_positive_case_recall"], 0) + self.assertEqual(report["holdout_decisions"]["model_only_selected_unwanted_hint_rate"], 0.5) + self.assertTrue(report["checks"]["unwanted_hint_rate"]) + self.assertFalse(report["checks"]["positive_case_recall"]) + self.assertFalse(report["qualified_against_frozen_tolerances"]) + + def test_single_selection_ties_follow_frozen_question_order(self): + positive = {"labels": ["impact"], "probs": {"impact": 0.9, "call_graph": 0.9}} + correct_first = qualification.holdout_decision_metrics( + [positive], {"impact": 0.6, "call_graph": 0.6}, ("impact", "call_graph"), + ) + wrong_first = qualification.holdout_decision_metrics( + [positive], {"call_graph": 0.6, "impact": 0.6}, ("call_graph", "impact"), + ) + self.assertEqual(correct_first["model_only_selected_positive_case_recall"], 1) + self.assertEqual(wrong_first["model_only_selected_positive_case_recall"], 0) + self.assertEqual(correct_first["unwanted_hint_rate"], 1) + self.assertEqual(wrong_first["unwanted_hint_rate"], 1) + + def test_threshold_manifest_key_order_cannot_change_scout_tie_selection(self): + # Production policy retains its configured question sequence on tied + # u16 relevance. The threshold object is an independent JSON map; + # changing only its key order must never turn a wrong first hint + # into a passing held-out positive case. + self.assertLess( + list(self.questions).index("call_graph"), + list(self.questions).index("impact"), + ) + result = self.signed_result() + result["rows"][0]["probs"]["call_graph"] = 0.9 + frozen = copy.deepcopy(self.manifest) + frozen["thresholds"] = dict(reversed(list(frozen["thresholds"].items()))) + self.assertLess( + list(frozen["thresholds"]).index("impact"), + list(frozen["thresholds"]).index("call_graph"), + ) + frozen["error_tolerances"]["max_unwanted_hint_rate"] = 1.0 + manifest_sha = qualification.sha256_json(frozen) + result["experiment"]["qualification_manifest_sha256"] = manifest_sha + result["experiment_sha256"] = score.evidence_digest(result["experiment"]) + report = qualification.qualified_report( + result, frozen, manifest_sha, self.questions, self.dataset_bytes, + ) + self.assertEqual( + report["holdout_decisions"]["model_only_selected_positive_case_recall"], 0, + ) + self.assertFalse(report["checks"]["positive_case_recall"]) + self.assertFalse(report["qualified_against_frozen_tolerances"]) + + def test_rust_permille_rounding_rejects_false_negative_case_pass(self): + result = self.signed_result() + # 0.5996 is below the historical float threshold of 0.6, but both + # become 600 permille in Scout's actual decision policy. + result["rows"][1]["probs"]["call_graph"] = 0.5996 + report = qualification.qualified_report( + result, self.manifest, self.manifest_sha, self.questions, + self.dataset_bytes, + ) + self.assertEqual(report["fixed_threshold_metrics"]["none_false_hint_rate"], 0) + self.assertEqual(report["holdout_decisions"]["none_false_hint_cases"], 1) + self.assertEqual(report["holdout_decisions"]["none_false_hint_rate"], 1) + self.assertFalse(report["checks"]["none_false_hint_rate"]) + self.assertFalse(report["qualified_against_frozen_tolerances"]) + + def test_rust_permille_rounding_admits_real_positive_and_refuses_below(self): + result = self.signed_result() + result["rows"][0]["probs"]["impact"] = 0.5996 + report = qualification.qualified_report( + result, self.manifest, self.manifest_sha, self.questions, + self.dataset_bytes, + ) + self.assertEqual( + report["fixed_threshold_metrics"]["pos_cases_with_a_correct_hint"], 0 + ) + self.assertEqual(report["holdout_decisions"]["model_only_selected_positive_case_recall"], 1) + self.assertTrue(report["checks"]["positive_case_recall"]) + self.assertTrue(report["qualified_against_frozen_tolerances"]) + + result["rows"][0]["probs"]["impact"] = 0.5994 + refused = qualification.qualified_report( + result, self.manifest, self.manifest_sha, self.questions, + self.dataset_bytes, + ) + self.assertEqual(refused["holdout_decisions"]["model_only_selected_positive_case_recall"], 0) + self.assertFalse(refused["checks"]["positive_case_recall"]) + + def test_scout_u16_relevance_tie_retains_question_order(self): + # The later incorrect answer is higher as an f64, but indistinguishable + # after Rust rounds relevance to u16. + row = { + "labels": ["impact"], + "probs": {"impact": 0.9, "call_graph": 0.9000000001}, + } + self.assertEqual( + qualification.runtime_probability_score(0.9, 65_535), + qualification.runtime_probability_score(0.9000000001, 65_535), + ) + correct = qualification.holdout_decision_metrics( + [row], {"impact": 0.6, "call_graph": 0.6}, ("impact", "call_graph"), + ) + wrong = qualification.holdout_decision_metrics( + [row], {"call_graph": 0.6, "impact": 0.6}, ("call_graph", "impact"), + ) + self.assertEqual(correct["model_only_selected_positive_case_recall"], 1) + self.assertEqual(wrong["model_only_selected_positive_case_recall"], 0) + + def test_native_kev_truncation_is_measured_not_assumed_from_a_flag(self): + # Native Kev emits state_tokens/state_tokens_used and a top-level + # truncated flag, not necessarily state_tokens_dropped. + with tempfile.TemporaryDirectory() as temp: + here = Path(temp) + (here / "dataset.json").write_bytes(self.dataset_bytes) + (here / "manifest.json").write_bytes(self.manifest_bytes) + (here / "server.json").write_text(json.dumps(self.server_config)) + args = [ + "--url", self.endpoint, "--tag", "native-kev", + "--server-config", str(here / "server.json"), + "--qualification-manifest", str(here / "manifest.json"), + "B_noul_text", + ] + calls = [] + + def native_answer(url, body, *, timeout): + self.assertEqual(timeout, self.manifest["arm"]["timeout_ms"] / 1000) + index = len(calls) + calls.append(body) + case = self.dataset[index + 2] + return { + "model": "self-reported-test-model", + "answers": { + key: {"type": "noul", "noul": 0.9 if key in case["labels"] else 0.1} + for key in self.questions + }, + "usage": { + "input_tokens": 64, "state_tokens": 32, + "state_tokens_used": 32 if index == 0 else 24, + }, + "truncated": index == 1, + "latency_ms": 5.0, + }, 10.0 + + with patch.object(run_http, "HERE", str(here)), patch.object( + run_http, "post", side_effect=native_answer + ): + run_http.main(args) + self.assertEqual(len(calls), 2) + result = json.loads( + (here / "results" / "native-kev_B_noul_text.json").read_text() + ) + self.assertEqual([r["dropped"] for r in result["rows"]], [0, 8]) + self.assertEqual([r["state_tokens"] for r in result["rows"]], [32, 32]) + self.assertEqual([r["state_tokens_used"] for r in result["rows"]], [32, 24]) + self.assertEqual([r["truncated"] for r in result["rows"]], [False, True]) + report = qualification.qualified_report( + result, self.manifest, self.manifest_sha, self.questions, + self.dataset_bytes, + ) + self.assertEqual(report["latency"]["cases_truncated"], 1) + self.assertEqual(report["numeric_truncation_source_evidence"]["measured_cases"], 2) + self.assertFalse(report["checks"]["untruncated_input_evidence"]) + self.assertFalse(report["qualified_against_frozen_tolerances"]) + self.assertTrue(report["checks"]["reported_input_token_budget"]) + # Rehashing only the declared experiment cannot hide a rewritten + # normalized drop count when native server counts are present. + forged = copy.deepcopy(result) + forged["rows"][1]["dropped"] = 0 + with self.assertRaisesRegex(ValueError, "truncation row disagrees"): + qualification.qualified_report( + forged, self.manifest, self.manifest_sha, + self.questions, self.dataset_bytes, + ) + + def test_conflicting_native_truncation_is_refused_before_row_append(self): + cases = ( + ("used_exceeds_original", {"truncated": False, + "usage": {"state_tokens": 24, "state_tokens_used": 32}}), + ("flag_contradicts_counts", {"truncated": False, + "usage": {"state_tokens": 32, "state_tokens_used": 24}}), + ("direct_contradicts_native", {"truncated": True, + "usage": {"state_tokens": 32, "state_tokens_used": 24, + "state_tokens_dropped": 0}}), + ("invalid_flag", {"truncated": 0, + "usage": {"state_tokens": 32, "state_tokens_used": 32}}), + ("flags_disagree", {"truncated": False, + "usage": {"truncated": True}}), + ("invalid_count", {"truncated": False, + "usage": {"state_tokens": True, "state_tokens_used": 32}}), + ) + for label, metadata in cases: + with self.subTest(label=label), tempfile.TemporaryDirectory() as temp: + here = Path(temp) + (here / "dataset.json").write_bytes(self.dataset_bytes) + (here / "manifest.json").write_bytes(self.manifest_bytes) + (here / "server.json").write_text(json.dumps(self.server_config)) + args = [ + "--url", self.endpoint, "--tag", "bad-native", + "--server-config", str(here / "server.json"), + "--qualification-manifest", str(here / "manifest.json"), + "B_noul_text", + ] + response = { + "model": "self-reported-test-model", + "answers": { + key: {"type": "noul", "noul": 0.5} + for key in self.questions + }, + "usage": {"input_tokens": 64, **metadata["usage"]}, + "truncated": metadata["truncated"], + } + with patch.object(run_http, "HERE", str(here)), patch.object( + run_http, "post", return_value=(response, 10.0) + ) as post: + with self.assertRaisesRegex(ValueError, "truncation"): + run_http.main(args) + post.assert_called_once() + self.assertFalse( + (here / "results" / "bad-native_B_noul_text.json").exists() + ) + partial = here / "results" / "bad-native_B_noul_text.json.partial" + self.assertEqual(len(partial.read_text().splitlines()), 1) + + + def test_native_truncation_partial_tampering_refuses_without_network(self): + with tempfile.TemporaryDirectory() as temp: + here = Path(temp) + (here / "dataset.json").write_bytes(self.dataset_bytes) + (here / "manifest.json").write_bytes(self.manifest_bytes) + (here / "server.json").write_text(json.dumps(self.server_config)) + args = [ + "--url", self.endpoint, "--tag", "native-partial", + "--server-config", str(here / "server.json"), + "--qualification-manifest", str(here / "manifest.json"), + "B_noul_text", + ] + response = { + "model": "self-reported-test-model", + "answers": { + key: {"type": "noul", "noul": 0.9} + for key in self.questions + }, + "usage": { + "input_tokens": 64, "state_tokens": 30, + "state_tokens_used": 25, + }, + "truncated": True, + } + with patch.object(run_http, "HERE", str(here)): + with patch.object(run_http, "post", side_effect=[ + (response, 10.0), RuntimeError("network interruption") + ]) as first: + with self.assertRaisesRegex(RuntimeError, "network interruption"): + run_http.main(args) + self.assertEqual(first.call_count, 2) + partial = here / "results" / "native-partial_B_noul_text.json.partial" + header, row = partial.read_text().splitlines() + original = json.loads(row) + self.assertEqual(original["dropped"], 5) + original["dropped"] = 0 + partial.write_text(header + "\n" + json.dumps(original) + "\n") + before = partial.read_bytes() + with patch.object(run_http, "post") as blocked: + with self.assertRaisesRegex(ValueError, "truncation row disagrees"): + run_http.main(args) + blocked.assert_not_called() + self.assertEqual(partial.read_bytes(), before) + self.assertFalse( + (here / "results" / "native-partial_B_noul_text.json").exists() + ) + + def test_impossible_token_counters_cannot_qualify_even_with_rehashed_evidence(self): + control = qualification.qualified_report( + self.signed_result(), self.manifest, self.manifest_sha, + self.questions, self.dataset_bytes, + ) + self.assertTrue(control["qualified_against_frozen_tolerances"]) + # The old scorer accepted input_tokens=1 as proof of context fit + # even when Kev reported using 32 state tokens on that request. + forged = self.signed_result() + forged["rows"][0].update( + input_tokens=1, state_tokens=32, state_tokens_used=32, dropped=0, + ) + with self.assertRaisesRegex(ValueError, "used state tokens exceed total input tokens"): + qualification.qualified_report( + forged, self.manifest, self.manifest_sha, + self.questions, self.dataset_bytes, + ) + + # A second contradictory server receipt also cannot be interpreted + # as a legitimate, zero-truncation or measured-drop result. + forged = self.signed_result() + forged["rows"][0].update( + state_tokens=3, reported_state_tokens_dropped=4, dropped=0, + ) + with self.assertRaisesRegex(ValueError, "dropped state tokens exceed original"): + qualification.qualified_report( + forged, self.manifest, self.manifest_sha, + self.questions, self.dataset_bytes, + ) + + def test_qualified_scorer_refuses_impossible_normalized_legacy_counts(self): + result = self.signed_result() + # Remove the native source count but keep an otherwise self-consistent + # frozen experiment. Unknown provenance cannot excuse impossible math. + del result["rows"][0]["reported_state_tokens_dropped"] + result["rows"][0].update(state_tokens=3, dropped=4) + with self.assertRaisesRegex(ValueError, "normalized dropped tokens exceed original state"): + qualification.qualified_report( + result, self.manifest, self.manifest_sha, + self.questions, self.dataset_bytes, + ) + + def test_boolean_truncation_without_counts_is_unknown(self): + self.assertIsNone(run_http.dropped_state_tokens({"truncated": False}, {})) + self.assertIsNone(run_http.dropped_state_tokens({"truncated": True}, {})) + self.assertEqual( + run_http.dropped_state_tokens( + {"truncated": False}, {"state_tokens_dropped": 0} + ), 0 + ) + + def test_missing_or_dropped_server_tokens_cannot_qualify(self): + for value in (None, 1): + with self.subTest(state_tokens_dropped=value): + result = self.signed_result() + result["rows"][0].update( + dropped=value, reported_state_tokens_dropped=value, + ) + report = qualification.qualified_report( + result, self.manifest, self.manifest_sha, self.questions, self.dataset_bytes + ) + self.assertFalse(report["checks"]["untruncated_input_evidence"]) + self.assertFalse(report["qualified_against_frozen_tolerances"]) + self.assertEqual( + report["latency"]["truncation_measured_cases"], + 1 if value is None else 2, + ) + complete = qualification.qualified_report( + self.signed_result(), self.manifest, self.manifest_sha, + self.questions, self.dataset_bytes, + ) + self.assertTrue(complete["checks"]["untruncated_input_evidence"]) + self.assertTrue(complete["qualified_against_frozen_tolerances"]) + + def test_forged_zero_drops_without_server_counter_does_not_qualify(self): + result = self.signed_result() + del result["rows"][0]["reported_state_tokens_dropped"] + report = qualification.qualified_report( + result, self.manifest, self.manifest_sha, + self.questions, self.dataset_bytes, + ) + # The old aggregate accepted two claimed zeros even though one row + # had no original numeric server evidence. That is not measured zero. + self.assertEqual(report["latency"]["cases_truncated"], 0) + self.assertEqual(report["latency"]["truncation_measured_cases"], 2) + self.assertEqual( + report["numeric_truncation_source_evidence"], + {"measured_cases": 1, "missing_cases": 1}, + ) + self.assertFalse(report["checks"]["untruncated_input_evidence"]) + self.assertFalse(report["qualified_against_frozen_tolerances"]) + + def test_reviewed_labels_are_not_inferred_from_prediction_or_actions(self): + for name, mutate, refusal in ( + ("self_label", lambda m: m["annotations"][2].update(reviewer="case-creator"), "self-labelled"), + ("missing_review", lambda m: m["annotations"][2].update(review_ref=""), "review_ref"), + ("ambiguous_label", lambda m: m["annotations"][2].update(verified_labels=["impact", "impact"]), "invalid verified labels"), + ("missing_negative", lambda m: m["annotations"][3].update(split="tune"), "positive/negative"), + ("unreviewed_question", lambda m: m["annotations"][3].update(verified_labels=["unknown"]), "invalid verified labels"), + ): + with self.subTest(name=name): + manifest = copy.deepcopy(self.manifest) + mutate(manifest) + with self.assertRaisesRegex(ValueError, refusal): + self.valid(manifest) + changed = copy.deepcopy(self.dataset) + changed[2]["labels"] = [] + with self.assertRaisesRegex(ValueError, "disagrees with the dataset"): + self.valid(cases=changed) + + def test_question_or_threshold_drift_fails_before_inference(self): + for name, mutate, refusal in ( + ("question", lambda m: m.update(questions_sha256="f" * 64), "question vocabulary"), + ("threshold", lambda m: m["thresholds"].pop("impact"), "thresholds"), + ("nonfinite", lambda m: m["thresholds"].update(impact=float("nan")), "finite"), + ("unfrozen_tolerance", lambda m: m["error_tolerances"].pop("max_p95_ms"), "tolerances"), + ("unfrozen_unwanted", lambda m: m["error_tolerances"].pop("max_unwanted_hint_rate"), "tolerances"), + ("dataset", lambda m: m.update(dataset_sha256="f" * 64), "dataset digest mismatch"), + ("missing_calibration", lambda m: m.pop("noul_temperature"), "frozen client calibration"), + ("nonfinite_calibration", lambda m: m.update(noul_temperature=float("nan")), "client calibration"), + ("zero_calibration", lambda m: m.update(noul_temperature=0), "client calibration"), + ("boolean_calibration", lambda m: m.update(noul_temperature=True), "client calibration"), + ): + with self.subTest(name=name): + changed = copy.deepcopy(self.manifest) + mutate(changed) + with self.assertRaisesRegex(ValueError, refusal): + self.valid(changed) + + def signed_result(self): + rows = [] + for case in self.dataset[2:]: + probs = {cat: (0.9 if cat in case["labels"] else 0.1) for cat in CATS} + rendered = run_http.request_bytes({ + "state": run_http.STATES["B_noul_text"](case), + "questions": self.questions, + }) + rows.append({ + "id": case["id"], "labels": case["labels"], + "context_needed": case["context_needed"], + "probs": probs, + "ms": 10 if case["labels"] else 20, + "input_tokens": 64, + "dropped": 0, + "reported_state_tokens_dropped": 0, + "request_sha256": hashlib.sha256(rendered).hexdigest(), + "request_size_bytes": len(rendered), + "observed_model": "self-reported-test-model", + }) + experiment = { + "version": 2, + "config": "B_noul_text", + "requested_model": None, + "endpoint_sha256": self.manifest["arm"]["endpoint_sha256"], + "timeout_ms": self.manifest["arm"]["timeout_ms"], + "case_ids": [row["id"] for row in rows], + "dataset_sha256": self.dataset_sha, + "case_annotations_sha256": score.evidence_digest([ + [r["id"], r["labels"], r["context_needed"]] for r in rows + ]), + "requests_sha256": score.evidence_digest([ + [r["id"], r["request_sha256"]] for r in rows + ]), + "noul_temperature": None, + "server_config": copy.deepcopy(self.server_config), + "qualification_manifest_sha256": self.manifest_sha, + } + return { + "config": "trial_B_noul_text", + "dataset_sha256": self.dataset_sha, + "experiment": experiment, + "experiment_sha256": score.evidence_digest(experiment), + "rows": rows, + } + + def test_only_frozen_thresholds_qualify_and_latency_reports_p95(self): + result = self.signed_result() + report = qualification.qualified_report( + result, self.manifest, self.manifest_sha, self.questions, self.dataset_bytes + ) + self.assertTrue(report["qualified_against_frozen_tolerances"]) + self.assertEqual(report["cases"], 2) + self.assertEqual(report["latency"]["p50_ms"], 15) + self.assertEqual(report["latency"]["p95_ms"], 20) + self.assertEqual(report["request_size_bytes"]["frozen_limit"], 65536) + self.assertEqual(report["input_token_evidence"]["frozen_limit"], 64) + self.assertEqual(report["input_token_evidence"]["measured_cases"], 2) + self.assertEqual(report["input_token_evidence"]["max_reported_tokens"], 64) + self.assertTrue(report["checks"]["reported_input_token_budget"]) + self.assertEqual(report["request_size_bytes"]["total"], sum( + row["request_size_bytes"] for row in result["rows"] + )) + self.assertEqual(report["holdout_decisions"]["coverage_rate"], 0.5) + self.assertEqual(report["holdout_decisions"]["abstention_rate"], 0.5) + self.assertEqual(report["holdout_decisions"]["unwanted_hint_rate"], 0) + self.assertTrue(report["checks"]["untruncated_input_evidence"]) + self.assertEqual(report["reported_model"], "self-reported-test-model") + self.assertTrue(report["checks"]["reported_model_consistency"]) + self.assertEqual(report["fixed_threshold_metrics"]["none_false_hint_rate"], 0) + self.assertEqual(report["fixed_threshold_metrics"]["pos_cases_with_a_correct_hint"], 1) + + stricter = copy.deepcopy(self.manifest) + stricter["error_tolerances"]["max_p95_ms"] = 19 + # An altered post-run manifest is not a valid requalification. + with self.assertRaisesRegex(ValueError, "not bound"): + qualification.qualified_report(result, stricter, qualification.sha256_json(stricter), self.questions, self.dataset_bytes) + + + def test_independent_qualified_scoring_rechecks_first_response_sample(self): + result = self.signed_result() + first = result["rows"][0] + case = self.dataset[2] + request = { + "state": run_http.STATES["B_noul_text"](case), + "questions": self.questions, + } + response = { + "model": first["observed_model"], + "answers": { + key: {"type": "noul", "noul": value} + for key, value in first["probs"].items() + }, + "usage": {"input_tokens": 64, "state_tokens_dropped": 0}, + } + first["sample_response_sha256"] = run_http.digest(response) + sample = { + "request": request, + "response": response, + "experiment_sha256": result["experiment_sha256"], + "case_id": first["id"], + "request_sha256": first["request_sha256"], + "row_sha256": run_http.digest(first), + } + with tempfile.TemporaryDirectory() as temp: + path = Path(temp) / "raw-first-response.json" + path.write_text(json.dumps(sample), encoding="utf-8") + + def score_sample(value=result): + return qualification.qualified_report( + value, self.manifest, self.manifest_sha, self.questions, + self.dataset_bytes, sample_path=str(path), + ) + + report = score_sample() + self.assertTrue(report["qualified_against_frozen_tolerances"]) + self.assertTrue(report["first_response_sample_checked"]) + with self.assertRaisesRegex(ValueError, "must be provided together"): + qualification.qualified_report( + result, self.manifest, self.manifest_sha, + self.questions, self.dataset_bytes, + ) + + wrong = copy.deepcopy(sample) + wrong["response"]["answers"]["impact"]["noul"] = 0.31 + path.write_text(json.dumps(wrong), encoding="utf-8") + with self.assertRaisesRegex(ValueError, "raw response disagrees with first scored row"): + score_sample() + + wrong = copy.deepcopy(sample) + wrong["request"]["state"] += " Foreign transcript" + path.write_text(json.dumps(wrong), encoding="utf-8") + with self.assertRaisesRegex(ValueError, "request differs"): + score_sample() + + path.write_text(json.dumps(sample), encoding="utf-8") + altered_result = copy.deepcopy(result) + altered_result["rows"][0]["ms"] = 9 + with self.assertRaisesRegex(ValueError, "response or row disagrees"): + score_sample(altered_result) + + later_sample = copy.deepcopy(result) + later_sample["rows"][1]["sample_response_sha256"] = "f" * 64 + with self.assertRaisesRegex(ValueError, "outside the first"): + score_sample(later_sample) + + without_commitment = self.signed_result() + with self.assertRaisesRegex(ValueError, "must be provided together"): + score_sample(without_commitment) + + def test_rehashed_first_sample_cannot_disagree_with_scored_prediction_or_usage(self): + # A sample's row_sha256 and a row's sample_response_sha256 are + # mutually rehashable. The original raw response must independently + # reproduce the frozen model observation, probabilities, and telemetry. + result = self.signed_result() + first = result["rows"][0] + request = { + "state": run_http.STATES["B_noul_text"](self.dataset[2]), + "questions": self.questions, + } + raw_response = { + "model": first["observed_model"], + "answers": { + key: {"type": "noul", "noul": value} + for key, value in first["probs"].items() + }, + "usage": {"input_tokens": 64, "state_tokens_dropped": 0}, + } + with tempfile.TemporaryDirectory() as temp: + path = Path(temp) / "first-response.json" + + def score_raw(response): + adjusted = copy.deepcopy(result) + adjusted_first = adjusted["rows"][0] + adjusted_first["sample_response_sha256"] = run_http.digest(response) + evidence = { + "request": request, + "response": response, + "experiment_sha256": adjusted["experiment_sha256"], + "case_id": adjusted_first["id"], + "request_sha256": adjusted_first["request_sha256"], + "row_sha256": run_http.digest(adjusted_first), + } + path.write_text(json.dumps(evidence), encoding="utf-8") + return qualification.qualified_report( + adjusted, self.manifest, self.manifest_sha, + self.questions, self.dataset_bytes, sample_path=str(path), + ) + + self.assertTrue(score_raw(raw_response)["qualified_against_frozen_tolerances"]) + for corruption in ("probabilities", "model", "input_tokens", "dropped"): + with self.subTest(corruption=corruption): + edited = copy.deepcopy(raw_response) + if corruption == "probabilities": + edited["answers"]["impact"]["noul"] = 0.01 + elif corruption == "model": + edited["model"] = "different-checkpoint" + elif corruption == "input_tokens": + edited["usage"]["input_tokens"] = 1024 + else: + edited["usage"]["state_tokens_dropped"] = 1 + # Rehash *both* directions; these records used to pass + # despite contradicting the first response actually scored. + with self.assertRaisesRegex(ValueError, "raw response disagrees"): + score_raw(edited) + + def test_qualified_scorer_rejects_edited_choice_and_extra_none_evidence(self): + # These edits previously passed the generic historical scorer. An + # independently scored Noul-only result must reject them even when + # the request hashes, experiment SHA and verified labels still agree. + control = self.signed_result() + self.assertTrue(qualification.qualified_report( + control, self.manifest, self.manifest_sha, + self.questions, self.dataset_bytes, + )["qualified_against_frozen_tolerances"]) + + for variant in ("extra_none_probability", "choice_on_every_case"): + with self.subTest(variant=variant): + forged = copy.deepcopy(control) + if variant == "extra_none_probability": + forged["rows"][0]["probs"]["none"] = 0.95 + else: + for row in forged["rows"]: + row["choice"] = "none" + + # An unqualified historical result may legitimately have + # these fields. This passing control proves the old scored + # path would not refuse our adversarial evidence. + self.assertIn( + "latency", + score.score_config(forged, cats=tuple(self.questions)), + ) + with self.assertRaisesRegex(ValueError, "only frozen Noul probabilities"): + qualification.qualified_report( + forged, self.manifest, self.manifest_sha, + self.questions, self.dataset_bytes, + ) + + def test_qualification_reports_missing_or_drifting_model_identity(self): + for value in (None, "", True, "another-model"): + with self.subTest(observed_model=value): + result = self.signed_result() + result["rows"][1]["observed_model"] = value + report = qualification.qualified_report( + result, self.manifest, self.manifest_sha, + self.questions, self.dataset_bytes, + ) + self.assertFalse(report["qualified_against_frozen_tolerances"]) + self.assertFalse(report["checks"]["reported_model_consistency"]) + self.assertIsNone(report["reported_model"]) + + def test_qualification_rejects_model_different_from_frozen_selector(self): + manifest = copy.deepcopy(self.manifest) + manifest["arm"]["requested_model"] = "pinned-model-selector" + manifest["holdout_requests_sha256"] = self.holdout_request_digest(model="pinned-model-selector") + manifest_sha = qualification.sha256_bytes( + json.dumps(manifest).encode("utf-8") + ) + result = self.signed_result() + experiment = result["experiment"] + experiment["qualification_manifest_sha256"] = manifest_sha + experiment["requested_model"] = "pinned-model-selector" + for row in result["rows"]: + case = next(c for c in self.dataset if c["id"] == row["id"]) + body = { + "state": run_http.STATES["B_noul_text"](case), + "questions": self.questions, + "model": "pinned-model-selector", + } + rendered = run_http.request_bytes(body) + row["request_sha256"] = hashlib.sha256(rendered).hexdigest() + row["request_size_bytes"] = len(rendered) + experiment["requests_sha256"] = score.evidence_digest([ + [row["id"], row["request_sha256"]] for row in result["rows"] + ]) + result["experiment_sha256"] = score.evidence_digest(experiment) + report = qualification.qualified_report( + result, manifest, manifest_sha, self.questions, self.dataset_bytes + ) + self.assertFalse(report["checks"]["reported_model_consistency"]) + self.assertFalse(report["qualified_against_frozen_tolerances"]) + + def test_runner_refuses_explicit_model_mismatch_before_result_append(self): + with tempfile.TemporaryDirectory() as temp: + here = Path(temp) + (here / "dataset.json").write_bytes(self.dataset_bytes) + manifest = copy.deepcopy(self.manifest) + manifest["arm"]["requested_model"] = "pinned-model-selector" + manifest["holdout_requests_sha256"] = self.holdout_request_digest(model="pinned-model-selector") + (here / "manifest.json").write_text(json.dumps(manifest)) + (here / "server.json").write_text(json.dumps(self.server_config)) + response = { + "model": "wrong-model", + "answers": { + key: {"type": "noul", "noul": 0.5} + for key in self.questions + }, + } + args = [ + "--url", self.endpoint, + "--tag", "model-mismatch", + "--server-config", str(here / "server.json"), + "--qualification-manifest", str(here / "manifest.json"), + "--send-model", "pinned-model-selector", "B_noul_text", + ] + with patch.object(run_http, "HERE", str(here)): + with patch.object(run_http, "post", return_value=(response, 12.0)) as post: + with self.assertRaisesRegex(ValueError, "reported model differs"): + run_http.main(args) + self.assertEqual(post.call_count, 1) + self.assertFalse((here / "results" / "model-mismatch_B_noul_text.json").exists()) + partial = here / "results" / "model-mismatch_B_noul_text.json.partial" + self.assertEqual(len(partial.read_text().splitlines()), 1) + + def test_qualified_inference_refuses_server_model_change_before_second_append(self): + with tempfile.TemporaryDirectory() as temp: + here = Path(temp) + (here / "dataset.json").write_bytes(self.dataset_bytes) + (here / "manifest.json").write_bytes(self.manifest_bytes) + (here / "server.json").write_text(json.dumps(self.server_config)) + args = [ + "--url", self.endpoint, "--tag", "model-drift", + "--server-config", str(here / "server.json"), + "--qualification-manifest", str(here / "manifest.json"), + "B_noul_text", + ] + + def reply(model): + return { + "model": model, + "answers": {key: {"type": "noul", "noul": 0.5} + for key in self.questions}, + "usage": {"input_tokens": 64, "state_tokens": 32, + "state_tokens_used": 32}, + "truncated": False, + }, 10.0 + + with patch.object(run_http, "HERE", str(here)), patch.object( + run_http, "post", side_effect=[ + reply("reported-model-a"), reply("reported-model-b"), + ] + ) as post: + with self.assertRaisesRegex(ValueError, "mix reported models"): + run_http.main(args) + self.assertEqual(post.call_count, 2) + path = here / "results" / "model-drift_B_noul_text.json" + self.assertFalse(path.exists()) + header, first = (here / "results" / "model-drift_B_noul_text.json.partial").read_text().splitlines() + self.assertEqual(json.loads(first)["observed_model"], "reported-model-a") + self.assertEqual(json.loads(header)["experiment"]["case_ids"], + ["holdout-positive", "holdout-negative"]) + + def test_qualified_resume_refuses_forged_model_before_network(self): + for altered_model in (None, "different-model"): + with self.subTest(altered_model=altered_model), tempfile.TemporaryDirectory() as temp: + here = Path(temp) + (here / "dataset.json").write_bytes(self.dataset_bytes) + manifest = copy.deepcopy(self.manifest) + manifest["arm"]["requested_model"] = "selected-model" + manifest["holdout_requests_sha256"] = self.holdout_request_digest(model="selected-model") + (here / "manifest.json").write_text(json.dumps(manifest)) + (here / "server.json").write_text(json.dumps(self.server_config)) + args = [ + "--url", self.endpoint, "--tag", "model-resume", + "--server-config", str(here / "server.json"), + "--qualification-manifest", str(here / "manifest.json"), + "--send-model", "selected-model", "B_noul_text", + ] + answer = { + "model": "selected-model", + "answers": {key: {"type": "noul", "noul": 0.5} + for key in self.questions}, + "usage": {"input_tokens": 64, "state_tokens": 32, + "state_tokens_used": 32}, + "truncated": False, + } + with patch.object(run_http, "HERE", str(here)): + with patch.object(run_http, "post", side_effect=[ + (answer, 10.0), OSError("interrupted"), + ]) as first: + with self.assertRaisesRegex(OSError, "interrupted"): + run_http.main(args) + self.assertEqual(first.call_count, 2) + partial = here / "results" / "model-resume_B_noul_text.json.partial" + original = partial.read_bytes() + header, row = original.decode("utf-8").splitlines() + corrupted = json.loads(row) + corrupted["observed_model"] = altered_model + partial.write_text(header + "\n" + json.dumps(corrupted) + "\n") + before = partial.read_bytes() + with patch.object(run_http, "post") as blocked: + expected = "requires a server-reported model" if altered_model is None else "frozen selector" + with self.assertRaisesRegex(ValueError, expected): + run_http.main(args) + blocked.assert_not_called() + self.assertEqual(partial.read_bytes(), before) + self.assertFalse( + (here / "results" / "model-resume_B_noul_text.json").exists() + ) + # The valid original prefix must still resume normally. + partial.write_bytes(original) + with patch.object(run_http, "post", return_value=(answer, 10.0)) as remaining: + run_http.main(args) + remaining.assert_called_once() + self.assertEqual(len(json.loads( + (here / "results" / "model-resume_B_noul_text.json").read_text() + )["rows"]), 2) + + def test_frozen_arm_refuses_missing_reported_model_before_appending(self): + # With no explicit model selector, an absent identity used to be + # appended and the runner continued spending holdout calls. + good = { + "model": "observed-checkpoint", + "answers": {key: {"type": "noul", "noul": 0.5} + for key in self.questions}, + "usage": {"input_tokens": 64, "state_tokens": 32, + "state_tokens_used": 32}, + "truncated": False, + } + for index in (0, 1): + with self.subTest(failing_case=index), tempfile.TemporaryDirectory() as temp: + here = Path(temp) + (here / "dataset.json").write_bytes(self.dataset_bytes) + (here / "manifest.json").write_bytes(self.manifest_bytes) + (here / "server.json").write_text(json.dumps(self.server_config)) + replies = [copy.deepcopy(good), copy.deepcopy(good)] + del replies[index]["model"] + args = [ + "--url", self.endpoint, "--tag", "missing-model", + "--server-config", str(here / "server.json"), + "--qualification-manifest", str(here / "manifest.json"), + "B_noul_text", + ] + with patch.object(run_http, "HERE", str(here)), patch.object( + run_http, "post", side_effect=[(reply, 10.0) for reply in replies] + ) as post: + with self.assertRaisesRegex(ValueError, "requires a server-reported model"): + run_http.main(args) + self.assertEqual(post.call_count, index + 1) + completed = here / "results" / "missing-model_B_noul_text.json" + self.assertFalse(completed.exists()) + partial = Path(str(completed) + ".partial") + recorded = partial.read_text().splitlines() + self.assertEqual(len(recorded), index + 1) + if index: + self.assertEqual(json.loads(recorded[1])["observed_model"], + "observed-checkpoint") + + def test_frozen_resume_refuses_missing_default_model_before_network(self): + # No --send-model: absent/null/blank names must still be refused before + # another held-out inference. An unchanged valid prefix can resume. + with tempfile.TemporaryDirectory() as temp: + here = Path(temp) + (here / "dataset.json").write_bytes(self.dataset_bytes) + (here / "manifest.json").write_bytes(self.manifest_bytes) + (here / "server.json").write_text(json.dumps(self.server_config)) + args = [ + "--url", self.endpoint, "--tag", "missing-resume-model", + "--server-config", str(here / "server.json"), + "--qualification-manifest", str(here / "manifest.json"), + "B_noul_text", + ] + reply = { + "model": "observed-checkpoint", + "answers": {key: {"type": "noul", "noul": 0.5} + for key in self.questions}, + "usage": {"input_tokens": 64, "state_tokens": 32, + "state_tokens_used": 32}, + "truncated": False, + } + with patch.object(run_http, "HERE", str(here)): + with patch.object(run_http, "post", side_effect=[ + (reply, 10.0), OSError("interrupted") + ]): + with self.assertRaisesRegex(OSError, "interrupted"): + run_http.main(args) + completed = here / "results" / "missing-resume-model_B_noul_text.json" + partial = Path(str(completed) + ".partial") + original = partial.read_bytes() + header, row = original.decode("utf-8").splitlines() + for changed in ("absent", "null", "blank"): + with self.subTest(changed=changed): + forged = json.loads(row) + if changed == "absent": + del forged["observed_model"] + else: + forged["observed_model"] = None if changed == "null" else " " + partial.write_text(header + "\n" + json.dumps(forged) + "\n") + before = partial.read_bytes() + with patch.object(run_http, "post") as blocked: + with self.assertRaisesRegex(ValueError, "requires a server-reported model"): + run_http.main(args) + blocked.assert_not_called() + self.assertEqual(partial.read_bytes(), before) + self.assertFalse(completed.exists()) + partial.write_bytes(original) + with patch.object(run_http, "post", return_value=(reply, 10.0)) as remaining: + run_http.main(args) + remaining.assert_called_once() + self.assertEqual([r["observed_model"] for r in json.loads( + completed.read_text() + )["rows"]], ["observed-checkpoint", "observed-checkpoint"]) + + def test_qualification_requires_complete_serving_identity(self): + for field, change in ( + ("missing_server", lambda r: r["experiment"].pop("server_config")), + ("missing_checkpoint", lambda r: r["experiment"]["server_config"].pop("checkpoint")), + ("unknown_revision", lambda r: r["experiment"]["server_config"].update(server_revision="")), + ("nan_temperature", lambda r: r["experiment"]["server_config"].update(temperature=float("nan"))), + ): + with self.subTest(field=field): + result = self.signed_result() + change(result) + with self.assertRaisesRegex(ValueError, "serving"): + qualification.qualified_report( + result, self.manifest, self.manifest_sha, self.questions, self.dataset_bytes + ) + + def test_qualified_reporting_refuses_tampered_labels_and_unbound_results(self): + base = self.signed_result() + for field, mutation, pattern in ( + ("label", lambda r: r["rows"][0].update(labels=[]), "reviewed labels"), + ("order", lambda r: r["rows"].reverse(), "held-out case order"), + ("non_object_row", lambda r: r["rows"].append(None), "held-out case order"), + ("identity", lambda r: r["experiment"].pop("qualification_manifest_sha256"), "not bound"), + ("forged_review", lambda r: r["rows"][1].update(context_needed=True), "reviewed labels"), + ): + with self.subTest(field=field): + result = copy.deepcopy(base) + mutation(result) + with self.assertRaisesRegex(ValueError, pattern): + qualification.qualified_report(result, self.manifest, self.manifest_sha, self.questions, self.dataset_bytes) + + + def test_frozen_client_calibration_cannot_be_changed_by_rehashing(self): + # Rehashing both the declared manifest and experiment must not make + # an unfrozen post-hoc client temperature qualify. + for temperature in (0.5, 1.25): + with self.subTest(temperature=temperature): + changed = copy.deepcopy(self.manifest) + changed["noul_temperature"] = temperature + changed_sha = qualification.sha256_json(changed) + forged = self.signed_result() + forged["experiment"]["qualification_manifest_sha256"] = changed_sha + forged["experiment_sha256"] = score.evidence_digest(forged["experiment"]) + with self.assertRaisesRegex(ValueError, "client calibration differs"): + qualification.qualified_report( + forged, changed, changed_sha, self.questions, self.dataset_bytes + ) + + def test_http_runner_requires_frozen_client_calibration_before_network(self): + with tempfile.TemporaryDirectory() as temp: + here = Path(temp) + (here / "dataset.json").write_bytes(self.dataset_bytes) + manifest_path = here / "manifest.json" + manifest_path.write_bytes(self.manifest_bytes) + (here / "server.json").write_text(json.dumps(self.server_config)) + args = [ + "--url", "http://127.0.0.1:8009/v1/systemone", + "--tag", "calibration", + "--server-config", str(here / "server.json"), + "--qualification-manifest", str(manifest_path), + "--noul-temperature", "1.25", "B_noul_text", + ] + with patch.object(run_http, "HERE", str(here)): + with patch.object(run_http, "post") as blocked: + with self.assertRaisesRegex(ValueError, "client calibration differs"): + run_http.main(args) + blocked.assert_not_called() + self.assertFalse((here / "results").exists()) + frozen = copy.deepcopy(self.manifest) + frozen["noul_temperature"] = 1.25 + manifest_path.write_text(json.dumps(frozen)) + answer = {"model": "self-reported-test-model", "answers": { + key: {"type": "noul", "noul": 0.5} + for key in self.questions + }} + with patch.object(run_http, "post", return_value=(answer, 12.0)) as infer: + run_http.main(args) + self.assertEqual(infer.call_count, 2) + result = json.loads( + (here / "results" / "calibration_B_noul_text.json").read_text() + ) + self.assertEqual(result["experiment"]["noul_temperature"], 1.25) + self.assertEqual(result["rows"][0]["probs_raw"]["impact"], 0.5) + report = qualification.qualified_report( + result, frozen, qualification.sha256_bytes(manifest_path.read_bytes()), + self.questions, self.dataset_bytes, + ) + self.assertFalse(report["qualified_against_frozen_tolerances"]) + + def test_http_runner_asks_only_pre_frozen_holdout_and_resumes_with_exact_identity(self): + with tempfile.TemporaryDirectory() as temp: + here = Path(temp) + (here / "dataset.json").write_bytes(self.dataset_bytes) + (here / "manifest.json").write_bytes(self.manifest_bytes) + (here / "server.json").write_text(json.dumps(self.server_config)) + calls = [] + + def answer(url, body, **kwargs): + calls.append(body) + return {"model": "self-reported-test-model", "answers": { + key: {"type": "noul", "noul": 0.5} + for key in body["questions"] + }}, 12.0 + + args = [ + "--url", "http://127.0.0.1:8009/v1/systemone", + "--tag", "frozen", + "--server-config", str(here / "server.json"), + "--qualification-manifest", str(here / "manifest.json"), + "B_noul_text", + ] + with patch.object(run_http, "HERE", str(here)), patch.object(run_http, "post", side_effect=answer): + run_http.main(args) + self.assertEqual(len(calls), 2, "must exclude training/tune cases from inference") + result = json.loads((here / "results" / "frozen_B_noul_text.json").read_text()) + self.assertEqual( + [row["request_size_bytes"] for row in result["rows"]], + [len(run_http.request_bytes(body)) for body in calls], + ) + self.assertEqual([r["id"] for r in result["rows"]], [ + "holdout-positive", "holdout-negative" + ]) + self.assertEqual( + [r["observed_model"] for r in result["rows"]], + ["self-reported-test-model", "self-reported-test-model"], + ) + self.assertEqual(result["experiment"]["qualification_manifest_sha256"], self.manifest_sha) + self.assertEqual(result["experiment"]["case_ids"], [ + "holdout-positive", "holdout-negative" + ]) + + def test_qualified_sample_is_bound_to_the_frozen_experiment_on_resume(self): + with tempfile.TemporaryDirectory() as temp: + here = Path(temp) + (here / "dataset.json").write_bytes(self.dataset_bytes) + (here / "manifest.json").write_bytes(self.manifest_bytes) + (here / "server.json").write_text(json.dumps(self.server_config)) + sample = here / "first-response.json" + args = [ + "--url", self.endpoint, + "--tag", "sample-provenance", + "--server-config", str(here / "server.json"), + "--qualification-manifest", str(here / "manifest.json"), + "--sample-out", str(sample), + "B_noul_text", + ] + calls = [] + + def answer(url, body, **kwargs): + calls.append(body) + if len(calls) == 2: + raise OSError("interrupted before second response") + return { + "model": "self-reported-test-model", + "answers": {key: {"type": "noul", "noul": 0.2} + for key in body["questions"]}, + }, 4.0 + + partial = here / "results" / "sample-provenance_B_noul_text.json.partial" + final = here / "results" / "sample-provenance_B_noul_text.json" + with patch.object(run_http, "HERE", str(here)), patch.object( + run_http, "post", side_effect=answer + ): + with self.assertRaisesRegex(OSError, "interrupted before second response"): + run_http.main(args) + self.assertEqual(len(calls), 2) + self.assertTrue(partial.exists()) + self.assertTrue(sample.exists()) + self.assertFalse(final.exists()) + sample_bytes = sample.read_bytes() + evidence = json.loads(sample_bytes) + first_row = json.loads(partial.read_text().splitlines()[1]) + self.assertEqual(first_row["sample_response_sha256"], run_http.digest(evidence["response"])) + self.assertEqual(evidence["row_sha256"], run_http.digest(first_row)) + header = json.loads(partial.read_text().splitlines()[0]) + self.assertEqual(evidence["experiment_sha256"], header["experiment_sha256"]) + self.assertEqual(evidence["case_id"], "holdout-positive") + self.assertEqual( + evidence["request_sha256"], + hashlib.sha256(run_http.request_bytes(calls[0])).hexdigest(), + ) + partial_bytes = partial.read_bytes() + for field, replacement in ( + ("experiment_sha256", "0" * 64), + ("request_sha256", "f" * 64), + ): + with self.subTest(field=field): + forged = copy.deepcopy(evidence) + forged[field] = replacement + sample.write_text(json.dumps(forged)) + with patch.object(run_http, "HERE", str(here)), patch.object( + run_http, "post" + ) as post: + with self.assertRaisesRegex(ValueError, "qualified sample disagrees"): + run_http.main(args) + post.assert_not_called() + self.assertEqual(partial.read_bytes(), partial_bytes) + self.assertFalse(final.exists()) + # Previously a valid Noul response from another call could be + # substituted under the correct request hash and accepted. + forged = copy.deepcopy(evidence) + forged["response"]["answers"]["impact"]["noul"] = 0.9 + sample.write_text(json.dumps(forged)) + with patch.object(run_http, "HERE", str(here)), patch.object( + run_http, "post" + ) as post: + with self.assertRaisesRegex(ValueError, "raw response disagrees with first scored row"): + run_http.main(args) + post.assert_not_called() + self.assertEqual(partial.read_bytes(), partial_bytes) + + # An otherwise valid, rewritten result row must not describe a + # different original first-response sample. + sample.write_bytes(sample_bytes) + lines = partial.read_text().splitlines() + forged_row = json.loads(lines[1]) + forged_row["ms"] = 5.0 + lines[1] = json.dumps(forged_row) + partial.write_text("\n".join(lines) + "\n") + with patch.object(run_http, "HERE", str(here)), patch.object( + run_http, "post" + ) as post: + with self.assertRaisesRegex(ValueError, "sample response or row disagrees"): + run_http.main(args) + post.assert_not_called() + partial.write_bytes(partial_bytes) + + # A later case from the same frozen arm is not the first response. + forged = copy.deepcopy(evidence) + second = self.dataset[3] + second_request = {"state": run_http.STATES["B_noul_text"](second), + "questions": self.questions} + forged["case_id"] = second["id"] + forged["request"] = second_request + forged["request_sha256"] = hashlib.sha256( + run_http.request_bytes(second_request) + ).hexdigest() + sample.write_text(json.dumps(forged)) + with patch.object(run_http, "HERE", str(here)), patch.object( + run_http, "post" + ) as post: + with self.assertRaisesRegex(ValueError, "not the first held-out"): + run_http.main(args) + post.assert_not_called() + self.assertEqual(partial.read_bytes(), partial_bytes) + + # Refuse a deleted sample after a completed first row. + sample.unlink() + with patch.object(run_http, "HERE", str(here)), patch.object( + run_http, "post" + ) as post: + with self.assertRaisesRegex(ValueError, "sample is missing"): + run_http.main(args) + post.assert_not_called() + self.assertEqual(partial.read_bytes(), partial_bytes) + + # A sample published without a completed row is also insufficient + # to claim the first held-out call on resume. + sample.write_bytes(sample_bytes) + partial.write_text(partial_bytes.decode().splitlines()[0] + "\n") + with patch.object(run_http, "HERE", str(here)), patch.object( + run_http, "post" + ) as post: + with self.assertRaisesRegex(ValueError, "no completed first-response row"): + run_http.main(args) + post.assert_not_called() + partial.write_bytes(partial_bytes) + sample.write_bytes(sample_bytes) + with patch.object(run_http, "HERE", str(here)), patch.object( + run_http, "post", side_effect=lambda url, body, **kw: ( + {"model": "self-reported-test-model", + "answers": {key: {"type": "noul", "noul": 0.1} + for key in body["questions"]}}, + 3.0, + ) + ) as post: + run_http.main(args) + self.assertEqual(post.call_count, 1) + self.assertTrue(final.exists()) + self.assertFalse(partial.exists()) + self.assertEqual(sample.read_bytes(), sample_bytes) + self.assertEqual(len(json.loads(final.read_text())["rows"]), 2) + + # A new tag cannot accidentally publish a stale raw response as + # evidence of a completely separate experiment. + new_args = args.copy() + new_args[new_args.index("sample-provenance")] = "other-experiment" + with patch.object(run_http, "HERE", str(here)), patch.object( + run_http, "post" + ) as post: + with self.assertRaisesRegex(ValueError, "qualified sample already exists"): + run_http.main(new_args) + post.assert_not_called() + self.assertFalse( + (here / "results" / "other-experiment_B_noul_text.json.partial").exists() + ) + self.assertEqual(sample.read_bytes(), sample_bytes) + + def test_qualified_resume_bounds_sparse_partial_and_individual_rows(self): + with tempfile.TemporaryDirectory() as temp: + here = Path(temp) + (here / "dataset.json").write_bytes(self.dataset_bytes) + (here / "manifest.json").write_bytes(self.manifest_bytes) + (here / "server.json").write_text(json.dumps(self.server_config)) + args = [ + "--url", self.endpoint, "--tag", "bounded-resume", + "--server-config", str(here / "server.json"), + "--qualification-manifest", str(here / "manifest.json"), + "B_noul_text", + ] + partial = here / "results" / "bounded-resume_B_noul_text.json.partial" + completed = here / "results" / "bounded-resume_B_noul_text.json" + calls = 0 + + def interrupt(url, body, **kwargs): + nonlocal calls + calls += 1 + if calls == 2: + raise OSError("interrupted after first held-out result") + return { + "model": "self-reported-test-model", + "answers": {key: {"type": "noul", "noul": 0.2} + for key in body["questions"]}, + }, 4.0 + + with patch.object(run_http, "HERE", str(here)), patch.object( + run_http, "post", side_effect=interrupt, + ): + with self.assertRaisesRegex(OSError, "interrupted after first held-out"): + run_http.main(args) + self.assertEqual(calls, 2) + original = partial.read_bytes() + self.assertEqual(len(original.splitlines()), 2) + self.assertFalse(completed.exists()) + + header = json.loads(original.splitlines()[0]) + header_limit = len(json.dumps(header, allow_nan=False)) + 4096 + case_ids = ["holdout-positive", "holdout-negative"] + row_limit = 16 * ( + run_http.MAX_HTTP_RESPONSE_BYTES + + len(json.dumps(self.questions, allow_nan=False)) + + max(map(len, case_ids)) + ) + 4096 + total_limit = header_limit + 1 + len(case_ids) * (row_limit + 1) + + # A sparse oversized file fails at fstat rather than being read. + with partial.open("r+b") as output: + output.truncate(total_limit + 1) + with patch.object(run_http, "HERE", str(here)), patch.object( + run_http, "post", + ) as post: + with self.assertRaisesRegex(ValueError, "partial output exceeds bounded evidence"): + run_http.main(args) + post.assert_not_called() + self.assertEqual(partial.stat().st_size, total_limit + 1) + self.assertFalse(completed.exists()) + + # A single unterminated row exceeds its cap even below total cap. + partial.write_bytes(original) + with partial.open("r+b") as output: + output.truncate(len(original) + row_limit + 1) + self.assertLessEqual(partial.stat().st_size, total_limit) + with patch.object(run_http, "HERE", str(here)), patch.object( + run_http, "post", + ) as post: + with self.assertRaisesRegex(ValueError, "partial row at line 3 exceeds bounded"): + run_http.main(args) + post.assert_not_called() + self.assertFalse(completed.exists()) + + # Restoring the real prefix resumes exactly one remaining case. + partial.write_bytes(original) + with patch.object(run_http, "HERE", str(here)), patch.object( + run_http, "post", side_effect=lambda url, body, **kwargs: ( + {"model": "self-reported-test-model", + "answers": {key: {"type": "noul", "noul": 0.1} + for key in body["questions"]}}, + 3.0, + ), + ) as post: + run_http.main(args) + self.assertEqual(post.call_count, 1) + self.assertFalse(partial.exists()) + self.assertEqual(len(json.loads(completed.read_text())["rows"]), 2) + + def test_qualified_sample_refuses_oversize_and_growth_before_parsing(self): + with tempfile.TemporaryDirectory() as temp: + sample = Path(temp) / "first.json" + request = {"state": "bounded Scout window", "questions": self.questions} + response = { + "model": "reported-model", + "answers": {key: {"type": "noul", "noul": 0.2} + for key in self.questions}, + } + request_sha = hashlib.sha256(run_http.request_bytes(request)).hexdigest() + requests = {"holdout-positive": (request, request_sha)} + evidence = { + "case_id": "holdout-positive", + "experiment_sha256": "f" * 64, + "request_sha256": request_sha, + "row_sha256": "a" * 64, + "request": request, + "response": response, + } + valid_bytes = json.dumps(evidence).encode("utf-8") + sample.write_bytes(valid_bytes) + self.assertEqual( + run_http.validate_qualified_sample(str(sample), "f" * 64, requests), + (run_http.digest(response), "a" * 64, hashlib.sha256(valid_bytes).hexdigest()), + ) + sample_limit = 16 * ( + len(run_http.request_bytes(request)) + run_http.MAX_HTTP_RESPONSE_BYTES + ) + 4096 + + # An oversized sparse file must be refused from fstat alone, + # without materializing its huge attacker-controlled contents. + with sample.open("r+b") as output: + output.truncate(sample_limit + 1) + with patch.object(run_http.os, "fdopen") as fdopen: + with self.assertRaisesRegex(ValueError, "exceeds bounded evidence"): + run_http.validate_qualified_sample(str(sample), "f" * 64, requests) + fdopen.assert_not_called() + + # A writer may grow the same inode *after* the initial size + # check. The bounded read must refuse that race too. + sample.write_bytes(valid_bytes) + actual_fstat = os.fstat + changes = [] + + def grow_after_fstat(descriptor): + original_stat = actual_fstat(descriptor) + changes.append(original_stat.st_size) + with sample.open("r+b") as output: + output.truncate(sample_limit + 1) + return original_stat + + with patch.object(run_http.os, "fstat", side_effect=grow_after_fstat): + with self.assertRaisesRegex(ValueError, "exceeds bounded evidence"): + run_http.validate_qualified_sample(str(sample), "f" * 64, requests) + self.assertEqual(changes, [len(valid_bytes)]) + self.assertEqual(sample.stat().st_size, sample_limit + 1) + + @unittest.skipUnless( + os.name == "posix" and hasattr(os, "O_NOFOLLOW"), + "inode-safe sample admission requires POSIX O_NOFOLLOW", + ) + def test_qualified_sample_path_swaps_refuse_even_with_identical_evidence(self): + # The other file contains valid, byte-identical first-response evidence. + # Merely checking JSON digests would accept it after path substitution. + # Force each race at the syscall boundary, not through an idle symlink. + with tempfile.TemporaryDirectory() as temp: + sample = Path(temp) / "first.json" + other = Path(temp) / "other.json" + request = {"state": "original Scout window", "questions": self.questions} + response = { + "model": "reported-model", + "answers": {key: {"type": "noul", "noul": 0.2} + for key in self.questions}, + } + experiment_sha = "f" * 64 + requests = { + "holdout-positive": ( + request, hashlib.sha256(run_http.request_bytes(request)).hexdigest() + ), + } + evidence = { + "case_id": "holdout-positive", + "experiment_sha256": experiment_sha, + "request_sha256": requests["holdout-positive"][1], + "row_sha256": "a" * 64, + "request": request, + "response": response, + } + original_bytes = json.dumps(evidence).encode("utf-8") + other.write_bytes(original_bytes) + real_open, real_fstat = os.open, os.fstat + + def redirect(): + sample.unlink() + sample.symlink_to(other.name) + + for stage in ("before_open", "after_open"): + with self.subTest(stage=stage): + sample.write_bytes(original_bytes) + self.assertEqual( + run_http.validate_qualified_sample( + str(sample), experiment_sha, requests, + ), + (run_http.digest(response), evidence["row_sha256"], hashlib.sha256(original_bytes).hexdigest()), + ) + swaps = [] + + def swap_before_open(path, flags, *args, **kwargs): + if path == str(sample): + swaps.append(stage) + redirect() + return real_open(path, flags, *args, **kwargs) + + def swap_after_open(descriptor): + opened = real_fstat(descriptor) + swaps.append(stage) + redirect() + return opened + + replacement = ( + patch.object(run_http.os, "open", side_effect=swap_before_open) + if stage == "before_open" else + patch.object(run_http.os, "fstat", side_effect=swap_after_open) + ) + with replacement: + with self.assertRaisesRegex( + ValueError, "qualified sample path .*regular file|replaced or redirected", + ): + run_http.validate_qualified_sample( + str(sample), experiment_sha, requests, + ) + self.assertEqual(swaps, [stage], "the sample inode race must actually occur") + self.assertTrue(sample.is_symlink()) + self.assertEqual(other.read_bytes(), original_bytes) + sample.unlink() + + def test_qualified_sample_replaced_during_resume_refuses_completed_result(self): + with tempfile.TemporaryDirectory() as temp: + here = Path(temp) + (here / "dataset.json").write_bytes(self.dataset_bytes) + (here / "manifest.json").write_bytes(self.manifest_bytes) + (here / "server.json").write_text(json.dumps(self.server_config)) + sample = here / "first-response.json" + args = [ + "--url", self.endpoint, "--tag", "sample-race", + "--server-config", str(here / "server.json"), + "--qualification-manifest", str(here / "manifest.json"), + "--sample-out", str(sample), "B_noul_text", + ] + partial = here / "results" / "sample-race_B_noul_text.json.partial" + completed = here / "results" / "sample-race_B_noul_text.json" + calls = 0 + + def interrupt(url, body, **kwargs): + nonlocal calls + calls += 1 + if calls == 2: + raise OSError("interrupted after first sample") + return { + "model": "self-reported-test-model", + "answers": {key: {"type": "noul", "noul": 0.2} + for key in body["questions"]}, + }, 4.0 + + with patch.object(run_http, "HERE", str(here)), patch.object( + run_http, "post", side_effect=interrupt, + ): + with self.assertRaisesRegex(OSError, "interrupted after first sample"): + run_http.main(args) + self.assertEqual(calls, 2) + original_sample = sample.read_bytes() + self.assertEqual(len(partial.read_text().splitlines()), 2) + self.assertFalse(completed.exists()) + + def replace_after_admission(url, body, **kwargs): + replacement = json.loads(original_sample) + # Keep all semantic fields and row commitments unchanged, but + # swap in a differently encoded sample after admission. + # The older row/response-only check would publish this run. + alternate = here / "replacement.json" + alternate.write_text(json.dumps(replacement, sort_keys=True)) + alternate.replace(sample) + return { + "model": "self-reported-test-model", + "answers": {key: {"type": "noul", "noul": 0.1} + for key in body["questions"]}, + }, 3.0 + + # The first sample passed admission before the second model call. + # Replacing it during that call must not certify a completed run. + with patch.object(run_http, "HERE", str(here)), patch.object( + run_http, "post", side_effect=replace_after_admission, + ) as post: + with self.assertRaisesRegex(ValueError, "sample changed during inference"): + run_http.main(args) + self.assertEqual(post.call_count, 1) + self.assertFalse(completed.exists()) + self.assertEqual(len(partial.read_text().splitlines()), 3) + self.assertNotEqual(sample.read_bytes(), original_sample) + + # Restore exactly the recorded first response. A complete partial + # can then publish without replaying or inventing any model call. + sample.write_bytes(original_sample) + with patch.object(run_http, "HERE", str(here)), patch.object( + run_http, "post", + ) as post: + run_http.main(args) + post.assert_not_called() + self.assertTrue(completed.exists()) + self.assertFalse(partial.exists()) + self.assertEqual(len(json.loads(completed.read_text())["rows"]), 2) + + def test_runner_refuses_group_leakage_without_network_or_partial_output(self): + with tempfile.TemporaryDirectory() as temp: + here = Path(temp) + (here / "dataset.json").write_bytes(self.dataset_bytes) + changed = copy.deepcopy(self.manifest) + changed["annotations"][2]["project_id"] = "project-a" + (here / "manifest.json").write_text(json.dumps(changed)) + (here / "server.json").write_text(json.dumps(self.server_config)) + args = [ + "--url", "http://127.0.0.1:8009/v1/systemone", + "--tag", "refused", + "--server-config", str(here / "server.json"), + "--qualification-manifest", str(here / "manifest.json"), + "B_noul_text", + ] + with patch.object(run_http, "HERE", str(here)), patch.object(run_http, "post") as post: + with self.assertRaisesRegex(ValueError, "leaks project/session"): + run_http.main(args) + post.assert_not_called() + self.assertFalse((here / "results").exists()) + + + def test_frozen_manifest_cannot_reorder_a_dataset(self): + changed = copy.deepcopy(self.manifest) + changed["annotations"][0], changed["annotations"][1] = ( + changed["annotations"][1], changed["annotations"][0] + ) + with self.assertRaisesRegex(ValueError, "case ordering"): + self.valid(changed) + + def test_changed_manifest_cannot_resume_or_spend_inference_tokens(self): + with tempfile.TemporaryDirectory() as temp: + here = Path(temp) + (here / "dataset.json").write_bytes(self.dataset_bytes) + manifest_path = here / "manifest.json" + manifest_path.write_bytes(self.manifest_bytes) + (here / "server.json").write_text(json.dumps(self.server_config)) + args = [ + "--url", "http://127.0.0.1:8009/v1/systemone", + "--tag", "interrupted", + "--server-config", str(here / "server.json"), + "--qualification-manifest", str(manifest_path), + "B_noul_text", + ] + answer = {"model": "self-reported-test-model", "answers": { + key: {"type": "noul", "noul": 0.5} for key in self.questions + }} + with patch.object(run_http, "HERE", str(here)): + with patch.object(run_http, "post", side_effect=[ + (answer, 11.0), RuntimeError("interrupted"), + ]) as initial: + with self.assertRaisesRegex(RuntimeError, "interrupted"): + run_http.main(args) + self.assertEqual(initial.call_count, 2) + partial = here / "results" / "interrupted_B_noul_text.json.partial" + frozen_bytes = partial.read_bytes() + self.assertEqual(len(frozen_bytes.splitlines()), 2, + "first accepted prediction must be durably resumable") + header_line, row_line = frozen_bytes.splitlines() + falsified = json.loads(row_line) + falsified["request_size_bytes"] += 1 + corrupted_bytes = header_line + b"\n" + json.dumps(falsified).encode("utf-8") + b"\n" + partial.write_bytes(corrupted_bytes) + with patch.object(run_http, "post") as blocked: + with self.assertRaisesRegex(ValueError, "mismatched request byte count"): + run_http.main(args) + blocked.assert_not_called() + self.assertEqual(partial.read_bytes(), corrupted_bytes) + partial.write_bytes(frozen_bytes) + changed = copy.deepcopy(self.manifest) + changed["error_tolerances"]["max_p95_ms"] = 21.0 + manifest_path.write_text(json.dumps(changed)) + with patch.object(run_http, "post") as no_inference: + with self.assertRaisesRegex(ValueError, "another or unrecorded experiment"): + run_http.main(args) + no_inference.assert_not_called() + self.assertEqual(partial.read_bytes(), frozen_bytes) + manifest_path.write_bytes(self.manifest_bytes) + with patch.object(run_http, "post", return_value=(answer, 12.0)) as remaining: + run_http.main(args) + self.assertEqual(remaining.call_count, 1) + completed = here / "results" / "interrupted_B_noul_text.json" + self.assertEqual(len(json.loads(completed.read_text())["rows"]), 2) + + + def test_qualified_cli_is_separate_and_historical_cli_refuses_heldout(self): + with tempfile.TemporaryDirectory() as temp: + here = Path(temp) + for filename in ("score.py", "qualification.py", "common.py", "run_http.py"): + shutil.copyfile(Path(score.__file__).with_name(filename), here / filename) + (here / "results").mkdir() + result_path = here / "results" / "qualified.json" + result_path.write_text(json.dumps(self.signed_result())) + manifest_path = here / "qualification.json" + manifest_path.write_bytes(self.manifest_bytes) + (here / "dataset.json").write_bytes(self.dataset_bytes) + summary = here / "summary.json" + summary.write_bytes(b"existing historical evidence\\n") + + default = subprocess.run( + [sys.executable, str(here / "score.py")], + capture_output=True, text=True, check=False, + ) + self.assertNotEqual(default.returncode, 0) + self.assertIn("refuse in-sample scoring", default.stderr) + self.assertEqual(summary.read_bytes(), b"existing historical evidence\\n") + + qualified = subprocess.run( + [sys.executable, str(here / "score.py"), "--qualified", + str(manifest_path), str(here / "dataset.json"), str(result_path)], + capture_output=True, text=True, check=False, + ) + self.assertEqual(qualified.returncode, 0, qualified.stderr) + report_path = here / "qualification_summary.json" + report = json.loads(report_path.read_text()) + self.assertTrue(report["qualified_against_frozen_tolerances"]) + self.assertEqual(report["latency"]["p95_ms"], 20) + self.assertEqual(summary.read_bytes(), b"existing historical evidence\\n") + + corrupted = self.signed_result() + corrupted["rows"][0]["labels"] = [] + result_path.write_text(json.dumps(corrupted)) + previous_report = report_path.read_bytes() + rejected = subprocess.run( + [sys.executable, str(here / "score.py"), "--qualified", + str(manifest_path), str(here / "dataset.json"), str(result_path)], + capture_output=True, text=True, check=False, + ) + self.assertNotEqual(rejected.returncode, 0) + self.assertIn("reviewed labels", rejected.stderr) + self.assertEqual(report_path.read_bytes(), previous_report) + + def test_qualified_requires_original_cases_and_reproducible_requests(self): + result = self.signed_result() + altered_cases = copy.deepcopy(self.dataset) + altered_cases[2]["current_prompt"] = "A different prompt" + with self.assertRaisesRegex(ValueError, "dataset digest differs"): + qualification.qualified_report( + result, self.manifest, self.manifest_sha, self.questions, + json.dumps(altered_cases).encode("utf-8"), + ) + # The stored hashes can be re-computed consistently around a forged + # request fingerprint; reconstruction from real cases must catch it. + altered = copy.deepcopy(result) + altered["rows"][0]["request_sha256"] = "f" * 64 + altered["experiment"]["requests_sha256"] = score.evidence_digest([ + [row["id"], row["request_sha256"]] for row in altered["rows"] + ]) + # Keep the declaration and the result internally consistent so this + # exercises reconstruction against the original requests. + manifest = copy.deepcopy(self.manifest) + manifest["holdout_requests_sha256"] = altered["experiment"]["requests_sha256"] + manifest_sha = qualification.sha256_bytes(json.dumps(manifest).encode("utf-8")) + altered["experiment"]["qualification_manifest_sha256"] = manifest_sha + altered["experiment_sha256"] = score.evidence_digest(altered["experiment"]) + with self.assertRaisesRegex(ValueError, "request fingerprint disagrees"): + qualification.qualified_report( + altered, manifest, manifest_sha, self.questions, self.dataset_bytes, + ) + + def test_runner_can_consume_explicit_frozen_input_file(self): + with tempfile.TemporaryDirectory() as temp: + here = Path(temp) + dataset_path = here / "external-cases.json" + dataset_path.write_bytes(self.dataset_bytes) + (here / "manifest.json").write_bytes(self.manifest_bytes) + (here / "server.json").write_text(json.dumps(self.server_config)) + answer = {"model": "self-reported-test-model", "answers": { + key: {"type": "noul", "noul": 0.5} for key in self.questions + }} + with patch.object(run_http, "HERE", str(here)): + with patch.object(run_http, "post", return_value=(answer, 12.0)) as post: + run_http.main([ + "--url", "http://127.0.0.1:8009/v1/systemone", + "--tag", "external", + "--server-config", str(here / "server.json"), + "--dataset", str(dataset_path), + "--qualification-manifest", str(here / "manifest.json"), + "B_noul_text", + ]) + self.assertEqual(post.call_count, 2) + result = json.loads( + (here / "results" / "external_B_noul_text.json").read_text() + ) + qualification.qualified_report( + result, self.manifest, self.manifest_sha, + self.questions, self.dataset_bytes, + ) + + + + def test_qualified_timeout_is_enforced_and_scored_against_frozen_arm(self): + manifest = copy.deepcopy(self.manifest) + manifest["arm"]["timeout_ms"] = 250 + with tempfile.TemporaryDirectory() as temp: + here = Path(temp) + (here / "dataset.json").write_bytes(self.dataset_bytes) + (here / "manifest.json").write_text(json.dumps(manifest)) + (here / "server.json").write_text(json.dumps(self.server_config)) + args = [ + "--url", self.endpoint, "--tag", "deadline", + "--server-config", str(here / "server.json"), + "--qualification-manifest", str(here / "manifest.json"), + "--timeout-ms", "250", "B_noul_text", + ] + response = { + "model": "self-reported-test-model", + "answers": {key: {"noul": 0.9} for key in self.questions}, + "usage": {"input_tokens": 32, "state_tokens_dropped": 0}, + } + with patch.object(run_http, "HERE", str(here)): + with patch.object(run_http, "post", return_value=(response, 5.0)) as post: + run_http.main(args) + self.assertEqual(post.call_count, 2) + self.assertTrue(all(call.kwargs == {"timeout": 0.25} + for call in post.call_args_list)) + result = json.loads( + (here / "results" / "deadline_B_noul_text.json").read_text() + ) + self.assertEqual(result["experiment"]["timeout_ms"], 250) + # Rehashing result metadata cannot change the frozen deadline. + result["experiment"]["timeout_ms"] = 1_800_000 + result["experiment_sha256"] = score.evidence_digest(result["experiment"]) + with self.assertRaisesRegex(ValueError, "HTTP arm differs"): + qualification.qualified_report( + result, manifest, + qualification.sha256_bytes((here / "manifest.json").read_bytes()), + self.questions, self.dataset_bytes, + ) + + def test_frozen_http_arm_is_required_and_well_formed(self): + mutations = ( + (lambda m: m.pop("arm"), "frozen HTTP arm"), + (lambda m: m["arm"].pop("config"), "frozen HTTP arm"), + (lambda m: m["arm"].update(config=""), "renderer config"), + (lambda m: m["arm"].update(requested_model=""), "requested model"), + (lambda m: m["arm"].update(endpoint_sha256="x" * 64), "endpoint_sha256"), + (lambda m: m["arm"].update(server_config_sha256=None), "server_config_sha256"), + (lambda m: m["arm"].pop("timeout_ms"), "frozen HTTP arm"), + (lambda m: m["arm"].update(timeout_ms=0), "timeout_ms"), + (lambda m: m["arm"].update(timeout_ms=1_800_001), "timeout_ms"), + (lambda m: m["arm"].update(timeout_ms=True), "timeout_ms"), + ) + for mutate, error in mutations: + with self.subTest(error=error): + manifest = copy.deepcopy(self.manifest) + mutate(manifest) + with self.assertRaisesRegex(ValueError, error): + self.valid(manifest) + + def test_frozen_http_arm_refuses_post_hoc_reselection_before_inference(self): + with tempfile.TemporaryDirectory() as temp: + here = Path(temp) + (here / "dataset.json").write_bytes(self.dataset_bytes) + (here / "manifest.json").write_bytes(self.manifest_bytes) + server = here / "server.json" + normal = [ + "--url", self.endpoint, "--tag", "locked-arm", + "--server-config", str(server), + "--qualification-manifest", str(here / "manifest.json"), + "B_noul_text", + ] + trials = ( + ("renderer", lambda a: a.__setitem__(-1, "A_noul_prompt")), + ("multiple-renderers", lambda a: a.append("A_noul_prompt")), + ("model", lambda a: a.__setitem__(slice(-1, -1), ["--send-model", "unfrozen-model"])), + ("endpoint", lambda a: a.__setitem__(1, "http://127.0.0.1:8010/v1/systemone")), + ("timeout", lambda a: a.__setitem__(slice(-1, -1), ["--timeout-ms", "250"])), + ) + with patch.object(run_http, "HERE", str(here)): + for label, mutation in trials: + with self.subTest(label=label): + server.write_text(json.dumps(self.server_config)) + args = normal.copy() + mutation(args) + with patch.object(run_http, "post") as blocked: + with self.assertRaisesRegex(ValueError, "HTTP arm differs"): + run_http.main(args) + blocked.assert_not_called() + self.assertFalse((here / "results").exists()) + drifted = copy.deepcopy(self.server_config) + drifted["checkpoint"] = "another-checkpoint" + server.write_text(json.dumps(drifted)) + with patch.object(run_http, "post") as blocked: + with self.assertRaisesRegex(ValueError, "HTTP arm differs"): + run_http.main(normal) + blocked.assert_not_called() + self.assertFalse((here / "results").exists()) + + def test_scorer_refuses_rehashed_unfrozen_http_arm(self): + mutations = ( + lambda e: e.update(endpoint_sha256="f" * 64), + lambda e: e["server_config"].update(checkpoint="different-pinned-model"), + ) + for mutate in mutations: + with self.subTest(mutate=mutate): + result = self.signed_result() + mutate(result["experiment"]) + result["experiment_sha256"] = score.evidence_digest(result["experiment"]) + with self.assertRaisesRegex(ValueError, "HTTP arm differs"): + qualification.qualified_report( + result, self.manifest, self.manifest_sha, + self.questions, self.dataset_bytes, + ) + + + def test_frozen_token_budget_must_be_positive_integer(self): + for limit in (None, 0, -1, True, 1.5, "8192"): + with self.subTest(limit=limit): + manifest = copy.deepcopy(self.manifest) + manifest["max_input_tokens"] = limit + with self.assertRaisesRegex(ValueError, "positive integer max_input_tokens"): + self.valid(manifest) + + def test_missing_zero_or_overlimit_input_tokens_cannot_qualify(self): + for reported in (None, 0, 65): + with self.subTest(reported=reported): + result = self.signed_result() + result["rows"][0]["input_tokens"] = reported + report = qualification.qualified_report( + result, self.manifest, self.manifest_sha, + self.questions, self.dataset_bytes, + ) + self.assertFalse(report["qualified_against_frozen_tolerances"]) + self.assertFalse(report["checks"]["reported_input_token_budget"]) + evidence = report["input_token_evidence"] + self.assertEqual(evidence["measured_cases"], 1 if reported in (None, 0) else 2) + self.assertEqual(evidence["missing_or_zero_cases"], 1 if reported in (None, 0) else 0) + self.assertEqual(evidence["cases_over_frozen_limit"], int(reported == 65)) + + def test_rehashed_tighter_token_budget_does_not_certify_old_measurements(self): + restricted = copy.deepcopy(self.manifest) + restricted["max_input_tokens"] = 63 + restricted_sha = qualification.sha256_json(restricted) + result = self.signed_result() + result["experiment"]["qualification_manifest_sha256"] = restricted_sha + result["experiment_sha256"] = score.evidence_digest(result["experiment"]) + report = qualification.qualified_report( + result, restricted, restricted_sha, self.questions, self.dataset_bytes, + ) + self.assertFalse(report["qualified_against_frozen_tolerances"]) + self.assertFalse(report["checks"]["reported_input_token_budget"]) + self.assertEqual(report["input_token_evidence"]["cases_over_frozen_limit"], 2) + + def test_frozen_request_byte_budget_must_be_positive_integer(self): + for budget in (None, 0, -1, True, 0.5, "4096"): + with self.subTest(budget=budget): + changed = copy.deepcopy(self.manifest) + changed["max_request_bytes"] = budget + with self.assertRaisesRegex(ValueError, "positive integer max_request_bytes"): + self.valid(changed) + + def test_frozen_request_byte_budget_refuses_before_network(self): + with tempfile.TemporaryDirectory() as temp: + here = Path(temp) + (here / "dataset.json").write_bytes(self.dataset_bytes) + (here / "server.json").write_text(json.dumps(self.server_config)) + changed = copy.deepcopy(self.manifest) + changed["max_request_bytes"] = 1 + (here / "manifest.json").write_text(json.dumps(changed)) + args = [ + "--url", self.endpoint, "--tag", "oversized", + "--server-config", str(here / "server.json"), + "--qualification-manifest", str(here / "manifest.json"), + "B_noul_text", + ] + with patch.object(run_http, "HERE", str(here)): + with patch.object(run_http, "post") as post: + with self.assertRaisesRegex(ValueError, "exceeds frozen request byte limit"): + run_http.main(args) + post.assert_not_called() + self.assertFalse((here / "results").exists()) + + def test_qualification_reconstructs_measured_request_sizes(self): + original = self.signed_result() + report = qualification.qualified_report( + original, self.manifest, self.manifest_sha, + self.questions, self.dataset_bytes, + ) + self.assertGreater(report["request_size_bytes"]["min"], 0) + self.assertEqual( + report["request_size_bytes"]["max"], + max(row["request_size_bytes"] for row in original["rows"]), + ) + for value in (None, True, original["rows"][0]["request_size_bytes"] + 1): + with self.subTest(forged_size=value): + forged = copy.deepcopy(original) + forged["rows"][0]["request_size_bytes"] = value + with self.assertRaisesRegex(ValueError, "request byte count disagrees"): + qualification.qualified_report( + forged, self.manifest, self.manifest_sha, + self.questions, self.dataset_bytes, + ) + tighter = copy.deepcopy(self.manifest) + tighter["max_request_bytes"] = 1 + tighter_sha = qualification.sha256_json(tighter) + forged = copy.deepcopy(original) + forged["experiment"]["qualification_manifest_sha256"] = tighter_sha + forged["experiment_sha256"] = score.evidence_digest(forged["experiment"]) + with self.assertRaisesRegex(ValueError, "exceeds frozen request byte limit"): + qualification.qualified_report( + forged, tighter, tighter_sha, self.questions, self.dataset_bytes, + ) + + + def test_frozen_rendered_requests_refuse_renderer_drift_before_inference(self): + # The renderer's name, source cases, and questions may all remain + # unchanged while the actual outbound model prompt changes. + with tempfile.TemporaryDirectory() as temp: + here = Path(temp) + (here / "dataset.json").write_bytes(self.dataset_bytes) + (here / "manifest.json").write_bytes(self.manifest_bytes) + (here / "server.json").write_text(json.dumps(self.server_config)) + with patch.object(run_http, "HERE", str(here)), patch.dict( + run_http.STATES, {"B_noul_text": lambda case: "changed renderer output"}, + ), patch.object(run_http, "post") as inference: + with self.assertRaisesRegex(ValueError, "rendered held-out requests differ"): + run_http.main([ + "--url", self.endpoint, "--tag", "renderer-drift", + "--server-config", str(here / "server.json"), + "--qualification-manifest", str(here / "manifest.json"), + "B_noul_text", + ]) + inference.assert_not_called() + self.assertFalse((here / "results").exists()) + + def test_frozen_rendered_request_digest_refuses_rehashed_result(self): + for bad_digest in (None, 42, "f" * 64): + with self.subTest(frozen_digest=bad_digest): + changed = copy.deepcopy(self.manifest) + changed["holdout_requests_sha256"] = bad_digest + if bad_digest is None or bad_digest == 42: + with self.assertRaisesRegex(ValueError, "frozen held-out request digest"): + self.valid(changed) + continue + changed_sha = qualification.sha256_json(changed) + forged = self.signed_result() + forged["experiment"]["qualification_manifest_sha256"] = changed_sha + forged["experiment_sha256"] = score.evidence_digest(forged["experiment"]) + # Every result/experiment fingerprint still agrees internally. + # The independently frozen request body digest must also match. + with self.assertRaisesRegex(ValueError, "rendered held-out requests differ"): + qualification.qualified_report( + forged, changed, changed_sha, self.questions, self.dataset_bytes, + ) + + def test_frozen_question_order_refuses_a_second_holdout_arm_before_inference(self): + # Sorted-key JSON digests collide across question reordering although + # the runner sends dict entries in insertion order. That let the same + # frozen manifest authorize multiple held-out request sequences. + reordered = dict(reversed(list(self.questions.items()))) + self.assertEqual(set(reordered), set(self.questions)) + self.assertEqual( + qualification.sha256_json(reordered), + qualification.sha256_json(self.questions), + ) + self.assertNotEqual( + qualification.sha256_ordered_questions(reordered), + qualification.sha256_ordered_questions(self.questions), + ) + # Reordering type/instructions fields within one question does not + # change the contract: only actual HTTP question sequence is frozen. + equivalent = { + key: {"instructions": item["instructions"], "type": item["type"]} + for key, item in self.questions.items() + } + self.assertEqual( + qualification.sha256_ordered_questions(equivalent), + qualification.sha256_ordered_questions(self.questions), + ) + with self.assertRaisesRegex(ValueError, "question vocabulary or instruction"): + qualification.validate_manifest( + self.manifest, self.dataset_sha, reordered, self.dataset, + ) + + with tempfile.TemporaryDirectory() as temp: + here = Path(temp) + (here / "dataset.json").write_bytes(self.dataset_bytes) + (here / "manifest.json").write_bytes(self.manifest_bytes) + (here / "questions.json").write_text(json.dumps(reordered)) + (here / "server.json").write_text(json.dumps(self.server_config)) + with patch.object(run_http, "HERE", str(here)): + with patch.object(run_http, "post") as inference: + with self.assertRaisesRegex(ValueError, "question vocabulary or instruction"): + run_http.main([ + "--url", self.endpoint, "--tag", "reordered-holdout", + "--dataset", str(here / "dataset.json"), + "--server-config", str(here / "server.json"), + "--qualification-manifest", str(here / "manifest.json"), + "--questions-file", str(here / "questions.json"), + "B_noul_text", + ]) + inference.assert_not_called() + self.assertFalse( + (here / "results" / "reordered-holdout_B_noul_text.json.partial").exists() + ) + + # An original run must not be requalified using the altered order, + # even though the question set and old sorted-object digest agree. + with self.assertRaisesRegex(ValueError, "question vocabulary or instruction"): + qualification.qualified_report( + self.signed_result(), self.manifest, self.manifest_sha, + reordered, self.dataset_bytes, + ) + + def test_frozen_ten_question_vocabulary_runs_and_scores_without_historical_categories(self): + # This fixture exercises a declared ten-question contract; it is not a + # canonical Scout export and never claims Kev inference or parity. + questions = { + key: value for key, value in self.questions.items() + if key != "refactor_edit" + } + questions["atomic_edit"] = { + "type": "noul", "instructions": "Is the user requesting a mechanical edit?" + } + questions["build_diagnostics"] = { + "type": "noul", "instructions": "Does the user need a build error fixed?" + } + self.assertEqual(len(questions), 10) + manifest = copy.deepcopy(self.manifest) + manifest["questions_sha256"] = qualification.sha256_ordered_questions(questions) + manifest["thresholds"] = {key: 0.6 for key in questions} + manifest["holdout_requests_sha256"] = self.holdout_request_digest(questions=questions) + manifest_sha = qualification.sha256_bytes(json.dumps(manifest).encode()) + with tempfile.TemporaryDirectory() as temp: + here = Path(temp) + (here / "dataset.json").write_bytes(self.dataset_bytes) + (here / "manifest.json").write_text(json.dumps(manifest)) + (here / "questions.json").write_text(json.dumps(questions)) + (here / "server.json").write_text(json.dumps(self.server_config)) + + def inference(_url, body, **kwargs): + self.assertEqual(set(body["questions"]), set(questions)) + probability = 0.9 if "What changes?" in body["state"] else 0.1 + return ({ + "model": "kev-declared", + "answers": { + key: {"noul": probability if key == "impact" else 0.1} + for key in body["questions"] + }, + "usage": {"input_tokens": 42, "state_tokens_dropped": 0}, + "latency_ms": 5.0, + }, 5.0) + + args = [ + "--url", self.endpoint, "--tag", "ten-questions", + "--server-config", str(here / "server.json"), + "--qualification-manifest", str(here / "manifest.json"), + "--questions-file", str(here / "questions.json"), + "B_noul_text", + ] + with patch.object(run_http, "HERE", str(here)): + with patch.object(run_http, "post", side_effect=inference) as post: + run_http.main(args) + self.assertEqual(post.call_count, 2) + output = here / "results" / "ten-questions_B_noul_text.json" + result = json.loads(output.read_text()) + report = qualification.qualified_report( + result, manifest, manifest_sha, questions, self.dataset_bytes, + ) + self.assertTrue(report["qualified_against_frozen_tolerances"]) + self.assertEqual(report["cases"], 2) + self.assertEqual(len(score.score_config( + result, fixed_t=manifest["thresholds"], cats=tuple(questions) + )["per_cat"]), 10) + self.assertEqual(set(report["thresholds"]), set(questions)) + self.assertNotIn("refactor_edit", report["thresholds"]) + # The default historical vocabulary must not certify ten questions. + with self.assertRaisesRegex(ValueError, "question vocabulary"): + qualification.qualified_report( + result, manifest, manifest_sha, + self.questions, self.dataset_bytes, + ) + altered = copy.deepcopy(questions) + altered["atomic_edit"]["instructions"] = "A changed instruction" + (here / "questions.json").write_text(json.dumps(altered)) + args[3] = "another-tag" + with patch.object(run_http, "post") as blocked: + with self.assertRaisesRegex(ValueError, "question vocabulary"): + run_http.main(args) + blocked.assert_not_called() + self.assertFalse((here / "results" / "another-tag_B_noul_text.json.partial").exists()) + + def test_qualified_question_files_refuse_non_noul_or_ambiguous_vocabulary(self): + with tempfile.TemporaryDirectory() as temp: + path = Path(temp) / "questions.json" + for body, expected in ( + ('{"q": {"type": "choice", "instructions": "Pick"}}', "Noul"), + ('{"q": {"type": "noul", "instructions": ""}}', "Noul"), + ('{"q": {"type": "noul", "instructions": "Good"}, ' + '"q": {"type": "noul", "instructions": "Good"}}', "duplicate JSON"), + ('{"q": {"type": "noul", "instructions": "Good", "extra": 1}}', "Noul"), + ): + with self.subTest(body=body): + path.write_text(body) + with self.assertRaisesRegex(ValueError, expected): + qualification.load_questions(path) + + def captured_scout_fixture(self): + # Rust serde_json::Map emits lexicographically sorted object keys in + # the Bazel runtime. The settings Vec uses a different question order. + records = [] + for case in self.dataset[2:]: + body = {"questions": self.questions, "state": "User: " + case["current_prompt"]} + records.append({ + "id": case["id"], + "body": json.dumps(body, ensure_ascii=False, indent=2, sort_keys=True), + }) + capture_bytes = json.dumps(records, ensure_ascii=False).encode("utf-8") + manifest = copy.deepcopy(self.manifest) + manifest["arm"]["config"] = qualification.CAPTURED_SCOUT_CONFIG + manifest["max_state_bytes"] = 4096 + # The HTTP wire does not contain the local Scout selection margins. + manifest["margin_permille"] = {key: None for key in self.questions} + manifest["captured_requests_sha256"] = qualification.sha256_bytes(capture_bytes) + manifest["holdout_requests_sha256"] = score.evidence_digest([ + [record["id"], hashlib.sha256(record["body"].encode("utf-8")).hexdigest()] + for record in records + ]) + return capture_bytes, manifest + + def test_captured_scout_rejects_calibration_false_pass_before_inference_and_scoring(self): + # Scout forwards raw Noul probabilities; the optional historical + # calibration could make 0.55 clear a 0.60 production threshold. + capture, manifest = self.captured_scout_fixture() + self.valid(manifest) + self.assertLess(0.55, manifest["thresholds"]["impact"]) + self.assertGreater(calibrated(0.55, 0.25), manifest["thresholds"]["impact"]) + forged = copy.deepcopy(manifest) + forged["noul_temperature"] = 0.25 + forged_sha = qualification.sha256_json(forged) + + # Build a coherently refrozen *captured* result rather than relying on + # an unrelated config mismatch to make the independent scorer fail. + result = self.signed_result() + records = qualification.loads_unambiguous_json(capture) + for row, record in zip(result["rows"], records): + raw = {key: (0.55 if key in row["labels"] else 0.1) + for key in self.questions} + wire = record["body"].encode("utf-8") + row["probs_raw"] = raw + row["probs"] = {key: calibrated(p, 0.25) for key, p in raw.items()} + row["request_sha256"] = qualification.sha256_bytes(wire) + row["request_size_bytes"] = len(wire) + result["config"] = "trial_" + qualification.CAPTURED_SCOUT_CONFIG + experiment = result["experiment"] + experiment["config"] = qualification.CAPTURED_SCOUT_CONFIG + experiment["noul_temperature"] = 0.25 + experiment["qualification_manifest_sha256"] = forged_sha + experiment["requests_sha256"] = score.evidence_digest([ + [row["id"], row["request_sha256"]] for row in result["rows"] + ]) + result["experiment_sha256"] = score.evidence_digest(experiment) + self.assertEqual(experiment["requests_sha256"], forged["holdout_requests_sha256"]) + self.assertEqual( + qualification.holdout_decision_metrics( + [{**row, "probs": row["probs_raw"]} for row in result["rows"]], + forged["thresholds"], tuple(self.questions), forged["margin_permille"], + )["model_only_selected_positive_case_recall"], 0, + ) + self.assertEqual( + qualification.holdout_decision_metrics( + result["rows"], forged["thresholds"], tuple(self.questions), + forged["margin_permille"], + )["model_only_selected_positive_case_recall"], 1, + ) + with self.assertRaisesRegex(ValueError, "captured Scout parity forbids"): + qualification.qualified_report( + result, forged, forged_sha, self.questions, self.dataset_bytes, capture, + ) + + with tempfile.TemporaryDirectory() as temp: + here = Path(temp) + (here / "dataset.json").write_bytes(self.dataset_bytes) + (here / "manifest.json").write_text(json.dumps(forged)) + (here / "captured.json").write_bytes(capture) + (here / "server.json").write_text(json.dumps(self.server_config)) + with patch.object(run_http, "HERE", str(here)), patch.object( + run_http, "post" + ) as post: + with self.assertRaisesRegex(ValueError, "captured Scout parity forbids"): + run_http.main([ + "--url", self.endpoint, "--tag", "calibration-parity", + "--server-config", str(here / "server.json"), + "--qualification-manifest", str(here / "manifest.json"), + "--captured-requests", str(here / "captured.json"), + "--noul-temperature", "0.25", + qualification.CAPTURED_SCOUT_CONFIG, + ]) + post.assert_not_called() + self.assertFalse((here / "results").exists()) + + def test_captured_scout_margins_are_frozen_before_network(self): + capture, manifest = self.captured_scout_fixture() + self.valid(manifest) + variants = [ + ("missing", lambda m: m.pop("margin_permille")), + ("incomplete", lambda m: m["margin_permille"].pop("impact")), + ("negative", lambda m: m["margin_permille"].update(impact=-1)), + ("over-limit", lambda m: m["margin_permille"].update(impact=1001)), + ("fraction", lambda m: m["margin_permille"].update(impact=0.2)), + ("boolean", lambda m: m["margin_permille"].update(impact=True)), + ] + for name, mutate in variants: + with self.subTest(name=name), tempfile.TemporaryDirectory() as temp: + changed = copy.deepcopy(manifest) + mutate(changed) + here = Path(temp) + (here / "manifest.json").write_text(json.dumps(changed)) + (here / "dataset.json").write_bytes(self.dataset_bytes) + (here / "captured.json").write_bytes(capture) + (here / "server.json").write_text(json.dumps(self.server_config)) + with patch.object(run_http, "HERE", str(here)), patch.object( + run_http, "post" + ) as post: + with self.assertRaisesRegex(ValueError, "margin_permille"): + run_http.main([ + "--url", self.endpoint, "--tag", "margin-refusal", + "--server-config", str(here / "server.json"), + "--qualification-manifest", str(here / "manifest.json"), + "--captured-requests", str(here / "captured.json"), + qualification.CAPTURED_SCOUT_CONFIG, + ]) + post.assert_not_called() + self.assertFalse((here / "results").exists()) + + # Historical benchmark arms must not silently reinterpret a policy + # margin declaration as a property of their synthetic renderer. + historical = copy.deepcopy(self.manifest) + historical["margin_permille"] = {key: None for key in self.questions} + with self.assertRaisesRegex(ValueError, "only captured Scout"): + self.valid(historical) + + def test_model_only_scout_margin_uses_unqualified_runner_up(self): + thresholds = {"impact": 0.6, "call_graph": 0.95} + questions = ("impact", "call_graph") + rows = [{"labels": ["impact"], "probs": { + "impact": 0.90, "call_graph": 0.89, + }}] + # The competing 890-permille answer is BELOW its own 950 threshold. + # Rust still uses it as the runner-up for impact's 20-permille margin. + baseline = qualification.holdout_decision_metrics( + rows, thresholds, questions + ) + self.assertEqual(baseline["model_only_selected_positive_case_recall"], 1) + gated = qualification.holdout_decision_metrics( + rows, thresholds, questions, + {"impact": 20, "call_graph": None}, + ) + self.assertEqual(gated["model_only_selected_positive_case_recall"], 0) + self.assertEqual(gated["missed_positive_cases"], 1) + self.assertEqual(gated["abstained_cases"], 1) + # A true lead of at least 10 permille is accepted. + admitted = qualification.holdout_decision_metrics( + rows, thresholds, questions, + {"impact": 10, "call_graph": None}, + ) + self.assertEqual(admitted["model_only_selected_positive_case_recall"], 1) + with self.assertRaisesRegex(ValueError, "margins must cover"): + qualification.holdout_decision_metrics( + rows, thresholds, questions, {"impact": 20} + ) + + def test_margin_rejects_wrong_selected_hint_at_rounded_boundary(self): + rows = [{"labels": ["impact"], "probs": { + "impact": 0.9004, "call_graph": 0.9000, + }}] + thresholds = {"impact": 0.6, "call_graph": 0.6} + questions = ("impact", "call_graph") + baseline = qualification.holdout_decision_metrics(rows, thresholds, questions) + self.assertEqual(baseline["model_only_selected_positive_case_recall"], 1) + # Both probabilities round to 900-permille. A nonzero margin + # suppresses the correct intent; the competing no-margin intent wins. + gated = qualification.holdout_decision_metrics( + rows, thresholds, questions, + {"impact": 1, "call_graph": None}, + ) + self.assertEqual(gated["model_only_selected_positive_case_recall"], 0) + self.assertEqual(gated["model_only_selected_unwanted_hint_cases"], 1) + + def test_captured_scout_accepts_canonical_wire_order_not_settings_vector_order(self): + capture, manifest = self.captured_scout_fixture() + self.assertNotEqual(list(self.questions), sorted(self.questions), + "fixture must distinguish settings order from canonical wire order") + records = qualification.loads_unambiguous_json(capture) + wire_questions = qualification.loads_unambiguous_json(records[0]["body"])["questions"] + self.assertEqual(list(wire_questions), sorted(self.questions)) + self.assertNotEqual(list(wire_questions), list(self.questions)) + bodies = qualification.captured_request_bodies( + capture, ["holdout-positive", "holdout-negative"], + self.questions, None, manifest["max_request_bytes"], + manifest["max_state_bytes"], + ) + self.assertEqual(bodies["holdout-positive"], records[0]["body"].encode("utf-8")) + self.assertEqual(bodies["holdout-negative"], records[1]["body"].encode("utf-8")) + + def test_captured_scout_requires_production_endpoint_even_after_refreezing(self): + # A differently routed HTTP service can return valid-looking Noul + # predictions, but Scout's configured client never sends requests + # there. Rehash the manifest endpoint to exercise the actual parity + # guard instead of an incidental digest mismatch. + capture, manifest = self.captured_scout_fixture() + self.valid(manifest) + self.assertIsNone(run_http.require_scout_endpoint(self.endpoint)) + self.assertIsNone(run_http.require_scout_endpoint( + "https://example.test/adapter/v1/systemone" + )) + variants = ( + self.endpoint.replace("/v1/systemone", "/v1/predict"), + self.endpoint + "?model=other", + self.endpoint + "#different", + self.endpoint + "/", + ) + for endpoint in variants: + with self.subTest(endpoint=endpoint), tempfile.TemporaryDirectory() as temp: + here = Path(temp) + refrozen = copy.deepcopy(manifest) + refrozen["arm"]["endpoint_sha256"] = qualification.sha256_bytes( + endpoint.encode("utf-8") + ) + (here / "manifest.json").write_text(json.dumps(refrozen)) + (here / "dataset.json").write_bytes(self.dataset_bytes) + (here / "captured.json").write_bytes(capture) + (here / "server.json").write_text(json.dumps(self.server_config)) + with patch.object(run_http, "HERE", str(here)), patch.object( + run_http, "post" + ) as post: + with self.assertRaisesRegex( + ValueError, "canonical /v1/systemone endpoint" + ): + run_http.main([ + "--url", endpoint, "--tag", "captured-endpoint", + "--server-config", str(here / "server.json"), + "--qualification-manifest", str(here / "manifest.json"), + "--captured-requests", str(here / "captured.json"), + qualification.CAPTURED_SCOUT_CONFIG, + ]) + post.assert_not_called() + self.assertFalse((here / "results").exists()) + + def test_captured_scout_requests_replay_exact_bytes_and_score(self): + # This proves a captured arm is byte-exact, not that the synthetic + # fixture came from real Scout or that its labels were reviewed. + capture_bytes, manifest = self.captured_scout_fixture() + with tempfile.TemporaryDirectory() as temp: + here = Path(temp) + (here / "dataset.json").write_bytes(self.dataset_bytes) + (here / "manifest.json").write_text(json.dumps(manifest)) + (here / "captured.json").write_bytes(capture_bytes) + (here / "server.json").write_text(json.dumps(self.server_config)) + transmitted = [] + + def inference(_url, body, **kwargs): + self.assertIsInstance(body, bytes, "must not reserialize the Rust request") + transmitted.append(body) + request = qualification.loads_unambiguous_json(body) + probability = 0.9 if "What changes?" in request["state"] else 0.1 + return ({ + "model": "reported-scout-fixture", + "answers": { + key: {"noul": probability if key == "impact" else 0.1} + for key in self.questions + }, + "usage": {"input_tokens": 42, "state_tokens_dropped": 0}, + }, 5.0) + + args = [ + "--url", self.endpoint, "--tag", "captured", + "--server-config", str(here / "server.json"), + "--qualification-manifest", str(here / "manifest.json"), + "--captured-requests", str(here / "captured.json"), + qualification.CAPTURED_SCOUT_CONFIG, + ] + with patch.object(run_http, "HERE", str(here)): + with patch.object(run_http, "post", side_effect=inference) as post: + run_http.main(args) + self.assertEqual(post.call_count, 2, "only frozen heldout cases may be inferred") + expected = [ + record["body"].encode("utf-8") + for record in qualification.loads_unambiguous_json(capture_bytes) + ] + self.assertEqual(transmitted, expected) + result = json.loads((here / "results" / + "captured_S_scout_captured.json").read_text()) + report = qualification.qualified_report( + result, manifest, qualification.sha256_bytes( + (here / "manifest.json").read_bytes()), + self.questions, self.dataset_bytes, capture_bytes, + ) + self.assertTrue(report["qualified_against_frozen_tolerances"]) + self.assertEqual(report["cases"], 2) + self.assertEqual([row["request_size_bytes"] for row in result["rows"]], + [len(body) for body in expected]) + with self.assertRaisesRegex(ValueError, "original frozen Scout request capture"): + qualification.qualified_report( + result, manifest, qualification.sha256_bytes( + (here / "manifest.json").read_bytes()), + self.questions, self.dataset_bytes, + ) + altered = capture_bytes.replace(b"What changes?", b"What changed?") + with self.assertRaisesRegex(ValueError, "original frozen Scout request capture"): + qualification.qualified_report( + result, manifest, qualification.sha256_bytes( + (here / "manifest.json").read_bytes()), + self.questions, self.dataset_bytes, altered, + ) + + def test_capture_drift_and_invalid_contract_refuse_before_network(self): + capture_bytes, manifest = self.captured_scout_fixture() + with tempfile.TemporaryDirectory() as temp: + here = Path(temp) + (here / "dataset.json").write_bytes(self.dataset_bytes) + (here / "manifest.json").write_text(json.dumps(manifest)) + (here / "captured.json").write_bytes(capture_bytes) + (here / "server.json").write_text(json.dumps(self.server_config)) + args = [ + "--url", self.endpoint, "--tag", "capture-refusal", + "--server-config", str(here / "server.json"), + "--qualification-manifest", str(here / "manifest.json"), + "--captured-requests", str(here / "captured.json"), + qualification.CAPTURED_SCOUT_CONFIG, + ] + records = qualification.loads_unambiguous_json(capture_bytes) + invalid = [] + reordered = copy.deepcopy(records) + reordered.reverse() + invalid.append(("ordering", reordered)) + injected = copy.deepcopy(records) + injected[0]["body"] = injected[0]["body"].replace( + '"type":"noul"', '"type":"choice"', 1 + ) + invalid.append(("question contract", injected)) + extra = copy.deepcopy(records) + extra[0]["body"] = extra[0]["body"].replace( + '{"questions":', '{"debug":"injected","questions":', 1 + ) + invalid.append(("extra request field", extra)) + for name, changed in invalid: + with self.subTest(name=name): + changed_bytes = json.dumps(changed, ensure_ascii=False).encode("utf-8") + with self.assertRaises(ValueError): + qualification.captured_request_bodies( + changed_bytes, ["holdout-positive", "holdout-negative"], + self.questions, None, manifest["max_request_bytes"], + manifest["max_state_bytes"], + ) + for name, changed in [ + ("changed text", capture_bytes.replace(b"What changes?", b"What changed?")), + ("unexpected case", json.dumps(records[:-1]).encode("utf-8")), + ]: + with self.subTest(name=name): + (here / "captured.json").write_bytes(changed) + with patch.object(run_http, "HERE", str(here)): + with patch.object(run_http, "post") as post: + with self.assertRaises(ValueError): + run_http.main(args) + post.assert_not_called() + self.assertFalse((here / "results" / + "capture-refusal_S_scout_captured.json.partial").exists()) + (here / "captured.json").write_bytes(capture_bytes) + with patch.object(run_http, "HERE", str(here)): + with patch.object(run_http, "post") as post: + with self.assertRaisesRegex(ValueError, "--captured-requests"): + run_http.main(args[:-3] + [qualification.CAPTURED_SCOUT_CONFIG]) + post.assert_not_called() + + def test_captured_question_order_cannot_be_changed_even_when_all_hashes_are_refrozen(self): + # Dict equality ignores insertion order, but the model receives ordered + # question bytes. Rehash every capture/manifest/result identity to + # isolate the missing order check rather than a digest mismatch. + capture, manifest = self.captured_scout_fixture() + records = qualification.loads_unambiguous_json(capture) + for record in records: + body = qualification.loads_unambiguous_json(record["body"]) + body["questions"] = dict(reversed(list(body["questions"].items()))) + self.assertEqual(body["questions"], self.questions) + self.assertNotEqual(list(body["questions"]), sorted(self.questions)) + record["body"] = json.dumps(body, ensure_ascii=False, separators=(",", ":")) + reordered = json.dumps(records, ensure_ascii=False).encode("utf-8") + manifest["captured_requests_sha256"] = qualification.sha256_bytes(reordered) + manifest["holdout_requests_sha256"] = score.evidence_digest([ + [record["id"], qualification.sha256_bytes(record["body"].encode("utf-8"))] + for record in records + ]) + self.valid(manifest) + with self.assertRaisesRegex(ValueError, "canonical Scout wire order"): + qualification.captured_request_bodies( + reordered, ["holdout-positive", "holdout-negative"], + self.questions, None, manifest["max_request_bytes"], + manifest["max_state_bytes"], + ) + + # Runner refusal precedes network traffic and evidence-file creation. + with tempfile.TemporaryDirectory() as temp: + here = Path(temp) + (here / "dataset.json").write_bytes(self.dataset_bytes) + manifest_bytes = json.dumps(manifest).encode("utf-8") + (here / "manifest.json").write_bytes(manifest_bytes) + (here / "server.json").write_text(json.dumps(self.server_config)) + (here / "captured.json").write_bytes(reordered) + with patch.object(run_http, "HERE", str(here)): + with patch.object(run_http, "post") as post: + with self.assertRaisesRegex(ValueError, "question order"): + run_http.main([ + "--url", self.endpoint, "--tag", "reordered-capture", + "--server-config", str(here / "server.json"), + "--qualification-manifest", str(here / "manifest.json"), + "--captured-requests", str(here / "captured.json"), + qualification.CAPTURED_SCOUT_CONFIG, + ]) + post.assert_not_called() + self.assertFalse((here / "results").exists()) + + # An independently supplied result with matching rewritten request + # fingerprints and experiment SHA must be refused by the scorer too. + result = self.signed_result() + for row, record in zip(result["rows"], records, strict=True): + body_bytes = record["body"].encode("utf-8") + row["request_sha256"] = qualification.sha256_bytes(body_bytes) + row["request_size_bytes"] = len(body_bytes) + result["config"] = "trial_" + qualification.CAPTURED_SCOUT_CONFIG + experiment = result["experiment"] + experiment["config"] = qualification.CAPTURED_SCOUT_CONFIG + experiment["requests_sha256"] = manifest["holdout_requests_sha256"] + experiment["qualification_manifest_sha256"] = qualification.sha256_bytes(manifest_bytes) + result["experiment_sha256"] = score.evidence_digest(experiment) + with self.assertRaisesRegex(ValueError, "question order"): + qualification.qualified_report( + result, manifest, qualification.sha256_bytes(manifest_bytes), + self.questions, self.dataset_bytes, reordered, + ) + + def test_captured_state_budget_is_utf8_bytes_and_independently_checked(self): + capture, manifest = self.captured_scout_fixture() + records = qualification.loads_unambiguous_json(capture) + # Two Unicode scalar values are four UTF-8 bytes, not two bytes. + # Recompute every capture and request digest, so the refusal is + # caused by the runtime state limit, not by stale provenance hashes. + for record in records: + body = qualification.loads_unambiguous_json(record["body"]) + body["state"] = "éé" + record["body"] = json.dumps( + body, sort_keys=True, ensure_ascii=False, indent=2, + ) + capture = json.dumps(records, ensure_ascii=False).encode("utf-8") + manifest["captured_requests_sha256"] = qualification.sha256_bytes(capture) + manifest["holdout_requests_sha256"] = score.evidence_digest([ + [record["id"], qualification.sha256_bytes(record["body"].encode("utf-8"))] + for record in records + ]) + for limit in (None, True, 0, "4096"): + with self.subTest(invalid_limit=limit): + invalid = copy.deepcopy(manifest) + invalid["max_state_bytes"] = limit + with self.assertRaisesRegex(ValueError, "max_state_bytes"): + self.valid(invalid) + unexpected = copy.deepcopy(self.manifest) + unexpected["max_state_bytes"] = 4096 + with self.assertRaisesRegex(ValueError, "only captured Scout"): + self.valid(unexpected) + + manifest["max_state_bytes"] = 3 + self.valid(manifest) + with self.assertRaisesRegex(ValueError, "state byte budget"): + qualification.captured_request_bodies( + capture, ["holdout-positive", "holdout-negative"], + self.questions, None, manifest["max_request_bytes"], + manifest["max_state_bytes"], + ) + + with tempfile.TemporaryDirectory() as temp: + here = Path(temp) + (here / "dataset.json").write_bytes(self.dataset_bytes) + manifest_bytes = json.dumps(manifest).encode("utf-8") + (here / "manifest.json").write_bytes(manifest_bytes) + (here / "captured.json").write_bytes(capture) + (here / "server.json").write_text(json.dumps(self.server_config)) + with patch.object(run_http, "HERE", str(here)): + with patch.object(run_http, "post") as inference: + with self.assertRaisesRegex(ValueError, "state byte budget"): + run_http.main([ + "--url", self.endpoint, "--tag", "oversized-state", + "--server-config", str(here / "server.json"), + "--qualification-manifest", str(here / "manifest.json"), + "--captured-requests", str(here / "captured.json"), + qualification.CAPTURED_SCOUT_CONFIG, + ]) + inference.assert_not_called() + self.assertFalse((here / "results").exists()) + + result = self.signed_result() + for row, record in zip(result["rows"], records, strict=True): + encoded = record["body"].encode("utf-8") + row["request_sha256"] = qualification.sha256_bytes(encoded) + row["request_size_bytes"] = len(encoded) + result["config"] = "trial_" + qualification.CAPTURED_SCOUT_CONFIG + experiment = result["experiment"] + experiment["config"] = qualification.CAPTURED_SCOUT_CONFIG + experiment["requests_sha256"] = manifest["holdout_requests_sha256"] + experiment["qualification_manifest_sha256"] = qualification.sha256_bytes(manifest_bytes) + result["experiment_sha256"] = score.evidence_digest(experiment) + with self.assertRaisesRegex(ValueError, "state byte budget"): + qualification.qualified_report( + result, manifest, qualification.sha256_bytes(manifest_bytes), + self.questions, self.dataset_bytes, capture, + ) + + # Boundary acceptance is equally important: the runtime would send + # this four-byte state unchanged with a four-byte budget. + manifest["max_state_bytes"] = 4 + self.valid(manifest) + accepted = qualification.captured_request_bodies( + capture, ["holdout-positive", "holdout-negative"], + self.questions, None, manifest["max_request_bytes"], + manifest["max_state_bytes"], + ) + self.assertEqual(len(accepted), 2) + self.assertEqual( + qualification.loads_unambiguous_json(accepted["holdout-positive"])["state"], + "éé", + ) + + def test_noncanonical_wire_bytes_refuse_even_after_rehashing_all_evidence(self): + # A syntactically different request can decode to identical Noul + # semantics, yet cannot be the original Rust send_json wire payload. + # Freeze new capture/holdout/result digests to bypass hash-mismatch + # guards and exercise the actual production-wire-shape check. + capture, _ = self.captured_scout_fixture() + original = qualification.loads_unambiguous_json(capture) + for encoding in ("top_level_order", "json_spacing"): + with self.subTest(encoding=encoding): + records = copy.deepcopy(original) + for record in records: + decoded = qualification.loads_unambiguous_json(record["body"]) + if encoding == "top_level_order": + changed = json.dumps( + {"state": decoded["state"], "questions": decoded["questions"]}, + ensure_ascii=False, separators=(",", ":"), + ) + else: + changed = json.dumps(decoded, sort_keys=True, ensure_ascii=False) + self.assertEqual(qualification.loads_unambiguous_json(changed), decoded) + self.assertNotEqual(changed, record["body"]) + record["body"] = changed + altered = json.dumps(records, ensure_ascii=False).encode("utf-8") + _, manifest = self.captured_scout_fixture() + manifest["captured_requests_sha256"] = qualification.sha256_bytes(altered) + manifest["holdout_requests_sha256"] = score.evidence_digest([ + [record["id"], qualification.sha256_bytes(record["body"].encode("utf-8"))] + for record in records + ]) + self.valid(manifest) + with self.assertRaisesRegex(ValueError, "canonical Rust JSON wire bytes"): + qualification.captured_request_bodies( + altered, ["holdout-positive", "holdout-negative"], + self.questions, None, manifest["max_request_bytes"], + manifest["max_state_bytes"], + ) + + with tempfile.TemporaryDirectory() as temp: + here = Path(temp) + (here / "dataset.json").write_bytes(self.dataset_bytes) + manifest_bytes = json.dumps(manifest).encode("utf-8") + (here / "manifest.json").write_bytes(manifest_bytes) + (here / "server.json").write_text(json.dumps(self.server_config)) + (here / "captured.json").write_bytes(altered) + with patch.object(run_http, "HERE", str(here)): + with patch.object(run_http, "post") as post: + with self.assertRaisesRegex(ValueError, "canonical Rust JSON wire bytes"): + run_http.main([ + "--url", self.endpoint, "--tag", "noncanonical", + "--server-config", str(here / "server.json"), + "--qualification-manifest", str(here / "manifest.json"), + "--captured-requests", str(here / "captured.json"), + qualification.CAPTURED_SCOUT_CONFIG, + ]) + post.assert_not_called() + self.assertFalse((here / "results").exists()) + + result = self.signed_result() + for row, record in zip(result["rows"], records, strict=True): + raw = record["body"].encode("utf-8") + row["request_sha256"] = qualification.sha256_bytes(raw) + row["request_size_bytes"] = len(raw) + result["config"] = "trial_" + qualification.CAPTURED_SCOUT_CONFIG + experiment = result["experiment"] + experiment["config"] = qualification.CAPTURED_SCOUT_CONFIG + experiment["requests_sha256"] = manifest["holdout_requests_sha256"] + experiment["qualification_manifest_sha256"] = qualification.sha256_bytes( + manifest_bytes + ) + result["experiment_sha256"] = score.evidence_digest(experiment) + with self.assertRaisesRegex(ValueError, "canonical Rust JSON wire bytes"): + qualification.qualified_report( + result, manifest, + qualification.sha256_bytes(manifest_bytes), + self.questions, self.dataset_bytes, altered, + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/evals/intent_engine/test_qualification_cli.py b/evals/intent_engine/test_qualification_cli.py new file mode 100644 index 00000000000..495dda9f0eb --- /dev/null +++ b/evals/intent_engine/test_qualification_cli.py @@ -0,0 +1,254 @@ +"""Exercise qualification exit status through the real scoring entry point. + +The shared cases are evaluator fixtures, not model-qualification evidence. +""" +import json +import shutil +import subprocess +import sys +import tempfile +import unittest +from pathlib import Path + +import test_qualification as fixtures + + +class QualificationCliTests(unittest.TestCase): + def setUp(self): + self.fixture = fixtures.QualificationTests() + self.fixture.setUp() + staging = tempfile.TemporaryDirectory() + self.addCleanup(staging.cleanup) + self.root = Path(staging.name) + source = Path(__file__).resolve().parent + for name in ("common.py", "qualification.py", "run_http.py", "score.py"): + shutil.copyfile(source / name, self.root / name) + (self.root / "manifest.json").write_bytes(self.fixture.manifest_bytes) + (self.root / "dataset.json").write_bytes(self.fixture.dataset_bytes) + + def run_score(self, result): + (self.root / "result.json").write_text( + json.dumps(result, allow_nan=False), encoding="utf-8" + ) + return subprocess.run( + [sys.executable, str(self.root / "score.py"), "--qualified", + str(self.root / "manifest.json"), str(self.root / "dataset.json"), + str(self.root / "result.json")], + cwd=self.root, capture_output=True, text=True, timeout=30, + ) + + def published_report(self, process): + report = json.loads(process.stdout) + published = json.loads( + (self.root / "qualification_summary.json").read_text(encoding="utf-8") + ) + self.assertEqual(published, report) + self.assertEqual(report["cases"], 2) + self.assertFalse((self.root / "summary.json").exists()) + return report + + def test_captured_arm_cli_requires_original_request_bytes(self): + # Synthetic captured requests demonstrate byte-identity enforcement, + # not production Scout provenance or real Kev qualification. + capture_bytes, manifest = self.fixture.captured_scout_fixture() + manifest_bytes = json.dumps(manifest).encode("utf-8") + (self.root / "manifest.json").write_bytes(manifest_bytes) + (self.root / "captured.json").write_bytes(capture_bytes) + records = fixtures.qualification.loads_unambiguous_json(capture_bytes) + result = self.fixture.signed_result() + for row, record in zip(result["rows"], records, strict=True): + raw = record["body"].encode("utf-8") + row["request_sha256"] = fixtures.qualification.sha256_bytes(raw) + row["request_size_bytes"] = len(raw) + experiment = result["experiment"] + experiment["config"] = fixtures.qualification.CAPTURED_SCOUT_CONFIG + experiment["requests_sha256"] = fixtures.score.evidence_digest([ + [row["id"], row["request_sha256"]] for row in result["rows"] + ]) + experiment["qualification_manifest_sha256"] = ( + fixtures.qualification.sha256_bytes(manifest_bytes) + ) + result["config"] = "trial_S_scout_captured" + result["experiment_sha256"] = fixtures.score.evidence_digest(experiment) + (self.root / "result.json").write_text(json.dumps(result), encoding="utf-8") + argv = [ + sys.executable, str(self.root / "score.py"), "--qualified", + str(self.root / "manifest.json"), str(self.root / "dataset.json"), + str(self.root / "result.json"), + ] + missing = subprocess.run(argv, cwd=self.root, capture_output=True, text=True, timeout=30) + self.assertNotEqual(missing.returncode, 0) + self.assertIn("original frozen Scout request capture", missing.stderr) + self.assertFalse((self.root / "qualification_summary.json").exists()) + + complete = argv + ["--captured-requests", str(self.root / "captured.json")] + passing = subprocess.run(complete, cwd=self.root, capture_output=True, text=True, timeout=30) + self.assertEqual(passing.returncode, 0, passing.stderr) + report = self.published_report(passing) + self.assertTrue(report["qualified_against_frozen_tolerances"]) + frozen_report = (self.root / "qualification_summary.json").read_bytes() + (self.root / "captured.json").write_bytes( + capture_bytes.replace(b"What changes?", b"What changed?") + ) + rejected = subprocess.run(complete, cwd=self.root, capture_output=True, text=True, timeout=30) + self.assertNotEqual(rejected.returncode, 0) + self.assertIn("original frozen Scout request capture", rejected.stderr) + self.assertEqual((self.root / "qualification_summary.json").read_bytes(), frozen_report) + + def test_raw_first_response_sample_is_checked_by_independent_cli(self): + result = self.fixture.signed_result() + first = result["rows"][0] + request = { + "state": fixtures.run_http.STATES["B_noul_text"](self.fixture.dataset[2]), + "questions": self.fixture.questions, + } + response = { + "model": first["observed_model"], + "answers": { + key: {"type": "noul", "noul": value} + for key, value in first["probs"].items() + }, + "usage": { + "input_tokens": first["input_tokens"], + "state_tokens_dropped": first["reported_state_tokens_dropped"], + }, + } + first["sample_response_sha256"] = fixtures.run_http.digest(response) + sample = { + "request": request, + "response": response, + "experiment_sha256": result["experiment_sha256"], + "case_id": first["id"], + "request_sha256": first["request_sha256"], + "row_sha256": fixtures.run_http.digest(first), + } + (self.root / "result.json").write_text(json.dumps(result), encoding="utf-8") + sample_file = self.root / "sample.json" + sample_file.write_text(json.dumps(sample), encoding="utf-8") + argv = [ + sys.executable, str(self.root / "score.py"), "--qualified", + str(self.root / "manifest.json"), str(self.root / "dataset.json"), + str(self.root / "result.json"), + ] + missing = subprocess.run( + argv, cwd=self.root, capture_output=True, text=True, timeout=30, + ) + self.assertNotEqual(missing.returncode, 0) + self.assertIn("must be provided together", missing.stderr) + self.assertFalse((self.root / "qualification_summary.json").exists()) + + qualified = argv + ["--sample-file", str(sample_file)] + passing = subprocess.run( + qualified, cwd=self.root, capture_output=True, text=True, timeout=30, + ) + self.assertEqual(passing.returncode, 0, passing.stderr) + report = self.published_report(passing) + self.assertTrue(report["first_response_sample_checked"]) + previous = (self.root / "qualification_summary.json").read_bytes() + + altered = json.loads(sample_file.read_text(encoding="utf-8")) + altered["response"]["answers"]["impact"]["noul"] = 0.4 + sample_file.write_text(json.dumps(altered), encoding="utf-8") + rejected = subprocess.run( + qualified, cwd=self.root, capture_output=True, text=True, timeout=30, + ) + self.assertNotEqual(rejected.returncode, 0) + self.assertIn("response or row disagrees", rejected.stderr) + self.assertEqual(rejected.stdout, "") + self.assertEqual((self.root / "qualification_summary.json").read_bytes(), previous) + + def test_duplicate_result_keys_cannot_publish_a_passing_report(self): + valid = self.fixture.signed_result() + raw = json.dumps(valid, allow_nan=False) + self.assertIn('"config": "trial_B_noul_text"', raw) + (self.root / "result.json").write_text( + raw.replace('"config": "trial_B_noul_text"', + '"config": "trial_B_noul_text", "config": "trial_B_noul_text"', 1), + encoding="utf-8", + ) + process = subprocess.run( + [sys.executable, str(self.root / "score.py"), "--qualified", + str(self.root / "manifest.json"), str(self.root / "dataset.json"), + str(self.root / "result.json")], + cwd=self.root, capture_output=True, text=True, timeout=30, + ) + self.assertNotEqual(process.returncode, 0) + self.assertIn("duplicate JSON object key", process.stderr) + self.assertEqual(process.stdout, "") + self.assertFalse((self.root / "qualification_summary.json").exists()) + + def test_passing_qualification_exits_zero_with_matching_report(self): + process = self.run_score(self.fixture.signed_result()) + self.assertEqual(process.returncode, 0, process.stderr) + self.assertEqual(process.stderr, "") + report = self.published_report(process) + self.assertTrue(report["qualified_against_frozen_tolerances"]) + self.assertTrue(all(report["checks"].values())) + + def test_failed_checks_exit_nonzero_and_replace_a_previous_passing_report(self): + failures = ( + ("negative_false_hint", "none_false_hint_rate", + lambda r: r["rows"][1]["probs"].update(impact=0.9)), + ("extra_positive_hint", "unwanted_hint_rate", + lambda r: r["rows"][0]["probs"].update(call_graph=0.9)), + ("missed_positive", "positive_case_recall", + lambda r: r["rows"][0]["probs"].update(impact=0.1)), + ("slow_response", "p95_ms", lambda r: r["rows"][1].update(ms=21)), + ("unknown_truncation", "untruncated_input_evidence", + lambda r: r["rows"][0].update( + dropped=None, reported_state_tokens_dropped=None)), + ("truncated_input", "untruncated_input_evidence", + lambda r: r["rows"][0].update( + dropped=1, reported_state_tokens_dropped=1)), + ("forged_zero_without_numeric_source", "untruncated_input_evidence", + lambda r: r["rows"][0].pop("reported_state_tokens_dropped")), + ("unmeasured_input_tokens", "reported_input_token_budget", + lambda r: r["rows"][0].update(input_tokens=None)), + ("zero_input_tokens", "reported_input_token_budget", + lambda r: r["rows"][0].update(input_tokens=0)), + ("over_budget_input_tokens", "reported_input_token_budget", + lambda r: r["rows"][0].update(input_tokens=65)), + ) + for name, failed_check, mutate in failures: + with self.subTest(name=name): + passing = self.run_score(self.fixture.signed_result()) + self.assertEqual(passing.returncode, 0, passing.stderr) + self.assertTrue( + self.published_report(passing)["qualified_against_frozen_tolerances"] + ) + result = self.fixture.signed_result() + mutate(result) + process = self.run_score(result) + self.assertEqual(process.returncode, 1, process.stderr) + report = self.published_report(process) + self.assertFalse(report["qualified_against_frozen_tolerances"]) + self.assertFalse(report["checks"][failed_check]) + self.assertIn("qualification failed frozen checks:", process.stderr) + self.assertIn(failed_check, process.stderr) + + def test_invalid_dataset_fails_without_publishing_a_qualification(self): + (self.root / "dataset.json").write_bytes(self.fixture.dataset_bytes + b"\n") + process = self.run_score(self.fixture.signed_result()) + self.assertNotEqual(process.returncode, 0) + self.assertIn("dataset digest differs from supplied evidence", process.stderr) + self.assertEqual(process.stdout, "") + self.assertFalse((self.root / "qualification_summary.json").exists()) + + def test_unqualified_mode_reports_the_complete_required_command(self): + results = self.root / "results" + results.mkdir() + (results / "heldout.json").write_text( + json.dumps(self.fixture.signed_result(), allow_nan=False), encoding="utf-8" + ) + process = subprocess.run( + [sys.executable, str(self.root / "score.py")], + cwd=self.root, capture_output=True, text=True, timeout=30, + ) + self.assertNotEqual(process.returncode, 0) + self.assertIn("--qualified MANIFEST DATASET RESULT", process.stderr) + self.assertFalse((self.root / "summary.json").exists()) + self.assertFalse((self.root / "qualification_summary.json").exists()) + + +if __name__ == "__main__": + unittest.main() diff --git a/evals/intent_engine/test_run_http.py b/evals/intent_engine/test_run_http.py index 6d5e710698c..74479810c2e 100644 --- a/evals/intent_engine/test_run_http.py +++ b/evals/intent_engine/test_run_http.py @@ -78,6 +78,88 @@ def assert_refused_unchanged(self, extra=()): self.assertEqual(self.partial.read_bytes(), before) self.assertFalse(self.output.exists()) + def test_ambiguous_http_json_is_rejected_before_response_admission(self): + for name, payload, message in ( + ("duplicate_answer_map", b'{"answers": {}, "answers": {}}', + "duplicate JSON object key"), + ("nonfinite_unused_field", b'{"answers": {}, "debug": NaN}', + "nonfinite JSON constant"), + ): + with self.subTest(name=name): + with patch("urllib.request.build_opener") as opener: + response = io.BytesIO(payload) + response.status = 200 + opener.return_value.open.return_value = response + with self.assertRaisesRegex(ValueError, message): + run_http.post("http://localhost:8009/v1/systemone", + {"state": "test", "questions": {}}) + + def test_http_response_limit_reads_only_bounded_bytes_and_accepts_boundary(self): + body = b'{"answers":{}}' + max_bytes = run_http.MAX_HTTP_RESPONSE_BYTES + for size, accepted in ((max_bytes, True), (max_bytes + 1, False)): + with self.subTest(size=size): + payload = body + b" " * (size - len(body)) + with patch("urllib.request.build_opener") as opener: + # The HTTP body is read from the context manager's + # entered stream, not the mock returned by opener.open(). + response = opener.return_value.open.return_value.__enter__.return_value + response.status = 200 + response.read.side_effect = io.BytesIO(payload).read + if accepted: + result, latency = run_http.post( + "http://localhost:8009/v1/systemone", + {"state": "test", "questions": {}}, + ) + self.assertEqual(result, {"answers": {}}) + self.assertGreaterEqual(latency, 0) + else: + with self.assertRaisesRegex(ValueError, "bounded byte limit"): + run_http.post( + "http://localhost:8009/v1/systemone", + {"state": "test", "questions": {}}, + ) + response.read.assert_called_once_with(max_bytes + 1) + + def test_oversized_response_preserves_resumable_evidence_without_sample(self): + sample = self.root / "sample.json" + payload = b" " * (run_http.MAX_HTTP_RESPONSE_BYTES + 1) + with patch("urllib.request.build_opener") as opener: + response = io.BytesIO(payload) + response.status = 200 + opener.return_value.open.return_value = response + with self.assertRaisesRegex(ValueError, "bounded byte limit"): + self.run_main(("--sample-out", str(sample))) + self.assertFalse(sample.exists()) + self.assertFalse(self.output.exists()) + self.assertTrue(self.partial.exists()) + self.assertEqual(len(self.partial.read_text().splitlines()), 1) + initial = self.partial.read_bytes() + with patch.object(run_http, "post", side_effect=self.response) as post: + self.run_main(("--sample-out", str(sample))) + self.assertEqual(post.call_count, 2) + self.assertTrue(sample.exists()) + self.assertEqual(len(json.loads(self.output.read_text())["rows"]), 2) + self.assertFalse(self.partial.exists()) + self.assertTrue(initial.endswith(b"\n")) + + def test_ambiguous_partial_row_refuses_without_changing_resumable_evidence(self): + self.interrupt() + original = self.partial.read_bytes() + lines = original.decode("utf-8").splitlines() + self.assertEqual(len(lines), 2) + self.assertIn('"id": "first"', lines[1]) + lines[1] = lines[1].replace('"id": "first"', + '"id": "first", "id": "first"', 1) + self.partial.write_text("\n".join(lines) + "\n") + self.assert_refused_unchanged() + # The original interrupted prefix remains safe to replay. + self.partial.write_bytes(original) + with patch.object(run_http, "post", side_effect=self.response) as post: + self.run_main() + post.assert_called_once() + self.assertEqual(len(json.loads(self.output.read_text())["rows"]), 2) + def test_invalid_dataset_is_refused_before_network_or_partial_output(self): malformed = [ ("not_a_list", {"cases": self.dataset}), @@ -129,6 +211,51 @@ def test_exact_resume_only_requests_the_missing_case(self): self.assertFalse(self.partial.exists()) + def test_resume_refuses_out_of_order_or_skipped_rows_before_network(self): + self.interrupt() + original = self.partial.read_bytes() + header, first_json = original.decode("utf-8").splitlines() + first = json.loads(first_json) + second = self.dataset[1] + body = { + "state": run_http.STATES[self.config](second), + "questions": run_http.noul_questions(), + } + forged_second = { + **first, + "id": second["id"], + "labels": second["labels"], + "context_needed": second["context_needed"], + "request_sha256": hashlib.sha256( + run_http.request_bytes(body) + ).hexdigest(), + } + for name, rows in ( + ("skipped_first", [forged_second]), + ("reversed_prefix", [forged_second, first]), + ): + with self.subTest(name=name): + self.partial.write_text( + header + "\n" + "".join(json.dumps(row) + "\n" for row in rows) + ) + before = self.partial.read_bytes() + with patch.object(run_http, "post") as post: + with self.assertRaisesRegex(ValueError, "contiguous prefix"): + self.run_main() + post.assert_not_called() + self.assertEqual(self.partial.read_bytes(), before) + self.assertFalse(self.output.exists()) + + # A genuine interrupted prefix must still be resumable. + self.partial.write_bytes(original) + with patch.object(run_http, "post", side_effect=self.response) as post: + self.run_main() + post.assert_called_once() + self.assertEqual( + [row["id"] for row in json.loads(self.output.read_text())["rows"]], + ["first", "second"], + ) + def test_completed_rows_cannot_be_relabelled_for_scoring(self): with patch.object(run_http, "post", side_effect=self.response): self.run_main() @@ -315,6 +442,90 @@ def test_reported_usage_and_zero_counts_survive_resume(self): self.assertEqual(timing["truncation_measured_cases"], 2) self.assertEqual(timing["cases_truncated"], 1) + def test_inconsistent_state_usage_is_rejected_before_resume_or_new_append(self): + self.interrupt() + header, row = self.partial.read_text().splitlines() + forged = json.loads(row) + forged.update( + input_tokens=1, state_tokens=32, state_tokens_used=32, dropped=0, + ) + self.partial.write_text(header + "\n" + json.dumps(forged) + "\n") + before = self.partial.read_bytes() + with patch.object(run_http, "post") as post: + with self.assertRaisesRegex(ValueError, "used state tokens exceed total input tokens"): + self.run_main() + post.assert_not_called() + self.assertEqual(self.partial.read_bytes(), before) + self.assertFalse(self.output.exists()) + + # The same impossible values in a fresh server response must not + # append a row or poison the original interrupted evidence. + valid, elapsed = self.response("", {"questions": run_http.noul_questions()}) + impossible = copy.deepcopy(valid) + impossible["usage"] = { + "input_tokens": 1, "state_tokens": 32, "state_tokens_used": 32, + } + impossible["truncated"] = False + # Restore a genuine first row before exercising the new response. + self.partial.write_text(header + "\n" + row + "\n") + genuine_prefix = self.partial.read_bytes() + with patch.object(run_http, "post", return_value=(impossible, elapsed)) as post: + with self.assertRaisesRegex(ValueError, "used state tokens exceed total input tokens"): + self.run_main() + post.assert_called_once() + self.assertEqual(self.partial.read_bytes(), genuine_prefix) + self.assertFalse(self.output.exists()) + + with patch.object(run_http, "post", return_value=(valid, elapsed)) as post: + self.run_main() + post.assert_called_once() + self.assertTrue(self.output.exists()) + self.assertFalse(self.partial.exists()) + self.assertEqual(len(json.loads(self.output.read_text())["rows"]), 2) + + def test_dropped_count_exceeding_original_state_is_refused(self): + with self.assertRaisesRegex(ValueError, "dropped state tokens exceed original"): + run_http.dropped_state_tokens( + {}, {"state_tokens": 3, "state_tokens_dropped": 4}, + ) + self.interrupt() + before = self.partial.read_bytes() + impossible, elapsed = self.response("", {"questions": run_http.noul_questions()}) + impossible["usage"] = { + "input_tokens": 12, "state_tokens": 3, "state_tokens_dropped": 4, + } + with patch.object(run_http, "post", return_value=(impossible, elapsed)) as post: + with self.assertRaisesRegex(ValueError, "dropped state tokens exceed original"): + self.run_main() + post.assert_called_once() + self.assertEqual(self.partial.read_bytes(), before) + self.assertFalse(self.output.exists()) + + def test_legacy_normalized_drops_cannot_exceed_original_on_resume(self): + self.interrupt() + header, row = self.partial.read_text().splitlines() + forged = json.loads(row) + # No source counters/flags: the normalized persisted row must still + # obey the basic nonnegative used-state invariant. + forged.update(state_tokens=3, dropped=4, input_tokens=12) + self.partial.write_text(header + "\n" + json.dumps(forged) + "\n") + bad_bytes = self.partial.read_bytes() + with patch.object(run_http, "post") as post: + with self.assertRaisesRegex(ValueError, "normalized dropped tokens exceed original state"): + self.run_main() + post.assert_not_called() + self.assertEqual(self.partial.read_bytes(), bad_bytes) + self.assertFalse(self.output.exists()) + + # A genuine interrupted prefix remains resumable after refusal. + self.partial.write_text(header + "\n" + row + "\n") + response, elapsed = self.response("", {"questions": run_http.noul_questions()}) + with patch.object(run_http, "post", return_value=(response, elapsed)) as post: + self.run_main() + post.assert_called_once() + self.assertTrue(self.output.exists()) + self.assertEqual(len(json.loads(self.output.read_text())["rows"]), 2) + def test_missing_usage_survives_resume_and_scoring(self): response, elapsed = self.response("", {"questions": run_http.noul_questions()}) response["usage"] = None @@ -852,22 +1063,48 @@ def test_dangling_sample_link_refuses_without_inference_or_partial_output(self): self.assertFalse(sample.exists()) self.assertFalse(self.partial.exists()) # Also guard the atomic-publish race after the initial preflight. - with self.assertRaisesRegex(ValueError, "not a regular file"): + with self.assertRaisesRegex(ValueError, "sample path exists or was replaced during publication"): run_http.publish_sample(str(sample), '{"valid": true}') self.assertEqual(list(self.root.glob(".noul-sample-*")), []) @unittest.skipUnless(os.name == "posix", "POSIX umask semantics") - def test_published_sample_matches_normal_exclusive_creation_mode(self): + def test_private_samples_and_results_ignore_permissive_umask(self): sample = self.root / "first-response.json" - previous_umask = os.umask(0o022) + previous_umask = os.umask(0) try: with patch.object(run_http, "post", side_effect=self.response): self.run_main(["--sample-out", str(sample)]) finally: os.umask(previous_umask) - self.assertEqual(stat.S_IMODE(sample.stat().st_mode), 0o644) + self.assertEqual(stat.S_IMODE(sample.stat().st_mode), 0o600) + self.assertEqual(stat.S_IMODE(self.output.stat().st_mode), 0o600) self.assertEqual(len(json.loads(self.output.read_text())["rows"]), 2) + @unittest.skipUnless(os.name == "posix", "POSIX file permissions") + def test_private_partial_creation_and_tightening_on_resume(self): + previous_umask = os.umask(0) + try: + self.interrupt() + self.assertEqual(stat.S_IMODE(self.partial.stat().st_mode), 0o600) + # A partial written by the previous evaluator may have been + # group/world-readable. Preserve its bytes and resume safely. + frozen_bytes = self.partial.read_bytes() + os.chmod(self.partial, 0o666) + + def assert_private_before_network(url, body): + self.assertEqual(stat.S_IMODE(self.partial.stat().st_mode), 0o600) + self.assertEqual(self.partial.read_bytes(), frozen_bytes) + return self.response(url, body) + + with patch.object(run_http, "post", side_effect=assert_private_before_network) as post: + self.run_main() + post.assert_called_once() + self.assertFalse(self.partial.exists()) + self.assertEqual(stat.S_IMODE(self.output.stat().st_mode), 0o600) + self.assertEqual(len(json.loads(self.output.read_text())["rows"]), 2) + finally: + os.umask(previous_umask) + def test_calibration_remains_finite_at_extreme_positive_temperatures(self): for temperature in (1e-300, 1.0, 1e300): for probability in (0, 1e-300, 0.25, 0.5, 0.75, 1): @@ -878,6 +1115,7 @@ def test_calibration_remains_finite_at_extreme_positive_temperatures(self): def test_transmitted_bytes_match_the_recorded_request_fingerprint(self): response = io.BytesIO(json.dumps({"answers": {}}).encode()) + response.status = 200 body = {"state": "Unicode λ", "questions": {"a": {}, "b": {}}} with patch("urllib.request.build_opener") as build_opener: build_opener.return_value.open.return_value = response @@ -949,6 +1187,55 @@ def test_bad_auth_configuration_refuses_before_http_or_partial_writes(self): post.assert_not_called() self.assertFalse(self.partial.exists()) + def test_non_200_success_status_does_not_become_scout_evidence(self): + # urllib accepts all 2xx statuses, but the production Scout client + # refuses any status other than 200 before parsing a typed response. + status = [201] + requests = [] + reply = json.dumps({ + "answers": {key: {"noul": 0.25} for key in run_http.noul_questions()}, + "usage": {"input_tokens": 12}, + }).encode("utf-8") + + class RespondWithStatus(BaseHTTPRequestHandler): + def do_POST(self): + requests.append(self.rfile.read(int(self.headers["Content-Length"]))) + self.send_response(status[0]) + self.send_header("Content-Length", str(len(reply))) + self.end_headers() + self.wfile.write(reply) + + def log_message(self, *_): + pass + + server = HTTPServer(("127.0.0.1", 0), RespondWithStatus) + worker = Thread(target=server.serve_forever, daemon=True) + worker.start() + self.base[1] = f"http://127.0.0.1:{server.server_port}/v1/systemone" + sample = self.root / "http-status-sample.json" + try: + for code in (201, 202, 206): + with self.subTest(status=code): + status[0] = code + with self.assertRaisesRegex(ValueError, f"decision HTTP status {code}"): + self.run_main(("--sample-out", str(sample))) + self.assertTrue(self.partial.exists()) + self.assertEqual(len(self.partial.read_bytes().splitlines()), 1) + self.assertFalse(self.output.exists()) + self.assertFalse(sample.exists()) + # A legitimate 200 on the same frozen experiment can recover + # without accepting the previously refused response as a row. + status[0] = 200 + self.run_main(("--sample-out", str(sample))) + self.assertEqual(len(json.loads(self.output.read_text())["rows"]), 2) + self.assertTrue(sample.exists()) + self.assertFalse(self.partial.exists()) + self.assertEqual(len(requests), 5) + finally: + server.shutdown() + worker.join(timeout=2) + server.server_close() + def test_bearer_transport_sends_only_to_selected_loopback_server(self): received = []