From 5f5e8bd1f96bdb638afd44838c9d5a4ee96b81ff Mon Sep 17 00:00:00 2001 From: Robb Walters Date: Fri, 20 Nov 2026 02:05:24 -0800 Subject: [PATCH 1/3] fix(forge): recover admission reads after GraphQL exhaustion --- .../src/claim_reconciliation/gh_call.rs | 26 +- loom-daemon/src/cmd_out.rs | 7 +- loom-daemon/src/forge_check_claim.rs | 170 +++++++++++- loom-daemon/src/gh_invocation/mod.rs | 3 + loom-daemon/src/gh_invocation/outcome.rs | 11 +- .../src/gh_invocation/reader_route_v2.rs | 26 +- .../gh_invocation/reader_route_v2_tests.rs | 97 +++++++ .../ops/safehouse_forge_lookup.rs | 1 + loom-daemon/src/parent_link.rs | 247 +++++++++++++++++- .../src/release_resolve/resolve/tests.rs | 1 + 10 files changed, 567 insertions(+), 22 deletions(-) diff --git a/loom-daemon/src/claim_reconciliation/gh_call.rs b/loom-daemon/src/claim_reconciliation/gh_call.rs index d9966fce87..527e06613f 100644 --- a/loom-daemon/src/claim_reconciliation/gh_call.rs +++ b/loom-daemon/src/claim_reconciliation/gh_call.rs @@ -109,7 +109,9 @@ pub(crate) fn forge_op_for(op: &str) -> Option { "park_hold.issue_view" | "release_veto.issue_labels" | "intake.recheck" - | "landed.issue_labels" => ops::ISSUE_VIEW_STATE, + | "landed.issue_labels" + | "claim.labels_rest" => ops::ISSUE_VIEW_STATE, + "parent_link.marker_children_rest" => ops::ISSUE_SEARCH, "park_hold.issue_body" => ops::ISSUE_EDIT_BODY, "quarantine.issue_list" => ops::ISSUE_LIST, "claim.issue_reclaim" @@ -210,3 +212,25 @@ pub(crate) fn ok_stdout(inv: GhInvocation) -> Option> { pub(crate) fn stderr(out: &Output) -> String { String::from_utf8_lossy(&out.stderr).trim().to_string() } + +/// Run a known GraphQL admission read. `None` specifically means its primary +/// GraphQL budget is exhausted, whether reported by gh or by the reader router. +pub(crate) fn graphql_output(inv: GhInvocation) -> Result> { + let op = inv.operation().as_str(); + match inv.run() { + CmdOutcome::Ran(out) if graphql_budget_exhausted(&out) => Ok(None), + CmdOutcome::Ran(out) => Ok(Some(out)), + CmdOutcome::Unavailable(crate::cmd_out::Unavailable::Shed { + graphql_budget_exhausted: true, + .. + }) => Ok(None), + CmdOutcome::Unavailable(why) => Err(anyhow!("failed to invoke gh ({op}): {why}")), + } +} + +fn graphql_budget_exhausted(out: &Output) -> bool { + let diagnostic = String::from_utf8_lossy(&out.stderr); + !out.status.success() + && crate::rate_limit_breaker::indicates_rate_limit(&diagnostic) + && !crate::rate_limit_breaker::evidence::is_secondary_limit(&diagnostic) +} diff --git a/loom-daemon/src/cmd_out.rs b/loom-daemon/src/cmd_out.rs index 4e9edc46b2..4aecc0ebcc 100644 --- a/loom-daemon/src/cmd_out.rs +++ b/loom-daemon/src/cmd_out.rs @@ -107,7 +107,10 @@ pub enum Unavailable { /// exhausted until `until`. Nothing was sent. Its text deliberately /// matches no rate-limit signature, so feeding it to the rate-limit /// breaker can never trip it. - Shed { until: std::time::SystemTime }, + Shed { + until: std::time::SystemTime, + graphql_budget_exhausted: bool, + }, /// Not run (#12641): a forge read the work-finder tick stopped (its /// deadline passed, or its discovery was ended early). Nothing was sent /// and the reader budget plays no part, so it is deliberately not @@ -125,7 +128,7 @@ impl std::fmt::Display for Unavailable { Unavailable::RoutingBlocked(e) => write!(f, "routing_blocked: {e}"), Unavailable::AdapterUnavailable(e) => write!(f, "adapter_unavailable: {e}"), Unavailable::DeadlineReached => write!(f, "deferred: tick deadline reached"), - Unavailable::Shed { until } => write!( + Unavailable::Shed { until, .. } => write!( f, "deferred: reader budget low until {} (loom-shed)", chrono::DateTime::::from(*until) diff --git a/loom-daemon/src/forge_check_claim.rs b/loom-daemon/src/forge_check_claim.rs index 5a3e0b5a2b..6eeaae7a35 100644 --- a/loom-daemon/src/forge_check_claim.rs +++ b/loom-daemon/src/forge_check_claim.rs @@ -712,12 +712,14 @@ pub(crate) fn read_claim_labels_for( "labels,state,url", ]) .args(crate::claim_reconciliation::gh_call::loom_repo_flag()); - let Ok(out) = crate::claim_reconciliation::gh_call::output(inv) else { - return LabelLeg::Unknown; + let out = match crate::claim_reconciliation::gh_call::graphql_output(inv) { + Ok(Some(out)) if out.status.success() => out, + Ok(None) => { + return read_claim_labels_rest(gh_bin, root, issue, claim_owned) + .unwrap_or(LabelLeg::Unknown); + } + _ => return LabelLeg::Unknown, }; - if !out.status.success() { - return LabelLeg::Unknown; - } // `gh issue view --json labels,state,url` emits // `{"labels":[{"name":"..."},…],"state":"OPEN","url":"https://…/issues/42"}`. let Ok(parsed) = serde_json::from_slice::(&out.stdout) else { @@ -746,13 +748,56 @@ pub(crate) fn read_claim_labels_for( .iter() .filter_map(|l| l.get("name").and_then(|n| n.as_str()).map(String::from)) .collect(); - if crate::work_finder::is_frozen(&labels) { + classify_claim_labels(&labels, claim_owned) +} + +fn read_claim_labels_rest( + gh_bin: &Path, + root: &Path, + issue: u32, + claim_owned: bool, +) -> Option { + use crate::claim_reconciliation::gh_call; + // The facade applies LOOM_REPO as GH_REPO for the REST placeholders, + // just as the GraphQL call above applies it through --repo. + let path = format!("repos/{{owner}}/{{repo}}/issues/{issue}"); + let read = gh_call::read("claim.labels_rest", gh_bin, root).args(["api", &path]); + let json: serde_json::Value = serde_json::from_slice(&gh_call::ok_stdout(read)?).ok()?; + if json.get("number")?.as_u64()? != u64::from(issue) { + return None; + } + let pull_request = json.get("pull_request"); + if let Some(pr) = pull_request { + if pr.is_object() { + return Some(LabelLeg::IsPullRequest); + } + return None; + } + match json.get("state")?.as_str()? { + "closed" => return Some(LabelLeg::Closed), + "open" => {} + _ => return None, + } + let labels: Vec = json + .get("labels")? + .as_array()? + .iter() + .map(|label| { + let name = label.get("name")?.as_str()?; + (!name.is_empty()).then(|| name.to_owned()) + }) + .collect::>()?; + Some(classify_claim_labels(&labels, claim_owned)) +} + +fn classify_claim_labels(labels: &[String], claim_owned: bool) -> LabelLeg { + if crate::work_finder::is_frozen(labels) { return LabelLeg::Frozen; } - if crate::work_finder::is_operator_hold(&labels) { + if crate::work_finder::is_operator_hold(labels) { return LabelLeg::OperatorHold; } - let claims = claim_labels_in(&labels); + let claims = claim_labels_in(labels); let triage = labels .iter() .any(|l| l == crate::intake_reconcile::TRIAGE_LABEL); @@ -1840,6 +1885,115 @@ mod tests { assert_eq!(read_claim_labels(&garbage, dir.path(), 42), LabelLeg::Unknown); } + fn claim_label_fallback( + directory: &Path, + diagnostic: &str, + rest_reply: &str, + claim_owned: bool, + ) -> LabelLeg { + let script = format!( + r#"case "$1 $2" in + "issue view") printf '%s\n' '{diagnostic}' >&2; exit 1 ;; + "api repos/{{owner}}/{{repo}}/issues/42") printf '%s\n' '{rest_reply}' ;; + *) exit 76 ;; +esac"#, + ); + let gh = write_exec(directory, "gh-label-fallback.sh", &script); + read_claim_labels_for(&gh, directory, 42, claim_owned) + } + + const GRAPHQL_EXHAUSTED: &str = + "GraphQL: API rate limit already exceeded for installation ID 1"; + + #[test] + fn exhausted_graphql_labels_use_the_same_claim_predicates_via_rest() { + let dir = tempdir().unwrap(); + let rest = |payload: &str, owned| { + claim_label_fallback(dir.path(), GRAPHQL_EXHAUSTED, payload, owned) + }; + for (payload, expected) in [ + ( + r#"{"number":42,"state":"open","labels":[],"pull_request":{}}"#, + LabelLeg::IsPullRequest, + ), + (r#"{"number":42,"state":"closed","labels":[]}"#, LabelLeg::Closed), + ( + r#"{"number":42,"state":"open","labels":[{"name":"loom:frozen"}]}"#, + LabelLeg::Frozen, + ), + ( + r#"{"number":42,"state":"open","labels":[{"name":"loom:operator"}]}"#, + LabelLeg::OperatorHold, + ), + ( + r#"{"number":42,"state":"open","labels":[{"name":"loom:triage"}]}"#, + LabelLeg::AwaitingCurator, + ), + (r#"{"number":42,"state":"open","labels":[]}"#, LabelLeg::Clean), + ] { + assert_eq!(rest(payload, false), expected); + } + for label in ["loom:building", "loom:reviewing", "loom:treating"] { + let payload = + format!(r#"{{"number":42,"state":"open","labels":[{{"name":"{label}"}}]}}"#); + assert_eq!(rest(&payload, false), LabelLeg::Claimed(vec![label.into()])); + } + assert_eq!( + rest( + r#"{"number":42,"state":"open","labels":[{"name":"loom:building"},{"name":"loom:triage"}]}"#, + true + ), + LabelLeg::AwaitingCurator + ); + } + + #[test] + fn rest_labels_refuse_unreadable_or_incomplete_answers() { + let dir = tempdir().unwrap(); + for payload in [ + "not json", + r#"{"number":43,"state":"open","labels":[]}"#, + r#"{"number":42,"labels":[]}"#, + r#"{"number":42,"state":"open","labels":[{}]}"#, + r#"{"number":42,"state":"open","labels":[],"pull_request":false}"#, + ] { + assert_eq!( + claim_label_fallback(dir.path(), GRAPHQL_EXHAUSTED, payload, false), + LabelLeg::Unknown + ); + } + let gh = write_exec( + dir.path(), + "gh-both-limited.sh", + r#"case "$1" in + issue) echo 'GraphQL: API rate limit exceeded' >&2 ;; + *) printf '%s\n' '{"number":42,"state":"open","labels":[]}' + echo 'HTTP 403: API rate limit exceeded' >&2 ;; +esac +exit 1"#, + ); + assert_eq!(read_claim_labels(&gh, dir.path(), 42), LabelLeg::Unknown); + } + + #[test] + fn permissions_and_secondary_limits_do_not_try_label_fallback() { + let dir = tempdir().unwrap(); + for diagnostic in [ + "HTTP 403: Resource not accessible by integration", + "GraphQL: You have exceeded a secondary rate limit", + ] { + assert_eq!( + claim_label_fallback( + dir.path(), + diagnostic, + r#"{"number":42,"state":"open","labels":[]}"#, + false, + ), + LabelLeg::Unknown + ); + } + } + #[test] fn read_freshest_live_lease_distinguishes_held_none_and_read_failed() { let dir = tempdir().unwrap(); diff --git a/loom-daemon/src/gh_invocation/mod.rs b/loom-daemon/src/gh_invocation/mod.rs index 6539156408..72d3d7f7ef 100644 --- a/loom-daemon/src/gh_invocation/mod.rs +++ b/loom-daemon/src/gh_invocation/mod.rs @@ -310,6 +310,9 @@ pub enum GhCompletion { owner: String, resource: crate::forge_bucket_book::Resource, until: std::time::SystemTime, + /// A gate read exhausted its primary GraphQL reader budget. + /// Other sheds give no evidence that another transport can read. + graphql_budget_exhausted: bool, }, /// Not run (#12641): the work-finder tick stopped this read (deadline /// passed, or discovery ended). No request was sent and no reader budget diff --git a/loom-daemon/src/gh_invocation/outcome.rs b/loom-daemon/src/gh_invocation/outcome.rs index 8440dc699d..9b01f0edd2 100644 --- a/loom-daemon/src/gh_invocation/outcome.rs +++ b/loom-daemon/src/gh_invocation/outcome.rs @@ -77,9 +77,14 @@ pub(super) fn classify( format!("passthrough completion ({status}) has no captured output"), )), // W4-C: a deferred read is "no answer", never a negative one. - Ok(GhCompletion::Shed { until, .. }) => { - CmdOutcome::Unavailable(Unavailable::Shed { until }) - } + Ok(GhCompletion::Shed { + until, + graphql_budget_exhausted, + .. + }) => CmdOutcome::Unavailable(Unavailable::Shed { + until, + graphql_budget_exhausted, + }), // #12641: the tick stopped the read; not a reader shed. Ok(GhCompletion::DeadlineReached) => CmdOutcome::Unavailable(Unavailable::DeadlineReached), Err(ExecError::Spawn(e)) => CmdOutcome::Unavailable(Unavailable::Spawn(e.to_string())), diff --git a/loom-daemon/src/gh_invocation/reader_route_v2.rs b/loom-daemon/src/gh_invocation/reader_route_v2.rs index e379a66e34..98336796a4 100644 --- a/loom-daemon/src/gh_invocation/reader_route_v2.rs +++ b/loom-daemon/src/gh_invocation/reader_route_v2.rs @@ -159,6 +159,7 @@ impl GhInvocation { let mut tried: HashSet = HashSet::new(); let mut last_app: Option = None; let mut last_failure: Option = None; + let mut primary_graphql_failures_only = true; let mut decision = lookup(&request); loop { let (dir, app_id) = match decision { @@ -180,7 +181,15 @@ impl GhInvocation { )); } RouteDecision::Exhausted { until, cause } => { - let why = ShedWhy::Router(cause.as_str()); + let why = if cause == ExhaustCause::Budget + && class == ReadClass::Gate + && resource == Resource::Graphql + && primary_graphql_failures_only + { + ShedWhy::GraphqlBudget + } else { + ShedWhy::Router(cause.as_str()) + }; return Ok(self.shed_read( class, &owner, @@ -278,6 +287,15 @@ impl GhInvocation { // A rate limit, a refused token, or a 403/404 from a reader with // no evidence it can see the repo: withdraw it (for this repo // only on coverage) and ask the router for the next one. + primary_graphql_failures_only &= matches!( + failure, + Failure::RateLimited { + resource: Resource::Graphql, + secondary: false, + remaining: None | Some(0), + .. + } + ); withdraw(&app_id, &slug, failure, &why); tried.insert(app_id.clone()); last_app = Some(app_id); @@ -398,6 +416,7 @@ impl GhInvocation { owner: owner.to_string(), resource, until, + graphql_budget_exhausted: why == ShedWhy::GraphqlBudget, } } @@ -437,7 +456,7 @@ impl GhInvocation { // and its source), so a shed run is diagnosable from the log. let readers = match why { ShedWhy::NoRoute => None, - ShedWhy::Router(_) => self + ShedWhy::Router(_) | ShedWhy::GraphqlBudget => self .reader_slug() .or_else(|| super::super::accounting::resolved_identity(self).repo) .and_then(|slug| { @@ -471,6 +490,8 @@ pub(crate) enum ShedWhy { /// asked (a checkout whose local sources disagree, an argv shape it does /// not route). NoRoute, + /// A GraphQL gate's eligible reader budget is exhausted. + GraphqlBudget, /// The router's answer: an [`crate::forge_identity::ExhaustCause`], or /// what the read chain found. Router(&'static str), @@ -485,6 +506,7 @@ impl ShedWhy { so no reader was asked" .to_string(), Self::Router(cause) => format!("router: {cause}"), + Self::GraphqlBudget => "router: budget".to_string(), } } } diff --git a/loom-daemon/src/gh_invocation/reader_route_v2_tests.rs b/loom-daemon/src/gh_invocation/reader_route_v2_tests.rs index 82c91ffbe8..5e84b46dce 100644 --- a/loom-daemon/src/gh_invocation/reader_route_v2_tests.rs +++ b/loom-daemon/src/gh_invocation/reader_route_v2_tests.rs @@ -216,6 +216,102 @@ fn a_gate_read_is_shed_like_any_other_and_an_unclassified_read_is_gate() { assert_eq!(unclassified.class(), ReadClass::Gate); } +#[test] +fn only_primary_graphql_gate_exhaustion_is_preserved_for_a_rest_fallback() { + const PRIMARY: &str = "echo 'GraphQL: API rate limit already exceeded' >&2; exit 1"; + const SECONDARY: &str = "echo 'GraphQL: You have exceeded a secondary rate limit' >&2; exit 1"; + for (label, reader, class, cause, pre, expected) in [ + ( + "already exhausted", + PRIMARY, + ReadClass::Gate, + ExhaustCause::Budget, + &["app-1", "app-2"][..], + true, + ), + ( + "observed primary", + PRIMARY, + ReadClass::Gate, + ExhaustCause::Budget, + &[][..], + true, + ), + ("permission", NOT_FOUND, ReadClass::Gate, ExhaustCause::Budget, &[][..], false), + ("secondary", SECONDARY, ReadClass::Gate, ExhaustCause::Budget, &[][..], false), + ("withdrawn", PRIMARY, ReadClass::Gate, ExhaustCause::Withdrawn, &[][..], false), + ( + "headroom reserve", + PRIMARY, + ReadClass::Hygiene, + ExhaustCause::Budget, + &[][..], + false, + ), + ] { + let p = pool(PRIMARY, reader, OK); + let read = GhInvocation::new( + Operation::new("parent_link.marker_children"), + AccessIntent::Read, + GhTarget::None, + Duration::from_secs(10), + ) + .program(&p.gh) + .reader_only() + .reader_repo("acme/admission") + .args(["issue", "list", "--repo", "acme/admission"]) + .read_class(class); + let (out, _, _) = p.run_as(read, pre, SystemTime::now(), cause); + assert_eq!( + p.who(), + if pre.is_empty() { + vec![Who::R2, Who::R1] + } else { + Vec::new() + }, + "{label}: never uses the writer", + ); + let outcome = super::super::outcome::classify(Ok(out), Duration::from_secs(10)); + let CmdOutcome::Unavailable(Unavailable::Shed { + graphql_budget_exhausted, + .. + }) = outcome + else { + panic!("{label}: expected a shed, got {outcome:?}"); + }; + assert_eq!(graphql_budget_exhausted, expected, "{label}"); + } +} + +#[test] +fn repo_scoped_rest_search_uses_the_intended_reader() { + let p = pool(OK, OK, "exit 76"); + let request = GhInvocation::new( + Operation::new("parent_link.marker_children_rest"), + AccessIntent::Read, + GhTarget::None, + Duration::from_secs(10), + ) + .program(&p.gh) + .reader_only() + .reader_repo("acme/admission-search") + .args([ + "api", + "search/issues", + "--method", + "GET", + "-f", + "q=marker repo:acme/admission-search is:issue", + ]); + let (out, _, asked) = p.run(request, &[], SystemTime::now()); + assert!(matches!( + out, + GhCompletion::Captured(Completion::Exited(out)) if out.status.success() + )); + assert_eq!(asked, ["acme/admission-search"]); + assert_eq!(p.calls(), vec![(Who::R2, "acme/admission-search".into())]); +} + #[test] fn exhausted_up_front_every_class_sheds_with_no_request() { for (name, class) in [ @@ -568,6 +664,7 @@ fn every_reader_refusing_the_repo_sheds_never_absent_and_never_the_writer() { fn a_shed_never_reads_as_a_rate_limit() { let shed = Unavailable::Shed { until: SystemTime::UNIX_EPOCH + Duration::from_secs(1_900_000_000), + graphql_budget_exhausted: false, }; let text = shed.to_string(); assert!(text.starts_with("deferred: reader budget low until 2030-"), "{text}"); diff --git a/loom-daemon/src/observability/ops/safehouse_forge_lookup.rs b/loom-daemon/src/observability/ops/safehouse_forge_lookup.rs index 0060131b59..683e820859 100644 --- a/loom-daemon/src/observability/ops/safehouse_forge_lookup.rs +++ b/loom-daemon/src/observability/ops/safehouse_forge_lookup.rs @@ -73,6 +73,7 @@ mod tests { }; let shed = Unavailable::Shed { until: SystemTime::now(), + graphql_budget_exhausted: false, }; assert_eq!(reason_class(&timed_out), "timed_out"); assert_eq!(reason_class(&shed), "shed"); diff --git a/loom-daemon/src/parent_link.rs b/loom-daemon/src/parent_link.rs index e4a3914225..3beb3f9283 100644 --- a/loom-daemon/src/parent_link.rs +++ b/loom-daemon/src/parent_link.rs @@ -206,11 +206,14 @@ fn parse_marker_output(text: &str, parent: u32) -> Result> { /// Either read failed, was malformed, or filled a whole page. An unreadable /// list is never evidence that the parent is split and finished. pub fn split_state(root: &Path, slug: &str, parent: u32) -> Result { - let gh = gh_program(); + split_state_with_gh(&gh_program(), root, slug, parent) +} + +fn split_state_with_gh(gh: &Path, root: &Path, slug: &str, parent: u32) -> Result { let mut all: Vec<(u32, bool)> = Vec::new(); let path = format!("repos/{slug}/issues/{parent}/sub_issues?per_page={CHILD_PAGE}&state=all"); - let sub = gh_call::read("parent_link.sub_issues", &gh, root).args([ + let sub = gh_call::read("parent_link.sub_issues", gh, root).args([ "api", &path, "--jq", @@ -233,7 +236,7 @@ pub fn split_state(root: &Path, slug: &str, parent: u32) -> Result { parent_marker(parent) ); let limit = MARKER_LIMIT.to_string(); - let marker = gh_call::read("parent_link.marker_children", &gh, root).args([ + let marker = gh_call::read("parent_link.marker_children", gh, root).args([ "issue", "list", "--repo", @@ -249,13 +252,118 @@ pub fn split_state(root: &Path, slug: &str, parent: u32) -> Result { "--jq", &jq, ]); - let out = gh_call::ok_stdout(marker) - .ok_or_else(|| anyhow::anyhow!("could not search marker children of #{parent}"))?; - let rows = parse_marker_output(&String::from_utf8_lossy(&out), parent)?; + let out = gh_call::graphql_output(marker) + .map_err(|_| anyhow::anyhow!("could not search marker children of #{parent}"))?; + let rows = match out { + Some(out) if out.status.success() => { + parse_marker_output(&String::from_utf8_lossy(&out.stdout), parent)? + } + None => rest_marker_children(gh, root, slug, parent, &query)?, + Some(_) => anyhow::bail!("could not search marker children of #{parent}"), + }; all.extend(rows.into_iter().filter(|&(n, _)| n != parent)); Ok(classify_children(&all)) } +#[derive(serde::Deserialize)] +struct RestMarkerPage { + total_count: usize, + incomplete_results: bool, + items: Vec, +} + +#[derive(serde::Deserialize)] +struct RestMarkerItem { + number: u32, + state: RestIssueState, + // A null body is normal. A missing or non-string body could hide a marker. + body: serde_json::Value, + repository_url: String, + #[serde(default, rename = "pull_request", deserialize_with = "field_present")] + has_pull_request: bool, +} + +fn field_present<'de, D: serde::Deserializer<'de>>(value: D) -> Result { + let _: serde::de::IgnoredAny = serde::Deserialize::deserialize(value)?; + Ok(true) +} + +#[derive(serde::Deserialize)] +#[serde(rename_all = "lowercase")] +enum RestIssueState { + Open, + Closed, +} + +fn rest_marker_children( + gh: &Path, + root: &Path, + slug: &str, + parent: u32, + query: &str, +) -> Result> { + let query_arg = format!("q={query} repo:{slug} is:issue"); + let per_page = format!("per_page={CHILD_PAGE}"); + let marker = parent_marker(parent); + let mut first_total = None; + let mut seen = std::collections::HashSet::new(); + let mut rows = Vec::new(); + for page in 1..=MARKER_LIMIT.div_ceil(CHILD_PAGE) { + let page_arg = format!("page={page}"); + // The scoped query carries its repo in q, not in the REST path. + // Name the target explicitly so the normal reader routing sees it. + let read = gh_call::read("parent_link.marker_children_rest", gh, root) + .reader_repo(slug) + .args([ + "api", + "search/issues", + "--method", + "GET", + "-f", + &query_arg, + "-F", + &per_page, + "-F", + &page_arg, + ]); + let reply = gh_call::ok_stdout(read) + .ok_or_else(|| anyhow::anyhow!("could not search REST marker children of #{parent}"))?; + let malformed = || anyhow::anyhow!("malformed REST marker-child list for #{parent}"); + let response: RestMarkerPage = serde_json::from_slice(&reply).map_err(|_| malformed())?; + if response.incomplete_results + || response.total_count >= MARKER_LIMIT + || first_total.is_some_and(|total| total != response.total_count) + || response.items.len() + != response + .total_count + .saturating_sub(seen.len()) + .min(CHILD_PAGE) + { + anyhow::bail!("REST marker-child search of #{parent} may be truncated"); + } + for item in response.items { + let scoped = item + .repository_url + .rsplit_once("/repos/") + .is_some_and(|(_, repo)| repo.eq_ignore_ascii_case(slug)); + if item.number == 0 || !scoped || item.has_pull_request || !seen.insert(item.number) { + return Err(malformed()); + } + if item.body.is_null() { + continue; + } + if item.body.as_str().ok_or_else(malformed)?.contains(&marker) { + rows.push((item.number, matches!(item.state, RestIssueState::Closed))); + } + } + if seen.len() == response.total_count { + return Ok(rows); + } + first_total = Some(response.total_count); + } + anyhow::bail!("REST marker-child search of #{parent} may be truncated"); +} + #[cfg(test)] mod tests { use super::*; @@ -314,6 +422,133 @@ mod tests { assert!(parse_marker_output("RAW x\n", 1).is_err()); } + #[cfg(unix)] + fn mocked_split( + native: &str, + graphql_diagnostic: &str, + pages: &[serde_json::Value], + ) -> Result { + use std::os::unix::fs::PermissionsExt; + let dir = tempfile::tempdir()?; + let gh = dir.path().join("gh-marker-fixture.sh"); + let script = r#"#!/bin/sh +case "$1 $2" in + "api repos/o/r/issues/12/sub_issues?per_page=100&state=all") cat native ;; + "issue list") cat diagnostic >&2; exit 1 ;; + "api search/issues") + test "$3" = "--method" && test "$4" = "GET" && + test "$6" = 'q="loom:parent #12" in:body repo:o/r is:issue' && + test "$8" = "per_page=100" || exit 76 + if test -f "${10}"; then cat "${10}"; else exit 1; fi + ;; + *) exit 76 ;; +esac"#; + std::fs::write(&gh, script)?; + std::fs::set_permissions(&gh, std::fs::Permissions::from_mode(0o700))?; + std::fs::write(dir.path().join("native"), native)?; + std::fs::write(dir.path().join("diagnostic"), graphql_diagnostic)?; + for (index, page) in pages.iter().enumerate() { + std::fs::write(dir.path().join(format!("page={}", index + 1)), page.to_string())?; + } + split_state_with_gh(&gh, dir.path(), "o/r", 12) + } + + #[cfg(unix)] + const GRAPHQL_EXHAUSTED: &str = + "GraphQL: API rate limit already exceeded for installation ID 1"; + + #[cfg(unix)] + fn marker_row(number: u32, state: &str, body: serde_json::Value) -> serde_json::Value { + serde_json::json!({ + "number": number, "state": state, "body": body, + "repository_url": "https://api.github.com/repos/o/r" + }) + } + + #[cfg(unix)] + fn marker_page(total: usize, items: Vec) -> serde_json::Value { + serde_json::json!({"total_count": total, "incomplete_results": false, "items": items}) + } + + #[cfg(unix)] + #[test] + fn rest_marker_search_preserves_the_exact_union_and_filters_self_links() { + assert_eq!( + mocked_split("", GRAPHQL_EXHAUSTED, &[marker_page(0, Vec::new())]).unwrap(), + SplitState::NoChildren, + ); + let closed = marker_row(7, "closed", parent_marker(12).into()); + let matches = marker_page( + 4, + vec![ + closed.clone(), + marker_row(8, "open", parent_marker(123).into()), + marker_row(12, "open", parent_marker(12).into()), + marker_row(99, "open", serde_json::Value::Null), + ], + ); + assert_eq!( + mocked_split("7 CLOSED\n9 CLOSED\n", GRAPHQL_EXHAUSTED, &[matches]).unwrap(), + SplitState::AllClosed(vec![7, 9]) + ); + let open = marker_page(1, vec![marker_row(7, "open", parent_marker(12).into())]); + assert_eq!( + mocked_split("7 CLOSED\n9 CLOSED\n", GRAPHQL_EXHAUSTED, &[open]).unwrap(), + SplitState::OpenChildren(1) + ); + } + + #[cfg(unix)] + #[test] + fn rest_marker_search_checks_both_pages_before_classifying() { + let first = marker_page( + CHILD_PAGE + 1, + (1..=100) + .map(|n| marker_row(n, "open", serde_json::Value::Null)) + .collect(), + ); + let second = marker_page(101, vec![marker_row(101, "open", parent_marker(12).into())]); + assert_eq!( + mocked_split("", GRAPHQL_EXHAUSTED, &[first.clone(), second]).unwrap(), + SplitState::OpenChildren(1) + ); + assert!(mocked_split("", GRAPHQL_EXHAUSTED, &[first.clone()]).is_err()); + let duplicate = marker_page(101, vec![marker_row(100, "open", parent_marker(12).into())]); + assert!(mocked_split("", GRAPHQL_EXHAUSTED, &[first, duplicate]).is_err()); + } + + #[cfg(unix)] + #[test] + fn rest_marker_search_fails_closed_on_missing_or_truncated_evidence() { + let mut wrong_repo = marker_row(7, "open", parent_marker(12).into()); + wrong_repo["repository_url"] = "https://api.github.com/repos/other/r".into(); + let mut malformed = marker_row(7, "open", parent_marker(12).into()); + malformed.as_object_mut().unwrap().remove("body"); + let mut wrong_kind = marker_row(7, "open", parent_marker(12).into()); + wrong_kind["pull_request"] = serde_json::json!({}); + let mut malformed_kind = wrong_kind.clone(); + malformed_kind["pull_request"] = serde_json::Value::Null; + for page in [ + marker_page(MARKER_LIMIT, Vec::new()), + marker_page(1, Vec::new()), + marker_page(1, vec![wrong_repo]), + marker_page(1, vec![malformed]), + marker_page(1, vec![wrong_kind]), + marker_page(1, vec![malformed_kind]), + serde_json::json!({"total_count": 0, "incomplete_results": true, "items": []}), + serde_json::json!({"total_count": 0, "items": []}), + ] { + assert!(mocked_split("7 CLOSED\n", GRAPHQL_EXHAUSTED, &[page]).is_err()); + } + assert!(mocked_split("7 CLOSED\n", GRAPHQL_EXHAUSTED, &[]).is_err()); + assert!(mocked_split( + "", + "HTTP 403: Resource not accessible by integration", + &[marker_page(0, Vec::new())], + ) + .is_err()); + } + #[test] fn body_gets_the_parent_marker() { let body = child_body("Do the thing.", 7, None); diff --git a/loom-daemon/src/release_resolve/resolve/tests.rs b/loom-daemon/src/release_resolve/resolve/tests.rs index 7446de9833..2d54299eca 100644 --- a/loom-daemon/src/release_resolve/resolve/tests.rs +++ b/loom-daemon/src/release_resolve/resolve/tests.rs @@ -844,6 +844,7 @@ fn secondary_limits_server_errors_and_shed_reads_are_unavailable() { } let shed: Query = Query::Unavailable(Unavailable::Shed { until: std::time::SystemTime::now(), + graphql_budget_exhausted: false, }); assert!(is_unavailable(&shed)); } From 2cd5eecc54848ed3d2f0ca8c5c7ba1558d403690 Mon Sep 17 00:00:00 2001 From: Robb Walters Date: Fri, 20 Nov 2026 03:04:35 -0800 Subject: [PATCH 2/3] Require primary GraphQL proof for admission fallback --- loom-daemon/src/ci_telemetry/api.rs | 1 + loom-daemon/src/forge_gateway.rs | 33 ++- loom-daemon/src/forge_identity/route.rs | 26 +- loom-daemon/src/forge_identity/withdrawal.rs | 44 ++- .../src/forge_identity/withdrawal_tests.rs | 75 +++++ loom-daemon/src/forge_read_pool.rs | 136 ++++++--- .../src/gh_invocation/reader_route_v2.rs | 13 +- .../gh_invocation/reader_route_v2_tests.rs | 261 +++++++++++++++++- loom-daemon/src/parent_link.rs | 2 +- 9 files changed, 534 insertions(+), 57 deletions(-) diff --git a/loom-daemon/src/ci_telemetry/api.rs b/loom-daemon/src/ci_telemetry/api.rs index 0426224b2e..ee206cf8ce 100644 --- a/loom-daemon/src/ci_telemetry/api.rs +++ b/loom-daemon/src/ci_telemetry/api.rs @@ -387,6 +387,7 @@ pub(crate) fn api_failure( secondary, retry_after: retry_after_secs.map(Duration::from_secs), remaining: None, + graphql_primary_message: false, }) } Err(ApiError::Http { status: 401, .. }) => Some(Failure::Credential), diff --git a/loom-daemon/src/forge_gateway.rs b/loom-daemon/src/forge_gateway.rs index 5c3ec217cc..433020d9a0 100644 --- a/loom-daemon/src/forge_gateway.rs +++ b/loom-daemon/src/forge_gateway.rs @@ -513,10 +513,35 @@ pub fn route_read(req: &RouteRequest<'_>, now: SystemTime) -> RouteDecision { } let pool = crate::forge_identity::route_read(req, now); let owner = crate::credential_preflight::owner_of_nwo(req.owner_repo).to_ascii_lowercase(); - let gateway_until = if owner.is_empty() || profile_for(&owner).is_none() { - None - } else { - withdrawn_until(&owner, req, now) + let gateway_configured = !owner.is_empty() && profile_for(&owner).is_some(); + let gateway_until = gateway_configured + .then(|| withdrawn_until(&owner, req, now)) + .flatten(); + let pool = match pool { + // The gateway was part of this route. A non-primary gateway hold + // must veto the stronger proof from the readers behind it. + RouteDecision::Exhausted { + until, + cause: ExhaustCause::GraphqlPrimaryBudget, + } if gateway_configured + && (crate::forge_read_pool::is_withdrawn_at(GATEWAY_APP_ID, now) + || crate::forge_identity::repo_withdrawn_at( + GATEWAY_APP_ID, + req.owner_repo, + now, + ) + || !crate::forge_identity::route::primary_graphql_withdrawn_at( + GATEWAY_APP_ID, + &owner, + now, + )) => + { + RouteDecision::Exhausted { + until, + cause: ExhaustCause::Budget, + } + } + decision => decision, }; match past_withdrawn_gateway(pool, gateway_until) { RouteDecision::NoPool if cli_read_route_required() => RouteDecision::Exhausted { diff --git a/loom-daemon/src/forge_identity/route.rs b/loom-daemon/src/forge_identity/route.rs index 0784bfe76c..f3b3cd87c4 100644 --- a/loom-daemon/src/forge_identity/route.rs +++ b/loom-daemon/src/forge_identity/route.rs @@ -182,6 +182,9 @@ pub enum ExhaustCause { /// above the headroom reserve. At least one such reader exists and no /// reader is excluded for any other reason. Budget, + /// A GraphQL gate whose readers are all held by proven primary limits. + /// This carries evidence for a reader-only fallback, with the same budget log label. + GraphqlPrimaryBudget, /// Every reader that could serve the repo is withdrawn or out of budget, /// and at least one withdrawal is not a rate limit (App-wide, a refused /// credential, the gateway's outage withdrawal). Every exclusion ends on @@ -196,7 +199,7 @@ impl ExhaustCause { #[must_use] pub fn as_str(self) -> &'static str { match self { - Self::Budget => "budget", + Self::Budget | Self::GraphqlPrimaryBudget => "budget", Self::Withdrawn => "withdrawn", Self::Unavailable => "unavailable", } @@ -784,6 +787,17 @@ fn is_github(host: Option<&str>) -> bool { host.is_none_or(|h| h.eq_ignore_ascii_case("github.com")) } +/// A live primary hold can authorize a fallback only while a newer reading +/// for the same bucket does not contradict it with positive headroom. +pub(crate) fn primary_graphql_withdrawn_at(app_id: &str, owner: &str, now: SystemTime) -> bool { + let Some(observed_at) = forge_read_pool::primary_graphql_observed_at(app_id, owner, now) else { + return false; + }; + !bucket_key(app_id, owner, Resource::Graphql) + .and_then(|key| forge_bucket_book::reading(&key, epoch(now))) + .is_some_and(|r| r.observed_at >= epoch(observed_at) && r.remaining.is_some_and(|n| n > 0)) +} + /// [`route_read`] against an explicit workspace, roster and environment. #[must_use] pub fn route_read_in( @@ -1000,6 +1014,7 @@ pub fn route_read_in( .and_then(|k| forge_bucket_book::projected_used_pct(&k, now_epoch)) }; let (mut budget, mut withdrawn, mut other, mut first_seen) = (0_usize, 0_usize, 0_usize, false); + let mut primary_graphql = 0; for off in 0..n { let r = readers[(start + off) % n]; if forge_read_pool::is_withdrawn_at(&r.app_id, now) { @@ -1013,6 +1028,13 @@ pub fn route_read_in( if forge_read_pool::is_budget_withdrawn_at(&r.app_id, owner, req.resource, now) { budget += 1; first_seen = true; + if req.class == ReadClass::Gate + && req.resource == Resource::Graphql + && primary_graphql_withdrawn_at(&r.app_id, owner, now) + && super::dir_is_fresh(&super::reader_dir(workspace_root, owner, r), now) + { + primary_graphql += 1; + } } else { withdrawn += 1; } @@ -1049,6 +1071,8 @@ pub fn route_read_in( ExhaustCause::Unavailable } else if withdrawn > 0 { ExhaustCause::Withdrawn + } else if primary_graphql == n { + ExhaustCause::GraphqlPrimaryBudget } else { ExhaustCause::Budget }; diff --git a/loom-daemon/src/forge_identity/withdrawal.rs b/loom-daemon/src/forge_identity/withdrawal.rs index bbea35f2f6..b4dd22b779 100644 --- a/loom-daemon/src/forge_identity/withdrawal.rs +++ b/loom-daemon/src/forge_identity/withdrawal.rs @@ -95,6 +95,8 @@ pub enum Failure { /// is GitHub saying the bucket still has budget, so the refusal is /// not an empty pool and is not held until the pool's reset. remaining: Option, + /// The CLI reported an explicit primary GraphQL exhaustion diagnostic. + graphql_primary_message: bool, }, /// The reader's token was refused (401, bad credentials). Credential, @@ -108,6 +110,24 @@ pub enum Failure { } impl Failure { + /// Positively classified primary GraphQL exhaustion with no reported headroom. + pub(crate) fn is_primary_graphql(self) -> bool { + match self { + Self::RateLimited { + resource: Resource::Graphql, + secondary: false, + retry_after: None, + remaining, + graphql_primary_message, + .. + } => match remaining { + Some(n) => n == 0, + None => graphql_primary_message, + }, + _ => false, + } + } + /// A primary rate limit of `resource` with no reset known. #[must_use] pub fn rate_limited(resource: Resource) -> Self { @@ -117,6 +137,7 @@ impl Failure { secondary: false, retry_after: None, remaining: None, + graphql_primary_message: false, } } @@ -140,12 +161,14 @@ impl Failure { secondary, retry_after, remaining, + graphql_primary_message, } => Self::RateLimited { resource, reset, secondary, retry_after, remaining, + graphql_primary_message, }, other => other, } @@ -244,6 +267,12 @@ pub fn classify_failure_in( secondary, retry_after, remaining: headers.and_then(|h| h.remaining), + graphql_primary_message: s.lines().any(|line| { + let line = line.trim_start(); + let line = line.strip_prefix("gh: ").unwrap_or(line); + line.starts_with("graphql: api rate limit already exceeded") + || line.starts_with("graphql: api rate limit exceeded") + }), }); } if credential { @@ -474,12 +503,19 @@ pub fn withdraw_after_in( probe: ProbeReset<'_>, ) -> Withdrawal { let plan = plan_withdrawal(owner_repo, failure, now, mode, app_id, probe); - apply(app_id, owner_repo, failure, &plan, why); + apply(app_id, owner_repo, failure, &plan, why, now); plan } /// Record `plan` in the routing tables, log it, and export it. -fn apply(app_id: &str, owner_repo: &str, failure: Failure, plan: &Withdrawal, why: &str) { +fn apply( + app_id: &str, + owner_repo: &str, + failure: Failure, + plan: &Withdrawal, + why: &str, + now: SystemTime, +) { let secs = |until: &SystemTime| { until .duration_since(SystemTime::now()) @@ -505,7 +541,9 @@ fn apply(app_id: &str, owner_repo: &str, failure: Failure, plan: &Withdrawal, wh // A rate limit (primary or secondary) is a budget withdrawal: the // one cause W4-C may shed a deferrable read on. A refused // credential is not — those reads still fall back to the writer. - let held = if matches!(failure, Failure::RateLimited { .. }) { + let held = if failure.is_primary_graphql() { + forge_read_pool::withdraw_primary_graphql_until(app_id, owner, *until, now) + } else if matches!(failure, Failure::RateLimited { .. }) { forge_read_pool::withdraw_scoped_budget_until(app_id, owner, *scope, *until) } else { forge_read_pool::withdraw_scoped_until(app_id, owner, *scope, *until) diff --git a/loom-daemon/src/forge_identity/withdrawal_tests.rs b/loom-daemon/src/forge_identity/withdrawal_tests.rs index 4a05c91058..34db99a03a 100644 --- a/loom-daemon/src/forge_identity/withdrawal_tests.rs +++ b/loom-daemon/src/forge_identity/withdrawal_tests.rs @@ -64,6 +64,7 @@ fn a_core_rate_limit_withdraws_only_that_owners_core_bucket() { secondary: false, retry_after: None, remaining: None, + graphql_primary_message: false, }; let w = withdraw_after_in( RoutingMode::Scoped, @@ -133,6 +134,7 @@ fn a_header_reset_books_the_bucket_exhausted() { secondary: false, retry_after: None, remaining: None, + graphql_primary_message: false, }; withdraw_after_in(RoutingMode::Scoped, app, "acme/x", failure, "test", now, &no_probe); let key = BucketKey::new("app-940002", "acme", Resource::Graphql); @@ -156,6 +158,7 @@ fn a_refusal_reporting_headroom_does_not_withdraw_the_reader_until_reset() { secondary: false, retry_after: None, remaining: Some(30), + graphql_primary_message: false, }; let w = withdraw_after_in(RoutingMode::Scoped, app, "acme/x", failure, "test", now, &no_probe); let (_, _, until, source, _) = scoped(&w); @@ -183,6 +186,7 @@ fn the_reset_comes_from_the_header_then_the_probe_then_the_default() { secondary: false, retry_after: None, remaining: None, + graphql_primary_message: false, }; plan_withdrawal("acme/x", f, now, RoutingMode::Scoped, "940003", probe) }; @@ -492,6 +496,7 @@ fn a_secondary_withdrawal_covers_every_resource_of_that_owner_only() { secondary: true, retry_after: Some(Duration::from_secs(120)), remaining: None, + graphql_primary_message: false, }; withdraw_after_in(RoutingMode::Scoped, app, "acme/x", f, "test", now, &no_probe); let during = now + Duration::from_secs(60); @@ -552,8 +557,10 @@ fn the_refused_pool_comes_from_the_response_else_the_static_pool() { secondary: false, retry_after: None, remaining: Some(0), + graphql_primary_message: false, } ); + assert!(f.is_primary_graphql()); let f = classify_failure_in(RoutingMode::Scoped, "", Some(429), None, Resource::Search); assert_eq!(f, Some(Failure::rate_limited(Resource::Search))); // A 403 whose headers say the pool is empty is a rate limit, not coverage. @@ -561,6 +568,74 @@ fn the_refused_pool_comes_from_the_response_else_the_static_pool() { assert!(matches!(f, Some(Failure::RateLimited { .. })), "{f:?}"); } +#[test] +fn explicit_primary_graphql_evidence_yields_to_retry_after_and_positive_remaining() { + let classify = |status, headers: Option<&RateLimitHeaders>| { + classify_failure_in( + RoutingMode::Scoped, + "GraphQL: API rate limit already exceeded", + status, + headers, + Resource::Graphql, + ) + .unwrap() + }; + let mut h = headers(None, None); + h.resource = Some("graphql".into()); + assert!(classify(None, None).is_primary_graphql()); + assert!(classify(Some(403), Some(&h)).is_primary_graphql()); + h.retry_after_secs = Some(60); + assert!(!classify(Some(429), Some(&h)).is_primary_graphql()); + h.retry_after_secs = None; + h.remaining = Some(12); + assert!(!classify(Some(403), Some(&h)).is_primary_graphql()); +} + +#[test] +fn a_new_primary_graphql_observation_keeps_its_own_shorter_expiry() { + let app = "940043"; + let owner = "proof-history"; + let repo = "proof-history/example"; + let now = t0(); + let at = |seconds| now + Duration::from_secs(seconds); + let primary = |observed_at, until| { + let failure = Failure::RateLimited { + resource: Resource::Graphql, + reset: Some(until), + secondary: false, + retry_after: None, + remaining: Some(0), + graphql_primary_message: false, + }; + withdraw_after_in(RoutingMode::Scoped, app, repo, failure, "test", observed_at, &no_probe); + }; + let proven = |now| super::super::route::primary_graphql_withdrawn_at(app, owner, now); + + primary(at(0), at(300)); + assert!(proven(at(10))); + forge_bucket_book::insert( + BucketKey::new("app-940043", owner, Resource::Graphql), + forge_bucket_book::Reading { + limit: Some(5000), + remaining: Some(12), + used: None, + reset_epoch: epoch_secs(at(300)), + observed_at: epoch_secs(at(30)), + source: forge_bucket_book::Source::Header, + }, + ); + assert!(!proven(at(30))); + + primary(at(60), at(120)); + assert!(proven(at(100))); + assert!(!proven(at(120))); + primary(at(20), at(500)); // a delayed older observation + primary(at(60), at(240)); // conflicting expiry for the same observation time + assert!(proven(at(100))); + assert!(!proven(at(120))); + assert!(forge_read_pool::is_budget_withdrawn_at(app, owner, Resource::Graphql, at(150))); +} + #[test] fn coverage_and_non_credential_failures_are_unchanged() { for mode in [RoutingMode::Scoped, RoutingMode::Legacy] { diff --git a/loom-daemon/src/forge_read_pool.rs b/loom-daemon/src/forge_read_pool.rs index 4be8ddd838..472656851b 100644 --- a/loom-daemon/src/forge_read_pool.rs +++ b/loom-daemon/src/forge_read_pool.rs @@ -353,42 +353,84 @@ impl ResourceScope { } } -/// `(app id, owner lowercased, scope)` -> the instant it becomes eligible. +/// `(app id, owner lowercased, scope)` -> the current withdrawal and its evidence. type ScopedKey = (String, String, ResourceScope); -fn scoped_withdrawals() -> &'static Mutex> { - static SCOPED: OnceLock>> = OnceLock::new(); +#[derive(Clone, Copy)] +struct PrimaryGraphql { + until: SystemTime, + observed_at: SystemTime, +} + +struct ScopedWithdrawal { + until: SystemTime, + budget_until: Option, + primary_graphql: Option, + other_until: Option, +} + +enum ScopedCause { + Other, + Budget, + PrimaryGraphql { observed_at: SystemTime }, +} + +fn scoped_withdrawals() -> &'static Mutex> { + static SCOPED: OnceLock>> = OnceLock::new(); SCOPED.get_or_init(|| Mutex::new(HashMap::new())) } -/// Withdraw reader `app_id` from `owner`'s `scope` until `until`. Like -/// [`withdraw_until`], an existing withdrawal is only ever extended, never -/// shortened. Returns the instant the withdrawal now ends (the longer of the -/// held and the requested one). -pub fn withdraw_scoped_until( +fn record_scoped_withdrawal( app_id: &str, owner: &str, scope: ResourceScope, until: SystemTime, + cause: ScopedCause, ) -> SystemTime { let Ok(mut map) = scoped_withdrawals().lock() else { return until; // a poisoned lock must not take reads down }; let held = map .entry((app_id.to_string(), owner.to_ascii_lowercase(), scope)) - .or_insert(until); - if until > *held { - *held = until; + .or_insert(ScopedWithdrawal { + until, + budget_until: None, + primary_graphql: None, + other_until: None, + }); + held.until = held.until.max(until); + if !matches!(cause, ScopedCause::Other) { + // Keep the generic budget hold at the effective withdrawal end. + held.budget_until = held.budget_until.max(Some(held.until)); + } + if let ScopedCause::PrimaryGraphql { observed_at } = cause { + let evidence = held + .primary_graphql + .get_or_insert(PrimaryGraphql { until, observed_at }); + // A newer observation may describe a different, shorter window. + // Keep its time and expiry together even while the generic hold stays longer. + if observed_at > evidence.observed_at { + *evidence = PrimaryGraphql { until, observed_at }; + } else if observed_at == evidence.observed_at { + evidence.until = evidence.until.min(until); + } + } else { + held.other_until = held.other_until.max(Some(until)); } - *held + held.until } -/// The scoped withdrawals that are about **budget** (a primary or secondary -/// rate limit), as opposed to a refused credential. Same key and end as the -/// entry [`withdraw_scoped_budget_until`] also writes to the scoped table. -fn budget_withdrawals() -> &'static Mutex> { - static BUDGET: OnceLock>> = OnceLock::new(); - BUDGET.get_or_init(|| Mutex::new(HashMap::new())) +/// Withdraw reader `app_id` from `owner`'s `scope` until `until`. Like +/// [`withdraw_until`], an existing withdrawal is only ever extended, never +/// shortened. Returns the instant the withdrawal now ends (the longer of the +/// held and the requested one). +pub fn withdraw_scoped_until( + app_id: &str, + owner: &str, + scope: ResourceScope, + until: SystemTime, +) -> SystemTime { + record_scoped_withdrawal(app_id, owner, scope, until, ScopedCause::Other) } /// [`withdraw_scoped_until`] for a rate limit: the reader is out of @@ -401,16 +443,43 @@ pub fn withdraw_scoped_budget_until( scope: ResourceScope, until: SystemTime, ) -> SystemTime { - let held = withdraw_scoped_until(app_id, owner, scope, until); - if let Ok(mut map) = budget_withdrawals().lock() { - let slot = map - .entry((app_id.to_string(), owner.to_ascii_lowercase(), scope)) - .or_insert(held); - if held > *slot { - *slot = held; - } + record_scoped_withdrawal(app_id, owner, scope, until, ScopedCause::Budget) +} + +/// Record a positively observed primary GraphQL refusal. Its proof expires at +/// its own end, even when other withdrawals keep the bucket ineligible longer. +pub(crate) fn withdraw_primary_graphql_until( + app_id: &str, + owner: &str, + until: SystemTime, + observed_at: SystemTime, +) -> SystemTime { + record_scoped_withdrawal( + app_id, + owner, + ResourceScope::Graphql, + until, + ScopedCause::PrimaryGraphql { observed_at }, + ) +} + +/// A live primary-only proof for this installation. An overlapping All-scope +/// or unqualified GraphQL hold prevents the narrower fallback authorization. +pub(crate) fn primary_graphql_observed_at( + app_id: &str, + owner: &str, + now: SystemTime, +) -> Option { + let map = scoped_withdrawals().lock().ok()?; + let mut key = (app_id.to_string(), owner.to_ascii_lowercase(), ResourceScope::All); + if map.get(&key).is_some_and(|held| now < held.until) { + return None; } - held + key.2 = ResourceScope::Graphql; + let held = map.get(&key)?; + let proof = held.primary_graphql?; + (now < proof.until && !held.other_until.is_some_and(|until| now < until)) + .then_some(proof.observed_at) } /// Whether reader `app_id` is withdrawn from `owner`'s `resource` at `now` @@ -423,7 +492,7 @@ pub fn is_budget_withdrawn_at( resource: Resource, now: SystemTime, ) -> bool { - let Ok(map) = budget_withdrawals().lock() else { + let Ok(map) = scoped_withdrawals().lock() else { return false; // unknown cause: never shed on it }; let owner = owner.to_ascii_lowercase(); @@ -431,7 +500,8 @@ pub fn is_budget_withdrawn_at( .into_iter() .any(|scope| { map.get(&(app_id.to_string(), owner.clone(), scope)) - .is_some_and(|&until| now < until) + .and_then(|held| held.budget_until) + .is_some_and(|until| now < until) }) } @@ -452,7 +522,7 @@ pub fn is_withdrawn_scoped_at( .into_iter() .any(|scope| { map.get(&(app_id.to_string(), owner.clone(), scope)) - .is_some_and(|&until| now < until) + .is_some_and(|held| now < held.until) }) } @@ -481,7 +551,7 @@ pub fn scoped_withdrawal_until( .into_iter() .filter_map(|scope| { map.get(&(app_id.to_string(), owner.clone(), scope)) - .copied() + .map(|held| held.until) }) .filter(|&until| now < until) .max() @@ -498,8 +568,8 @@ pub fn live_scoped_withdrawals( }; let mut out: Vec<_> = map .iter() - .filter(|(_, &until)| now < until) - .map(|((app, owner, scope), &until)| (app.clone(), owner.clone(), *scope, until)) + .filter(|(_, held)| now < held.until) + .map(|((app, owner, scope), held)| (app.clone(), owner.clone(), *scope, held.until)) .collect(); out.sort_by(|a, b| (&a.0, &a.1, a.2).cmp(&(&b.0, &b.1, b.2))); out diff --git a/loom-daemon/src/gh_invocation/reader_route_v2.rs b/loom-daemon/src/gh_invocation/reader_route_v2.rs index 98336796a4..1a1f6453a8 100644 --- a/loom-daemon/src/gh_invocation/reader_route_v2.rs +++ b/loom-daemon/src/gh_invocation/reader_route_v2.rs @@ -181,7 +181,7 @@ impl GhInvocation { )); } RouteDecision::Exhausted { until, cause } => { - let why = if cause == ExhaustCause::Budget + let why = if cause == ExhaustCause::GraphqlPrimaryBudget && class == ReadClass::Gate && resource == Resource::Graphql && primary_graphql_failures_only @@ -240,6 +240,7 @@ impl GhInvocation { if fault == crate::forge_gateway::outage::Fault::Withdraw { super::withdraw_gateway_outage(self.operation.as_str()); } + primary_graphql_failures_only = false; tried.insert(app_id.clone()); last_app = Some(app_id); last_failure = None; @@ -287,15 +288,7 @@ impl GhInvocation { // A rate limit, a refused token, or a 403/404 from a reader with // no evidence it can see the repo: withdraw it (for this repo // only on coverage) and ask the router for the next one. - primary_graphql_failures_only &= matches!( - failure, - Failure::RateLimited { - resource: Resource::Graphql, - secondary: false, - remaining: None | Some(0), - .. - } - ); + primary_graphql_failures_only &= failure.is_primary_graphql(); withdraw(&app_id, &slug, failure, &why); tried.insert(app_id.clone()); last_app = Some(app_id); diff --git a/loom-daemon/src/gh_invocation/reader_route_v2_tests.rs b/loom-daemon/src/gh_invocation/reader_route_v2_tests.rs index 5e84b46dce..156d196970 100644 --- a/loom-daemon/src/gh_invocation/reader_route_v2_tests.rs +++ b/loom-daemon/src/gh_invocation/reader_route_v2_tests.rs @@ -220,31 +220,62 @@ fn a_gate_read_is_shed_like_any_other_and_an_unclassified_read_is_gate() { fn only_primary_graphql_gate_exhaustion_is_preserved_for_a_rest_fallback() { const PRIMARY: &str = "echo 'GraphQL: API rate limit already exceeded' >&2; exit 1"; const SECONDARY: &str = "echo 'GraphQL: You have exceeded a secondary rate limit' >&2; exit 1"; + const GENERIC_429: &str = "echo 'gh: HTTP 429' >&2; exit 1"; for (label, reader, class, cause, pre, expected) in [ ( - "already exhausted", + "unqualified budget", PRIMARY, ReadClass::Gate, ExhaustCause::Budget, &["app-1", "app-2"][..], + false, + ), + ( + "established primary", + PRIMARY, + ReadClass::Gate, + ExhaustCause::GraphqlPrimaryBudget, + &["app-1", "app-2"][..], true, ), ( "observed primary", PRIMARY, ReadClass::Gate, - ExhaustCause::Budget, + ExhaustCause::GraphqlPrimaryBudget, &[][..], true, ), - ("permission", NOT_FOUND, ReadClass::Gate, ExhaustCause::Budget, &[][..], false), - ("secondary", SECONDARY, ReadClass::Gate, ExhaustCause::Budget, &[][..], false), + ( + "permission", + NOT_FOUND, + ReadClass::Gate, + ExhaustCause::GraphqlPrimaryBudget, + &[][..], + false, + ), + ( + "secondary", + SECONDARY, + ReadClass::Gate, + ExhaustCause::GraphqlPrimaryBudget, + &[][..], + false, + ), + ( + "unqualified 429", + GENERIC_429, + ReadClass::Gate, + ExhaustCause::GraphqlPrimaryBudget, + &[][..], + false, + ), ("withdrawn", PRIMARY, ReadClass::Gate, ExhaustCause::Withdrawn, &[][..], false), ( "headroom reserve", PRIMARY, ReadClass::Hygiene, - ExhaustCause::Budget, + ExhaustCause::GraphqlPrimaryBudget, &[][..], false, ), @@ -283,6 +314,226 @@ fn only_primary_graphql_gate_exhaustion_is_preserved_for_a_rest_fallback() { } } +#[test] +fn real_scoped_withdrawals_require_primary_only_graphql_evidence() { + use crate::forge_bucket_book::{self, BucketKey, Reading, Source}; + use crate::forge_identity::{self, route, Identity, Roster, RoutingMode}; + use crate::forge_read_pool::{self, ResourceScope}; + + enum Prior { + None, + LongerGeneric, + Coverage, + PositiveReading(i64), + } + + let pool = pool("exit 76", "exit 76", "exit 76"); + let start = SystemTime::now(); + let primary_for = |seconds| Failure::RateLimited { + resource: Resource::Graphql, + reset: Some(start + Duration::from_secs(seconds)), + secondary: false, + retry_after: None, + remaining: Some(0), + graphql_primary_message: false, + }; + let primary = primary_for(120); + let secondary = Failure::RateLimited { + resource: Resource::Graphql, + reset: None, + secondary: true, + retry_after: Some(Duration::from_secs(10)), + remaining: None, + graphql_primary_message: false, + }; + let headroom = Failure::RateLimited { + resource: Resource::Graphql, + reset: None, + secondary: false, + retry_after: None, + remaining: Some(12), + graphql_primary_message: false, + }; + let classify = |stderr, status| { + forge_identity::classify_failure_in( + RoutingMode::Scoped, + stderr, + status, + None, + Resource::Graphql, + ) + .unwrap() + }; + let unqualified_429 = classify("gh: HTTP 429", Some(429)); + let explicit_primary = classify("GraphQL: API rate limit already exceeded", None); + let primary_long = primary_for(300); + for (label, first, second, prior, at, expected) in [ + ("primary", vec![primary], primary, Prior::None, 0, true), + ("explicit primary", vec![explicit_primary], primary, Prior::None, 0, true), + ("unqualified 429", vec![unqualified_429], primary, Prior::None, 0, false), + ("secondary", vec![secondary], secondary, Prior::None, 0, false), + ("headroom", vec![headroom], primary, Prior::None, 0, false), + ("same reader headroom", vec![primary, headroom], primary, Prior::None, 0, false), + ( + "same reader secondary", + vec![primary, secondary], + primary, + Prior::None, + 0, + false, + ), + ( + "same reader credential", + vec![primary, Failure::Credential], + primary, + Prior::None, + 0, + false, + ), + ("secondary expired", vec![primary, secondary], primary, Prior::None, 11, true), + ( + "older positive reading", + vec![primary], + primary, + Prior::PositiveReading(-1), + 0, + true, + ), + ( + "newer positive reading", + vec![primary], + primary, + Prior::PositiveReading(1), + 1, + false, + ), + ( + "generic hold outlives proof", + vec![primary], + primary_long, + Prior::LongerGeneric, + 150, + false, + ), + ("coverage is not proof", vec![primary], primary, Prior::Coverage, 0, false), + ] { + let slug = format!("proof-{at}-{label}/admission").replace(' ', "-"); + let owner = slug.split_once('/').unwrap().0; + let readers: Vec<_> = ["90147", "90148"] + .into_iter() + .map(|app_id| Identity { + app_id: app_id.to_owned(), + slug: None, + private_key_path: pool._tmp.path().join("unused.pem"), + owners: None, + }) + .collect(); + let first_app = &readers[0].app_id; + match prior { + Prior::None => {} + Prior::LongerGeneric => { + forge_read_pool::withdraw_scoped_budget_until( + first_app, + owner, + ResourceScope::Graphql, + start + Duration::from_secs(300), + ); + } + Prior::Coverage => { + forge_identity::withdraw_reader_for_repo_until( + first_app, + &slug, + start + Duration::from_secs(300), + ); + } + Prior::PositiveReading(offset) => { + let seen = i64::try_from( + start + .duration_since(SystemTime::UNIX_EPOCH) + .unwrap() + .as_secs(), + ) + .unwrap() + + offset; + forge_bucket_book::insert( + BucketKey::new(&format!("app-{first_app}"), owner, Resource::Graphql), + Reading { + limit: Some(5000), + remaining: Some(12), + used: None, + reset_epoch: seen + 300, + observed_at: seen, + source: Source::Header, + }, + ); + } + } + let expires = (chrono::Utc::now() + chrono::Duration::minutes(50)).to_rfc3339(); + for (reader, failures) in readers.iter().zip([first, vec![second]]) { + let dir = forge_identity::reader_dir(pool._tmp.path(), owner, reader); + std::fs::create_dir_all(&dir).unwrap(); + std::fs::write(dir.join("hosts.yml"), "{}\n").unwrap(); + forge_identity::sidecar::write_sidecar( + &dir, + &forge_identity::Sidecar { + app_id: reader.app_id.clone(), + expires_at: expires.clone(), + installation_id: "1".to_owned(), + owner: Some(owner.to_owned()), + ..forge_identity::Sidecar::default() + }, + ) + .unwrap(); + for failure in failures { + forge_identity::withdraw_after_in( + RoutingMode::Scoped, + &reader.app_id, + &slug, + failure, + "fixture", + start, + &|_, _, _| None, + ); + } + } + let roster = Roster { + readers, + ..Roster::default() + }; + let cfg = route::RoutingConfig::default(); + let env = route::RouteEnv { + mode: RoutingMode::Scoped, + cfg: &cfg, + }; + let now = start + Duration::from_secs(at); + let output = GhInvocation::new( + Operation::new("parent_link.marker_children"), + AccessIntent::Read, + GhTarget::None, + Duration::from_secs(10), + ) + .program(&pool.gh) + .reader_only() + .reader_repo(&slug) + .args(["issue", "list", "--repo", &slug]) + .execute_routed_v2_at( + &|req| route::route_read_in(pool._tmp.path(), &roster, req, &env, now), + &|_, _, _, _| panic!("fixture should never execute gh"), + now, + ) + .unwrap(); + assert!(pool.who().is_empty(), "{label}"); + let GhCompletion::Shed { + graphql_budget_exhausted, + .. + } = output + else { + panic!("{label}: expected a shed, got {output:?}"); + }; + assert_eq!(graphql_budget_exhausted, expected, "{label}"); + } +} + #[test] fn repo_scoped_rest_search_uses_the_intended_reader() { let p = pool(OK, OK, "exit 76"); diff --git a/loom-daemon/src/parent_link.rs b/loom-daemon/src/parent_link.rs index 3beb3f9283..c5a8ef2151 100644 --- a/loom-daemon/src/parent_link.rs +++ b/loom-daemon/src/parent_link.rs @@ -512,7 +512,7 @@ esac"#; mocked_split("", GRAPHQL_EXHAUSTED, &[first.clone(), second]).unwrap(), SplitState::OpenChildren(1) ); - assert!(mocked_split("", GRAPHQL_EXHAUSTED, &[first.clone()]).is_err()); + assert!(mocked_split("", GRAPHQL_EXHAUSTED, std::slice::from_ref(&first)).is_err()); let duplicate = marker_page(101, vec![marker_row(100, "open", parent_marker(12).into())]); assert!(mocked_split("", GRAPHQL_EXHAUSTED, &[first, duplicate]).is_err()); } From 176ed8ff4937043b64e18dfb369f3bd715a00560 Mon Sep 17 00:00:00 2001 From: Robb Walters Date: Fri, 20 Nov 2026 03:26:07 -0800 Subject: [PATCH 3/3] Review the synthetic reader route fixture in the test inventory --- loom-daemon/src/forge_identity/tests.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/loom-daemon/src/forge_identity/tests.rs b/loom-daemon/src/forge_identity/tests.rs index 010071f879..a9f1bffa5d 100644 --- a/loom-daemon/src/forge_identity/tests.rs +++ b/loom-daemon/src/forge_identity/tests.rs @@ -351,6 +351,8 @@ fn only_reviewed_read_paths_request_reader_credentials() { // W4-C: the choke point's class-aware chain (the same reviewed // routing step, with the router injected). "gh_invocation/reader_route_v2.rs", + // The test-only route regression uses temporary reader directories and an inert gh. + "gh_invocation/reader_route_v2_tests.rs", // W7: the ETag store's W4-C deferrable chain (`ConditionalRead:: // deferrable`), the same reviewed routing step with the router injected. "forge_etag_store/deferrable.rs",