Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions loom-daemon/src/ci_telemetry/api.rs
Original file line number Diff line number Diff line change
Expand Up @@ -387,6 +387,7 @@ pub(crate) fn api_failure(
secondary,
retry_after: retry_after_secs.map(Duration::from_secs),
remaining: None,
graphql_primary_message: false,
})
}
Err(ApiError::Http { status: 401, .. }) => Some(Failure::Credential),
Expand Down
26 changes: 25 additions & 1 deletion loom-daemon/src/claim_reconciliation/gh_call.rs
Original file line number Diff line number Diff line change
Expand Up @@ -109,7 +109,9 @@ pub(crate) fn forge_op_for(op: &str) -> Option<ForgeOp> {
"park_hold.issue_view"
| "release_veto.issue_labels"
| "intake.recheck"
| "landed.issue_labels" => ops::ISSUE_VIEW_STATE,
| "landed.issue_labels"
| "claim.labels_rest" => ops::ISSUE_VIEW_STATE,
"parent_link.marker_children_rest" => ops::ISSUE_SEARCH,
"park_hold.issue_body" => ops::ISSUE_EDIT_BODY,
"quarantine.issue_list" => ops::ISSUE_LIST,
"claim.issue_reclaim"
Expand Down Expand Up @@ -210,3 +212,25 @@ pub(crate) fn ok_stdout(inv: GhInvocation) -> Option<Vec<u8>> {
pub(crate) fn stderr(out: &Output) -> String {
String::from_utf8_lossy(&out.stderr).trim().to_string()
}

/// Run a known GraphQL admission read. `None` specifically means its primary
/// GraphQL budget is exhausted, whether reported by gh or by the reader router.
pub(crate) fn graphql_output(inv: GhInvocation) -> Result<Option<Output>> {
let op = inv.operation().as_str();
match inv.run() {
CmdOutcome::Ran(out) if graphql_budget_exhausted(&out) => Ok(None),
CmdOutcome::Ran(out) => Ok(Some(out)),
CmdOutcome::Unavailable(crate::cmd_out::Unavailable::Shed {
graphql_budget_exhausted: true,
..
}) => Ok(None),
CmdOutcome::Unavailable(why) => Err(anyhow!("failed to invoke gh ({op}): {why}")),
}
}

fn graphql_budget_exhausted(out: &Output) -> bool {
let diagnostic = String::from_utf8_lossy(&out.stderr);
!out.status.success()
&& crate::rate_limit_breaker::indicates_rate_limit(&diagnostic)
&& !crate::rate_limit_breaker::evidence::is_secondary_limit(&diagnostic)
}
7 changes: 5 additions & 2 deletions loom-daemon/src/cmd_out.rs
Original file line number Diff line number Diff line change
Expand Up @@ -107,7 +107,10 @@ pub enum Unavailable {
/// exhausted until `until`. Nothing was sent. Its text deliberately
/// matches no rate-limit signature, so feeding it to the rate-limit
/// breaker can never trip it.
Shed { until: std::time::SystemTime },
Shed {
until: std::time::SystemTime,
graphql_budget_exhausted: bool,
},
/// Not run (#12641): a forge read the work-finder tick stopped (its
/// deadline passed, or its discovery was ended early). Nothing was sent
/// and the reader budget plays no part, so it is deliberately not
Expand All @@ -125,7 +128,7 @@ impl std::fmt::Display for Unavailable {
Unavailable::RoutingBlocked(e) => write!(f, "routing_blocked: {e}"),
Unavailable::AdapterUnavailable(e) => write!(f, "adapter_unavailable: {e}"),
Unavailable::DeadlineReached => write!(f, "deferred: tick deadline reached"),
Unavailable::Shed { until } => write!(
Unavailable::Shed { until, .. } => write!(
f,
"deferred: reader budget low until {} (loom-shed)",
chrono::DateTime::<chrono::Utc>::from(*until)
Expand Down
170 changes: 162 additions & 8 deletions loom-daemon/src/forge_check_claim.rs
Original file line number Diff line number Diff line change
Expand Up @@ -712,12 +712,14 @@ pub(crate) fn read_claim_labels_for(
"labels,state,url",
])
.args(crate::claim_reconciliation::gh_call::loom_repo_flag());
let Ok(out) = crate::claim_reconciliation::gh_call::output(inv) else {
return LabelLeg::Unknown;
let out = match crate::claim_reconciliation::gh_call::graphql_output(inv) {
Ok(Some(out)) if out.status.success() => out,
Ok(None) => {
return read_claim_labels_rest(gh_bin, root, issue, claim_owned)
.unwrap_or(LabelLeg::Unknown);
}
_ => return LabelLeg::Unknown,
};
if !out.status.success() {
return LabelLeg::Unknown;
}
// `gh issue view --json labels,state,url` emits
// `{"labels":[{"name":"..."},…],"state":"OPEN","url":"https://…/issues/42"}`.
let Ok(parsed) = serde_json::from_slice::<serde_json::Value>(&out.stdout) else {
Expand Down Expand Up @@ -746,13 +748,56 @@ pub(crate) fn read_claim_labels_for(
.iter()
.filter_map(|l| l.get("name").and_then(|n| n.as_str()).map(String::from))
.collect();
if crate::work_finder::is_frozen(&labels) {
classify_claim_labels(&labels, claim_owned)
}

fn read_claim_labels_rest(
gh_bin: &Path,
root: &Path,
issue: u32,
claim_owned: bool,
) -> Option<LabelLeg> {
use crate::claim_reconciliation::gh_call;
// The facade applies LOOM_REPO as GH_REPO for the REST placeholders,
// just as the GraphQL call above applies it through --repo.
let path = format!("repos/{{owner}}/{{repo}}/issues/{issue}");
let read = gh_call::read("claim.labels_rest", gh_bin, root).args(["api", &path]);
let json: serde_json::Value = serde_json::from_slice(&gh_call::ok_stdout(read)?).ok()?;
if json.get("number")?.as_u64()? != u64::from(issue) {
return None;
}
let pull_request = json.get("pull_request");
if let Some(pr) = pull_request {
if pr.is_object() {
return Some(LabelLeg::IsPullRequest);
}
return None;
}
match json.get("state")?.as_str()? {
"closed" => return Some(LabelLeg::Closed),
"open" => {}
_ => return None,
}
let labels: Vec<String> = json
.get("labels")?
.as_array()?
.iter()
.map(|label| {
let name = label.get("name")?.as_str()?;
(!name.is_empty()).then(|| name.to_owned())
})
.collect::<Option<_>>()?;
Some(classify_claim_labels(&labels, claim_owned))
}

fn classify_claim_labels(labels: &[String], claim_owned: bool) -> LabelLeg {
if crate::work_finder::is_frozen(labels) {
return LabelLeg::Frozen;
}
if crate::work_finder::is_operator_hold(&labels) {
if crate::work_finder::is_operator_hold(labels) {
return LabelLeg::OperatorHold;
}
let claims = claim_labels_in(&labels);
let claims = claim_labels_in(labels);
let triage = labels
.iter()
.any(|l| l == crate::intake_reconcile::TRIAGE_LABEL);
Expand Down Expand Up @@ -1840,6 +1885,115 @@ mod tests {
assert_eq!(read_claim_labels(&garbage, dir.path(), 42), LabelLeg::Unknown);
}

fn claim_label_fallback(
directory: &Path,
diagnostic: &str,
rest_reply: &str,
claim_owned: bool,
) -> LabelLeg {
let script = format!(
r#"case "$1 $2" in
"issue view") printf '%s\n' '{diagnostic}' >&2; exit 1 ;;
"api repos/{{owner}}/{{repo}}/issues/42") printf '%s\n' '{rest_reply}' ;;
*) exit 76 ;;
esac"#,
);
let gh = write_exec(directory, "gh-label-fallback.sh", &script);
read_claim_labels_for(&gh, directory, 42, claim_owned)
}

const GRAPHQL_EXHAUSTED: &str =
"GraphQL: API rate limit already exceeded for installation ID 1";

#[test]
fn exhausted_graphql_labels_use_the_same_claim_predicates_via_rest() {
let dir = tempdir().unwrap();
let rest = |payload: &str, owned| {
claim_label_fallback(dir.path(), GRAPHQL_EXHAUSTED, payload, owned)
};
for (payload, expected) in [
(
r#"{"number":42,"state":"open","labels":[],"pull_request":{}}"#,
LabelLeg::IsPullRequest,
),
(r#"{"number":42,"state":"closed","labels":[]}"#, LabelLeg::Closed),
(
r#"{"number":42,"state":"open","labels":[{"name":"loom:frozen"}]}"#,
LabelLeg::Frozen,
),
(
r#"{"number":42,"state":"open","labels":[{"name":"loom:operator"}]}"#,
LabelLeg::OperatorHold,
),
(
r#"{"number":42,"state":"open","labels":[{"name":"loom:triage"}]}"#,
LabelLeg::AwaitingCurator,
),
(r#"{"number":42,"state":"open","labels":[]}"#, LabelLeg::Clean),
] {
assert_eq!(rest(payload, false), expected);
}
for label in ["loom:building", "loom:reviewing", "loom:treating"] {
let payload =
format!(r#"{{"number":42,"state":"open","labels":[{{"name":"{label}"}}]}}"#);
assert_eq!(rest(&payload, false), LabelLeg::Claimed(vec![label.into()]));
}
assert_eq!(
rest(
r#"{"number":42,"state":"open","labels":[{"name":"loom:building"},{"name":"loom:triage"}]}"#,
true
),
LabelLeg::AwaitingCurator
);
}

#[test]
fn rest_labels_refuse_unreadable_or_incomplete_answers() {
let dir = tempdir().unwrap();
for payload in [
"not json",
r#"{"number":43,"state":"open","labels":[]}"#,
r#"{"number":42,"labels":[]}"#,
r#"{"number":42,"state":"open","labels":[{}]}"#,
r#"{"number":42,"state":"open","labels":[],"pull_request":false}"#,
] {
assert_eq!(
claim_label_fallback(dir.path(), GRAPHQL_EXHAUSTED, payload, false),
LabelLeg::Unknown
);
}
let gh = write_exec(
dir.path(),
"gh-both-limited.sh",
r#"case "$1" in
issue) echo 'GraphQL: API rate limit exceeded' >&2 ;;
*) printf '%s\n' '{"number":42,"state":"open","labels":[]}'
echo 'HTTP 403: API rate limit exceeded' >&2 ;;
esac
exit 1"#,
);
assert_eq!(read_claim_labels(&gh, dir.path(), 42), LabelLeg::Unknown);
}

#[test]
fn permissions_and_secondary_limits_do_not_try_label_fallback() {
let dir = tempdir().unwrap();
for diagnostic in [
"HTTP 403: Resource not accessible by integration",
"GraphQL: You have exceeded a secondary rate limit",
] {
assert_eq!(
claim_label_fallback(
dir.path(),
diagnostic,
r#"{"number":42,"state":"open","labels":[]}"#,
false,
),
LabelLeg::Unknown
);
}
}

#[test]
fn read_freshest_live_lease_distinguishes_held_none_and_read_failed() {
let dir = tempdir().unwrap();
Expand Down
33 changes: 29 additions & 4 deletions loom-daemon/src/forge_gateway.rs
Original file line number Diff line number Diff line change
Expand Up @@ -513,10 +513,35 @@ pub fn route_read(req: &RouteRequest<'_>, now: SystemTime) -> RouteDecision {
}
let pool = crate::forge_identity::route_read(req, now);
let owner = crate::credential_preflight::owner_of_nwo(req.owner_repo).to_ascii_lowercase();
let gateway_until = if owner.is_empty() || profile_for(&owner).is_none() {
None
} else {
withdrawn_until(&owner, req, now)
let gateway_configured = !owner.is_empty() && profile_for(&owner).is_some();
let gateway_until = gateway_configured
.then(|| withdrawn_until(&owner, req, now))
.flatten();
let pool = match pool {
// The gateway was part of this route. A non-primary gateway hold
// must veto the stronger proof from the readers behind it.
RouteDecision::Exhausted {
until,
cause: ExhaustCause::GraphqlPrimaryBudget,
} if gateway_configured
&& (crate::forge_read_pool::is_withdrawn_at(GATEWAY_APP_ID, now)
|| crate::forge_identity::repo_withdrawn_at(
GATEWAY_APP_ID,
req.owner_repo,
now,
)
|| !crate::forge_identity::route::primary_graphql_withdrawn_at(
GATEWAY_APP_ID,
&owner,
now,
)) =>
{
RouteDecision::Exhausted {
until,
cause: ExhaustCause::Budget,
}
}
decision => decision,
};
match past_withdrawn_gateway(pool, gateway_until) {
RouteDecision::NoPool if cli_read_route_required() => RouteDecision::Exhausted {
Expand Down
26 changes: 25 additions & 1 deletion loom-daemon/src/forge_identity/route.rs
Original file line number Diff line number Diff line change
Expand Up @@ -182,6 +182,9 @@ pub enum ExhaustCause {
/// above the headroom reserve. At least one such reader exists and no
/// reader is excluded for any other reason.
Budget,
/// A GraphQL gate whose readers are all held by proven primary limits.
/// This carries evidence for a reader-only fallback, with the same budget log label.
GraphqlPrimaryBudget,
/// Every reader that could serve the repo is withdrawn or out of budget,
/// and at least one withdrawal is not a rate limit (App-wide, a refused
/// credential, the gateway's outage withdrawal). Every exclusion ends on
Expand All @@ -196,7 +199,7 @@ impl ExhaustCause {
#[must_use]
pub fn as_str(self) -> &'static str {
match self {
Self::Budget => "budget",
Self::Budget | Self::GraphqlPrimaryBudget => "budget",
Self::Withdrawn => "withdrawn",
Self::Unavailable => "unavailable",
}
Expand Down Expand Up @@ -784,6 +787,17 @@ fn is_github(host: Option<&str>) -> bool {
host.is_none_or(|h| h.eq_ignore_ascii_case("github.com"))
}

/// A live primary hold can authorize a fallback only while a newer reading
/// for the same bucket does not contradict it with positive headroom.
pub(crate) fn primary_graphql_withdrawn_at(app_id: &str, owner: &str, now: SystemTime) -> bool {
let Some(observed_at) = forge_read_pool::primary_graphql_observed_at(app_id, owner, now) else {
return false;
};
!bucket_key(app_id, owner, Resource::Graphql)
.and_then(|key| forge_bucket_book::reading(&key, epoch(now)))
.is_some_and(|r| r.observed_at >= epoch(observed_at) && r.remaining.is_some_and(|n| n > 0))
}

/// [`route_read`] against an explicit workspace, roster and environment.
#[must_use]
pub fn route_read_in(
Expand Down Expand Up @@ -1000,6 +1014,7 @@ pub fn route_read_in(
.and_then(|k| forge_bucket_book::projected_used_pct(&k, now_epoch))
};
let (mut budget, mut withdrawn, mut other, mut first_seen) = (0_usize, 0_usize, 0_usize, false);
let mut primary_graphql = 0;
for off in 0..n {
let r = readers[(start + off) % n];
if forge_read_pool::is_withdrawn_at(&r.app_id, now) {
Expand All @@ -1013,6 +1028,13 @@ pub fn route_read_in(
if forge_read_pool::is_budget_withdrawn_at(&r.app_id, owner, req.resource, now) {
budget += 1;
first_seen = true;
if req.class == ReadClass::Gate
&& req.resource == Resource::Graphql
&& primary_graphql_withdrawn_at(&r.app_id, owner, now)
&& super::dir_is_fresh(&super::reader_dir(workspace_root, owner, r), now)
{
primary_graphql += 1;
}
} else {
withdrawn += 1;
}
Expand Down Expand Up @@ -1049,6 +1071,8 @@ pub fn route_read_in(
ExhaustCause::Unavailable
} else if withdrawn > 0 {
ExhaustCause::Withdrawn
} else if primary_graphql == n {
ExhaustCause::GraphqlPrimaryBudget
} else {
ExhaustCause::Budget
};
Expand Down
2 changes: 2 additions & 0 deletions loom-daemon/src/forge_identity/tests.rs
Original file line number Diff line number Diff line change
Expand Up @@ -351,6 +351,8 @@ fn only_reviewed_read_paths_request_reader_credentials() {
// W4-C: the choke point's class-aware chain (the same reviewed
// routing step, with the router injected).
"gh_invocation/reader_route_v2.rs",
// The test-only route regression uses temporary reader directories and an inert gh.
"gh_invocation/reader_route_v2_tests.rs",
// W7: the ETag store's W4-C deferrable chain (`ConditionalRead::
// deferrable`), the same reviewed routing step with the router injected.
"forge_etag_store/deferrable.rs",
Expand Down
Loading
Loading