diff --git a/bench/lanelab/README.md b/bench/lanelab/README.md index 20f14cc4e2..2bbd602148 100644 --- a/bench/lanelab/README.md +++ b/bench/lanelab/README.md @@ -76,8 +76,8 @@ go run ./bench/lanelab/gosim -proof -pace flat -trace # one line per t implementation agreeing with the first is not evidence about the build. So `gosim` replays the same three scenarios against the code that ships: `lane.Default()`'s real ledger primed from this directory's sheet fixture at -`lane.SheetWeight`, the real chooser, the real `lane.Watch`, the real -`lane.DefaultBudget()`, over `internal/lane/lanestub` — the fake router, which +`lane.SheetWeight`, the real chooser, the real `lane.Watch`, and the real purse +the plan carries (`lane.Spending(lane.PlanFor(...))`), over `internal/lane/lanestub` — the fake router, which honours `provider.order`/`only`/`ignore`, names the serving lane on every chunk, sends a usage frame with the exact cost, and counts the streams a client abandoned. **The ship decision is taken on the Go run** (see diff --git a/bench/lanelab/gosim/main.go b/bench/lanelab/gosim/main.go index 31402575cf..e27fb5d7e2 100644 --- a/bench/lanelab/gosim/main.go +++ b/bench/lanelab/gosim/main.go @@ -591,11 +591,12 @@ func run(w *world, s scenario, policy string, seed, n, speedup int, trace bool) r := &router{ world: w, scen: s, policy: policy, seed: seed, stub: stub, ledger: ledger, - // THE DESIGN'S OWN BUDGET, and it is the shipped one rather than a - // figure written here: two hedges in any twenty requests and a tenth of - // recent spend. An arm that picked its own allowance would be measuring - // a router nobody ships. - budget: lane.DefaultBudget(), + // THE DESIGN'S OWN RAIL, and it is the shipped one rather than a figure + // written here: what a call may spend rescuing itself is its own patience + // converted through λ, which [lane.PlanFor] derives and [lane.Spending] + // reads. An arm that picked its own allowance would be measuring a router + // nobody ships. + purse: lane.Spending(lane.PlanFor(lane.Choice{}, lane.Pace{}, proofRole, theMoment)), client: &http.Client{}, speedup: speedup, total: n, diff --git a/bench/lanelab/gosim/proof.go b/bench/lanelab/gosim/proof.go index 090e276128..8c08a5dbc0 100644 --- a/bench/lanelab/gosim/proof.go +++ b/bench/lanelab/gosim/proof.go @@ -1033,7 +1033,7 @@ func runProofSeed(w *world, s scenario, c proofCase, seed, n, speedup int, trace p := &prover{ world: w, scen: s, kase: c, seed: seed, stub: stub, ledger: ledger, - budget: lane.DefaultBudget(), client: &http.Client{}, + client: &http.Client{}, speedup: speedup, total: n, at: theMoment, trace: trace, pace: pace, mix: mix, policy: policy, } @@ -1172,7 +1172,6 @@ type prover struct { seed int stub *lanestub.Server ledger lane.Ledger - budget *lane.Budget client *http.Client speedup int total int @@ -1373,7 +1372,7 @@ func (p *prover) headFor(choice lane.Choice) string { func (p *prover) plan(choice lane.Choice, head string) control.Plan { plan := lane.PlanFor(choice, p.paceFor(head), proofRole, p.at) plan.Pinned = len(choice.Only) > 0 - plan.Purse = lane.Spending(p.budget) + plan.Purse = lane.Spending(plan) plan.Think = lane.Thinks(p.world.model, "", p.at) return plan } @@ -1523,8 +1522,9 @@ func (p *prover) mark(out *trial, act control.Act, asked time.Time, late time.Du // answer carries out the first act. // // A HEDGE GOES OUT AS A DEMAND, to the lane the controller named, and the purse -// is SPENT here rather than where it was asked: [lane.Budget.Allow] counts the -// arm it allows, and the controller only ever asked whether one was affordable. +// is asked again here rather than trusted from where the controller asked it: +// the price is this arm's own and the rail is the CALL'S BUDGET +// ([lane.Spending]), which asks and never spends, so asking twice costs nothing. // An offer with a reader is left standing, because a pin is asked and this bench // has nobody to answer with. An offer with nobody there becomes the borrow §E // describes — and that conversion is this file's, not the build's. @@ -1534,10 +1534,9 @@ func (p *prover) answer(ctx context.Context, act control.Act, primary *armed, se return false } price := p.priceOf(act.Lane) - if !p.budget.Allow(p.at, price) { + if purse := p.plan(lane.Choice{}, act.Lane).Purse; purse != nil && !purse.Allows(price, p.at) { return false } - p.budget.NoteHedge(price, p.at) out.armed, out.armPrice = true, price out.usd += price arm := p.open(ctx, nil, []string{act.Lane}) @@ -1598,8 +1597,6 @@ func (p *prover) close(index int, served string, out trial) { if out.thought && out.answered { lane.NoteThought(p.world.model, "", thinkFor, p.at) } - p.budget.NoteRequest(p.at) - p.budget.NoteSpend(out.usd, p.at) if p.trace { fmt.Fprintf(os.Stderr, "proof %-22s/%d %4d served %-14s %-36s action %7.2fs s %7.2fs $%.6f%s\n", p.kase.name, p.seed, index, served, kindWords[out.kind]+"/"+out.reason+fmt.Sprintf(" W%.2f A%.2f", out.wait, out.cost), out.action, out.silence, out.usd, @@ -1962,17 +1959,17 @@ func proofGates(rows []proofRow, store, mix, policy string) []proofGate { return out } if !steady || !natural { - // THE PURSE IS WHAT BOUNDS SPENDING NOTHING ELSE BOUNDS, and it is what - // grades the bill wherever §K's own clause is not deciding it: on a cold - // store, where the arms are exploration, and on the stress mix, where - // the total is mostly correct rescues. The ceiling is read off the - // shipped budget rather than restated here, because a figure written - // down twice is a figure that will drift from the one the build - // enforces. - purse := 100 * lane.DefaultBudget().Share() + // WHAT BOUNDS SPENDING IS NOW PER CALL, SO THE AGGREGATE BAR IS THE + // BENCH'S OWN. The shipped rail is [control.Plan.SpendUSD] — one call's + // patience converted through λ — and it publishes no share of a bill for + // a grader to read. A bench is the thing that states a bar, so this one + // states it: a tenth of the bill is what the deleted purse allowed, and + // holding the new rail to the old number is what makes the replacement a + // claim rather than a hope. + const purse = 10.0 out = append(out, proofGate{ Criterion: "loser spend inside the purse", - Threshold: fmt.Sprintf("<= %.0f%% of the bill, which lane.DefaultBudget() allows", purse), + Threshold: fmt.Sprintf("<= %.0f%% of the bill, which the deleted purse allowed", purse), Measured: fmt.Sprintf("%.2f%% of $%.4f", spendPct, usd), Where: "every case", Value: spendPct, Gated: true, Pass: spendPct <= purse}) diff --git a/bench/lanelab/gosim/router.go b/bench/lanelab/gosim/router.go index 454347ebb1..5fa4f915d5 100644 --- a/bench/lanelab/gosim/router.go +++ b/bench/lanelab/gosim/router.go @@ -49,6 +49,7 @@ import ( "time" "github.com/Agent-Field/aforge-v2/internal/lane" + "github.com/Agent-Field/aforge-v2/internal/lane/control" "github.com/Agent-Field/aforge-v2/internal/lane/lanestub" ) @@ -81,7 +82,7 @@ type router struct { seed int stub *lanestub.Server ledger lane.Ledger - budget *lane.Budget + purse control.Purse client *http.Client speedup int total int @@ -264,10 +265,6 @@ func (r *router) one(index int) (record, error) { map[bool]string{true: " HEDGED", false: ""}[out.hedged], map[bool]string{true: " BROKEN:" + broken, false: ""}[broken != ""]) } - // THE BUDGET'S DENOMINATOR IS REQUESTS, so it is told about every one of - // them and not only about the ones that needed rescuing (Part III, C4). - r.budget.NoteRequest(r.at) - r.budget.NoteSpend(out.usd, r.at) // The world moves on by what this took plus the seconds somebody spends // reading whatever of it they read. @@ -344,7 +341,7 @@ func (r *router) plain(order, only, ignore []string, req lane.Request) (answered } // race is one request with the shipped rescue on it: the real [lane.Watch] over -// the real [lane.Choice], and the real [lane.DefaultBudget]. +// the real [lane.Choice], and the call's own budget ([lane.Spending]). // // THE WINNER IS THE FIRST STREAM TO SPEAK, which is what the shipped transport // does (`internal/provider/hedge.go` picks the speaker). The loser is cancelled @@ -403,14 +400,10 @@ func (r *router) race(choice lane.Choice, req lane.Request) (answered, error) { case <-hedgeAt: hedgeAt = nil estimate := r.estimate(rescue) - if !r.budget.Allow(r.at, estimate) { + if r.purse != nil && !r.purse.Allows(estimate, r.at) { continue } hedged = true - // A hedge is charged when it is FIRED and at the estimate it was - // allowed on, because the stream that loses is cancelled and never - // reports what it cost. - r.budget.NoteHedge(estimate, r.at) altBegan = time.Now() alt = r.start(ctx, nil, []string{rescue}, nil, req) altFirst, altDone = alt.first, alt.done @@ -495,7 +488,7 @@ func (r *router) priceOfServing(name string) float64 { } // estimate is what the alternative is expected to cost, which is what the -// budget is asked about. +// purse is asked about. func (r *router) estimate(name string) float64 { return r.priceOfServing(name) } // settle turns one finished stream into the answer, in the world's units. diff --git a/cmd/aforge-census/report.go b/cmd/aforge-census/report.go index 16372b3fde..5b11907ce1 100644 --- a/cmd/aforge-census/report.go +++ b/cmd/aforge-census/report.go @@ -489,9 +489,15 @@ func surprises(out io.Writer, rows, finishes []row) { wildest float64 ) var ( - cutByABound int - exhausted int + cutByABound int + exhausted int + rescueDenied int ) + // refusedBy is what stopped a rescue the controller had already called for, + // by the machine word the row carries. It is a reading of its own because a + // rescue that was ASKED FOR and did not happen is the one shape a census of + // waits cannot see any other way: the row looks like an ordinary slow call. + refusedBy := map[string]int{} appliedBy := map[string]int{} endedBy := map[string]int{} started := map[string]bool{} @@ -530,6 +536,10 @@ func surprises(out io.Writer, rows, finishes []row) { cutByABound++ appliedBy[word]++ } + if word := strings.TrimSpace(r.Refused); word != "" { + rescueDenied++ + refusedBy[word]++ + } if r.Exhaust() { exhausted++ } @@ -567,12 +577,16 @@ func surprises(out io.Writer, rows, finishes []row) { fmt.Fprintf(out, "| closed by the transport because no path wrote it | %d | %d |\n", unwritten, len(finishes)) fmt.Fprintf(out, "| cut by a bound this build set, and says which | %d | %d |\n", cutByABound, len(finishes)) fmt.Fprintf(out, "| exhaust of a race that was won, counted as a failure until now | %d | %d |\n", exhausted, len(finishes)) + fmt.Fprintf(out, "| a rescue the controller called for that never left, and says why | %d | %d |\n", rescueDenied, len(finishes)) if unwritten > 0 { fmt.Fprintf(out, "\nClosed as: %s.\n", topOf(endedBy, 6)) } if cutByABound > 0 { fmt.Fprintf(out, "\nCut by: %s.\n", topOf(appliedBy, 6)) } + if rescueDenied > 0 { + fmt.Fprintf(out, "\nRescues refused by: %s.\n", topOf(refusedBy, 6)) + } if wildest > absurdSeconds { fmt.Fprintf(out, "\nThe largest figure in seconds on any row is %g.\n", wildest) } diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index edb33d7ba0..b7216ca2d9 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -519,7 +519,7 @@ throws away work that was already paid for. | 3 | the controller, mid-stream | `internal/lane/control` | `W(s)` over the gap between two VISIBLE tokens exceeds acting, and leaving beats finishing | the same acts | the same two bounds. Commitment is the same inequality with the tokens already written on the other side | | 4 | the ceiling | `internal/lane/roles.go` | the silence reaches the role's own patience, whatever is believed | something is done: hedge, ask, or report | `VisiblePatience` **10s** × the role's `Patience` (×0.5 probe … ×6 standing, judge, design) | | 5 | the dead path | `internal/lane/watch.go`, `internal/provider/armwatch.go` | an act fired and NOTHING has reached this stream — no token, no comment | names the act `no heartbeat` and flags it, so no belief is charged for somebody's wifi | `lane.DeadPathFloor` **3s** of silence before the claim may be made at all: a handshake on a cold path can honestly outlast a fast lane's whole believed wait | -| 6 | the purse | `internal/lane/hedge.go` | asked at the instant an act would send | allows or refuses the arm; a refusal turns the act into a report | at most **a tenth** of the last hour's spend, counted over `requestWindow` **20** requests; `maxArms` **4** for one question | +| 6 | the purse | `internal/lane/hedge.go`, `internal/lane/watch.go` | asked before the controller decides AND before the arm sends, with one price | allows or refuses the arm; a refusal turns the act into a report and the row says `plan cannot pay` | THE CALL'S OWN BUDGET: `control.Plan.SpendUSD` = `Role.GiveUp()` × what a second of this role's wait is worth (`λ / AttentionValue`, over `AttentionValue` **90** s/$) — $1.00 for `talk`, $0.75 for `leaf.unattended`; `maxArms` **4** for one question, and it is the only thing that counts arms | | 7 | the pacing wait | `internal/provider/retry.go` | a 429 | waits and re-sends; the phase clock says `paced` with the router's own `Retry-After` as a real countdown | `baseBackoff` **700ms** doubled and jittered, one wait never over `maxProviderWait` **1 min**; the call ends at `rateLimitAttempts` **6** / `watchedPacingBudget` **2 min** watched, `patientAttempts` **60** / `patientPacingBudget` **10 min** unwatched | | 8 | the relax ladder | `internal/provider/endpoints.go` | a refusal about the request's SHAPE (400/404/unsupported parameter) | re-asks at once with one field dropped | **6 rungs**, then `maxFallbackModels` **2** other models — and **NO WAIT BETWEEN RUNGS**, because nothing here is backing off from a fault | | 9 | admission | `internal/provider/limiter.go` | more than `limiterCeiling` **64** requests in flight | queues, and hands the next freed slot to the waiter | **no timer at all.** `sharedLimiter.acquire` selects on the queue and the caller's context and nothing else; a request held here is held by the caller's own deadline | diff --git a/docs/changes/unreleased/924-live-slow-wire.md b/docs/changes/unreleased/924-live-slow-wire.md new file mode 100644 index 0000000000..fdf86d0ba9 --- /dev/null +++ b/docs/changes/unreleased/924-live-slow-wire.md @@ -0,0 +1,40 @@ +--- +kind: fixed +title: a wire below its promised pace is rescued out of the call's own budget, and every attempt is watched +pr: 924 +surface: [chat, engine, docs] +invalidates: + - "Rescues were bounded by a rolling PROCESS-WIDE allowance — at most two in any twenty requests and at most a tenth of the last hour's bill (`lane.Budget`, `provider.SetHedgeBudget`). That whole mechanism is deleted. A count spread over twenty requests cannot tell the one that needs rescuing from the nineteen that do not, so it refused by arrival order: on 2026-09-11 the controller reached its ceiling, asked for a second machine, and was told no on behalf of requests that had already finished — while one endpoint wrote 604 tokens in 86 seconds with a person watching. What a call may spend rescuing itself is now its own: `control.Plan.SpendUSD`, derived as the longest wait it can still have times what one second of that wait is worth to whoever is waiting — `Role.GiveUp()` × (`λ` / `AttentionValue`) / `AttentionValue`, so a watched second is worth a hundredth of a cent and an unwatched one a quarter of that. $1.00 for `talk`, $0.75 for `leaf.unattended`. It is asked through the same `control.Purse`, before the controller decides AND before the arm sends, with one price (`control.Alternative.Extra`). How many arms one question may run at once is still `maxArms = 4` and is now the ONLY thing that counts arms." + - "`SetLaneGuard(false)` switched rescues off by installing a budget with a zero allowance. There is no allowance to zero; it is now a purse that refuses everything (`lane.NoSpending`), written onto the plan where every arm reads it. `provider.SetHedgeBudget` and `lane.NewBudget` / `lane.DefaultBudget` no longer exist." + - "A row whose rescue never left said `refused: budget`, which named a rolling allowance and therefore said \"some other request spent this one's rescue\". It now says `refused: plan cannot pay`, which names the only rail there is. `make census` reads the field for the first time and reports `a rescue the controller called for that never left, and says why`, with the words underneath." + - "The aggregate credit test judged every visible gap against the SERVING lane's belief, which `Controller.Serving` re-points the moment the stream says who is answering. So a machine that had been collapsing all afternoon was judged against its own collapse, and the more this process learned about it the less any stream it served could read as slow: 604 tokens in 86 seconds and 5,793 in 363 were both \"keeping up\" against a belief of six a second. A stream is now held to the pace its question was SENT EXPECTING — the gap belief the plan was built with, which is the machine the choice named — so a router substituting a ten-times-slower machine is noticed and one answering at the rate it was asked for is never touched." + - "A `control.Report` was the end of the story: nothing could be started, so nothing happened and the question went on being answered at the pace that earned the report. A report is now also a move — the serving machine is written onto the plan's move log (asked of `control.Next`, so a question with nowhere else to go keeps the only machine it has) and the person is told." + - "There was no word for a stream that is writing too slowly to read. `writing` was false about the wait and `all lanes slow · still waiting` was false about the silence, so the surface said nothing: the measured call showed one nudge and then eighty-six seconds. `provider.PhaseBelowPace` is the third word and `internal/tui3` draws it as `answering slowly · nowhere faster · 1m 26s`. Two things had to stop taking the word back for a person to read it: the phase for a wait nothing can be done about is chosen in ONE place now (`hedgeRace.theWait`, from the act's reason) instead of by `exhaust` and then again by whichever of `tellFirstPrompt`/`tellTheWait` ran next; and `phaseClock.enter` refuses to replace `below pace` with `writing`, because a stream below its pace IS writing and the next token used to erase the judgement after one token of the eighty-six seconds it was about." + - "A watch belonged to an ARM and an arm can send many times, so every attempt after the first ran inside the first one's once-only flags: `streamWatch.acted` keeps the FIRST act, so a later attempt's row could never be written, and the controller's own report and purse refusal are once per controller. The 363-second second attempt of 2026-09-11 carried no action, no reason and no silence at all. Every attempt now gets its own controller, dated from the moment its own bytes leave (`streamWatch.attempt`, called from `Client.send`); what belongs to the QUESTION — the deadline and the move log — is untouched." + - "`TestTheScreenshotScenarioAnswersThroughTheFourthMachine` was red on dev: the walk it stages died on the first machine's 404, because the walk was bounded by that rolling allowance. It now answers through the fourth machine, and its phase assertion is corrected — a walk between MACHINES says `switching` and names the one it is going to, which is the better sentence; the ordinal belongs to the retry loop, which is not what moved." +--- + +Replicated first as failing tests from the evidence, then fixed. `internal/lane/control` +stages the substituted machine (604 tokens at 137 ms each against a question sent +expecting sixty a second) and the two cases that must NOT fire; `internal/provider` +stages the same wire through `lanestub` twice — once where the call's budget affords the +rescue and the fast machine answers, once where it may spend nothing and the row, the +move log and the person's own line all have to carry it; `internal/lane` replays the +09:33 series and the 14:29 sighting and asserts the chain the controller waits against +learns the collapse in one sighting. + +What is deliberately NOT here, measured and written down at `lane.stepTo` rather than +patched: the flat `Belief.Rate` the CHOOSER ranks on is a separate, slower account of the +same number, and it still lags. A run of consistent sightings makes that single filter +confident — in this very replay its variance fell to 0.011 against an observation noise +of 0.36, a gain of three per cent — and its one escape hatch is the chain's change point, +which on a collapse like this one never fires: the CUSUM peaks at 3.4 against an alarm of +4.0 and then decays, because a chain that absorbs nine tenths of a surprise in one +observation leaves no run of surprises to accumulate. Lowering the alarm buys false steps +on ordinary noise and giving the flat belief its own change point is a third account of +one number, so the fix belongs where the two accounts are reconciled — and that moves how +every lane is ranked, which is a question for `bench/lanelab` and not for one afternoon. +The textbook repair is measured and refused: an offline replay over ten days of the call +log put a floored `P` through the same decisions and moved watched regret 0.9% (inside the +estimator's own 24% median error) while making unattended regret 17% worse and switching +machines 9–10% more. `lane.stepTo` says so, so the next reader does not re-derive it. diff --git a/docs/design/waiting/DESIGN.md b/docs/design/waiting/DESIGN.md index ffcc9be034..14a08d8976 100644 --- a/docs/design/waiting/DESIGN.md +++ b/docs/design/waiting/DESIGN.md @@ -247,9 +247,12 @@ anywhere in this design. `Hedge` may fire more than once. What bounds it is money, not a counter: -- **Arms are bounded by the purse**, which is today's `lane.Budget` unchanged: - at most a tenth of the last hour's spend, counted in requests rather than in - minutes. `maxArms = 4` stays as the absolute cap on one question. +- **Arms are bounded by the purse**, which since 2026-09-11 is the CALL'S OWN + budget (`control.Plan.SpendUSD`: its patience converted through λ) rather than + the rolling `lane.Budget` this section was written against — that one allowed + two rescues in any twenty requests and a tenth of the last hour's spend, and it + refused by arrival order. `maxArms = 4` stays as the absolute cap on one + question, and it is now the only thing that counts arms. - **First visible progress takes the voice.** Whichever arm writes the first word a person can read is the arm they hear; the others are held, exactly as today's one-voice rule holds them. @@ -784,7 +787,8 @@ func PlanFor(Choice, Pace, Role, time.Time) control.Plan func PaceOf(Belief) Pace func PaceFor(ID, time.Time) Pace func HeadOf(Choice) string -func Spending(*Budget) control.Purse // asks Affordable; the race counts at send +func Spending(control.Plan) control.Purse // asks Plan.SpendUSD; asks and never spends +func NoSpending() control.Purse // the guard off: refuses everything func Thinks(model, rung string, now time.Time) control.Survival func NoteThought(model, rung string, took time.Duration, at time.Time) @@ -800,7 +804,7 @@ func (Role) Ceiling() time.Duration ``` **Public names kept:** `lane.Choice`, `lane.Chooser`, `lane.Ledger`, -`lane.Sighting`, `lane.Outcome`, `lane.Belief`, `lane.Budget`, +`lane.Sighting`, `lane.Outcome`, `lane.Belief`, `lane.Spending`, `lane.PerceivedSeconds`, `lane.Lambda`, `provider.HedgeReport`, `provider.PhaseNews`, `provider.SetLanePin`. @@ -1191,7 +1195,9 @@ for a change that can check every surface's reading of that event. - **It does not race every lane on every request.** Three to five arms is three to five times the bill for a p50 the hedge already captures at about 2%. - The purse is what bounds arms, and it is deliberately tight. + `maxArms` is what bounds arms, and since 2026-09-11 it is the ONLY thing that + counts them. The purse bounds MONEY, out of the call's own budget, and in + practice it refuses only an arm dearer than the whole wait it would buy back. - **It does not add a second escalation ladder.** `Escalate` hands the wait to the one that exists. A controller that changed a model would be answering a person's question with a model nobody chose for it. diff --git a/internal/calllog/calllog.go b/internal/calllog/calllog.go index 96861a14d2..2d95f5ad37 100644 --- a/internal/calllog/calllog.go +++ b/internal/calllog/calllog.go @@ -321,7 +321,15 @@ type Record struct { // absent when nothing was acted on. Reason string `json:"reason,omitempty"` // Refused is why a hedge the controller called for never reached the wire: - // "budget", "no alt" or "no room". It is absent when nothing was refused. + // "plan cannot pay", "no alt" or "no room". It is absent when nothing was + // refused. + // + // IT USED TO SAY `budget` AND THAT NAMED THE WRONG THING. The rail was a + // rolling process-wide allowance until 2026-09-11 — two rescues in any twenty + // requests — so the word said "some other request spent this one's rescue", + // which is a fact about arrival order and not about this call. What may + // refuse now is the call's own budget (internal/lane/control's + // Plan.SpendUSD), and the word names it. Refused string `json:"refused,omitempty"` // Arms is how many requests this one question put on the wire, counting the // original. One is the ordinary case and is left off the row. diff --git a/internal/lane/belief.go b/internal/lane/belief.go index 8dac5d3b13..4926d41252 100644 --- a/internal/lane/belief.go +++ b/internal/lane/belief.go @@ -975,6 +975,38 @@ func (l *ledger) folded(of *chains, id ID, held Posterior, z, noise float64, at // posterior far too certain to move. Adopting the observation outright is // exactly what [Posterior.Update] already does for a belief that knows nothing, // which is what a belief whose subject has just changed IS. +// +// ── AND IT ONLY RUNS WHEN THE ALARM FIRES, WHICH ON A COLLAPSE LIKE THAT ONE +// IT DOES NOT ──────────────────────────────────────────────────────────────── +// +// This is a measured seam, left open on purpose. Replayed on 2026-09-11 from +// the sighting series above (see the replay in belief_test.go), the CUSUM peaks +// at 3.40 against [cusumAlarm]'s 4.0 and then DECAYS, so the reset below never +// runs and the flat belief goes on crawling: by the fifth collapsed sighting the +// chain was at 7.4 tokens a second and the flat belief the chooser ranks on was +// still at 17.1. +// +// The cause is structural rather than a knob set wrong. The test rides on the +// residual of [chains.fold], and the chain absorbs about four fifths of a +// surprise in ONE observation — its gain is (ΣP)/(ΣP+R) over all four levels, +// and [levelSpread] squared and summed is 2.86 against an observation noise of +// [defaultSpread] squared, which is 0.36, so the gain is about nine tenths — so a +// genuine step leaves one large residual and then a run of small ones, which is +// the opposite of the persistent offset a CUSUM exists to accumulate. Lowering +// the alarm would trade this for false steps on ordinary noise, and giving the +// flat belief its own change point would be a third account of one number. The +// fix belongs where the two accounts are reconciled, and it moves how every lane +// is ranked, so it is a bench question and not a lane's. +// +// AND THE OBVIOUS ANSWER — PROCESS NOISE — HAS BEEN MEASURED AND IS NOT IT. The +// filter freezes because a run of consistent sightings shrinks P toward R/n with +// nothing to inflate it between them, so the textbook repair is a Q added on +// every step, or a floor under P. Replayed offline over ten days of the call log +// (3,605 requests, 1,157 of them with a person watching), a floored P moved +// watched regret by 0.9% — inside the estimator's own 24% median error, so not a +// measurement of anything — and made UNATTENDED regret 17% worse while switching +// machines 9–10% more often, and at all three of the 2026-09-11 incidents it +// asked for the same machine this code already asks for. Do not re-derive it. func stepTo(z, noise float64) Posterior { return Posterior{}.Update(z, noise) } // stepped records that a change point has reset one pair's own component toward diff --git a/internal/lane/belief_test.go b/internal/lane/belief_test.go index aff38ff520..b895ec4b1b 100644 --- a/internal/lane/belief_test.go +++ b/internal/lane/belief_test.go @@ -407,3 +407,63 @@ func TestEightStreamsFinishingAtOnceIsOneLedger(t *testing.T) { } } } + +// ── THE 2026-09-11 COLLAPSE, REPLAYED ─────────────────────────────────────── + +// morphRow is the sheet's account of the machine in that day's evidence: sixty +// tokens a second, which is what it really did all morning. +func morphRow() Row { + return Row{ + ID: ID{Model: "deepseek/deepseek-v4.1-flash", Lane: "Morph"}, + Facts: Facts{ + Tools: true, Quant: "fp8", MaxOut: 32_000, Context: 345_000, + Uptime5m: 100, PriceIn: 0.0000009, PriceOut: 0.00000132, Caches: true, + }, + TTFTp50: 1900, TTFTp75: 2400, TTFTp90: 3100, TTFTp99: 5000, + Ratep50: 60, Ratep75: 70, Ratep90: 84, Ratep99: 110, + } +} + +// TestACollapsedRateIsLearnedInOneSightingByTheChainTheControllerWaitsAgainst +// is the replay of the 09:33 series and the 14:29 sighting: a morning at sixty +// tokens a second, then 604 tokens in 82 seconds. +// +// IT IS THE CHAIN THAT IS ASSERTED, because the chain is what decides whether a +// live stream is abnormal, and the 2026-09-11 evidence turned on the chain +// having the collapse right. The two accounts of this one number do not agree +// afterwards and this test does not pretend they do — the flat [Belief.Rate] the +// chooser ranks on is a single filter that a run of consistent sightings makes +// confident (P fell to 0.011 against an observation noise of 0.36 in this very +// series, a gain of three per cent), and its one escape hatch is [stepTo], which +// only runs when the chain's change point fires. THAT is the seam, measured and +// written down at [stepTo]; it is not closed here, because closing it moves how +// every lane is ranked and that is a question for the bench. +func TestACollapsedRateIsLearnedInOneSightingByTheChainTheControllerWaitsAgainst(t *testing.T) { + l := newLedger() + row := morphRow() + l.Prime(row, SheetWeight) + at := noon + // The morning: 600 tokens in ten seconds, over and over. + for range 10 { + l.Note(Sighting{ID: row.ID, TTFT: 1900 * time.Millisecond, Gen: 10 * time.Second, Tokens: 600, At: at}) + at = at.Add(2 * time.Minute) + } + morning, _ := l.Rate(row.ID, at).Predict() + if math.Abs(math.Exp(morning)-60) > 6 { + t.Fatalf("a morning at sixty tokens a second was waited against at %.1f", math.Exp(morning)) + } + // And the collapse, in the numbers the row carried: ttft 3993, 82.19s of + // generation, 604 completion tokens. + l.Note(Sighting{ID: row.ID, TTFT: 3993 * time.Millisecond, Gen: 82190 * time.Millisecond, Tokens: 604, At: at}) + + // AND THE FIGURE IS THE ANSWER'S OWN, never one written here. 604 tokens in + // 82.19 seconds is 7.35 a second; a controller still waiting against a rate + // nearer the morning's sixty than the afternoon's seven is a controller that + // will not call this stream abnormal until its ceiling does it for free. + measured := 604 / 82.19 + collapsed, _ := l.Rate(row.ID, at).Predict() + if math.Exp(collapsed) > 2*measured { + t.Fatalf("after 604 tokens in 82 seconds the machine was still waited against at %.1f t/s, against the %.1f it measured", + math.Exp(collapsed), measured) + } +} diff --git a/internal/lane/control/contract.go b/internal/lane/control/contract.go index 9a564c18f1..b803deca90 100644 --- a/internal/lane/control/contract.go +++ b/internal/lane/control/contract.go @@ -263,10 +263,25 @@ type Alternative struct { // Purse is the spend rail the controller asks before it acts. // -// It is an interface rather than a figure because the answer depends on what -// has been spent in the last hour, which is not this package's business to -// know. A purse that refuses is FINAL for that moment: the controller records -// the refusal and does not poll it. +// It is an interface rather than a figure because what a call may spend is +// assembled where a call's role and its prices are both known, which is not this +// package's business. A purse that refuses is FINAL for that moment: the +// controller records the refusal and does not poll it. +// +// ── IT IS THIS CALL'S OWN BUDGET AND NEVER A WINDOW'S ─────────────────────── +// +// It was a rolling process-wide allowance until 2026-09-11 — two rescues in any +// twenty requests and a tenth of the last hour's bill — and that shape is the +// measured defect. A count per window cannot tell the request that needs a +// rescue from the nineteen that do not, so it refuses by arrival order: on the +// owner's task that day the hazard said act at its ceiling, the allowance said +// no, and one machine wrote 604 tokens in 86 seconds with a person watching. +// What a rescue costs is a fact about THIS call — the prompt at this model's +// price plus what the second arm may write, which [Alternative.Extra] already +// carries — and what it may cost is a fact about [Plan.SpendUSD]. HOW MANY ARMS +// ONE QUESTION MAY HAVE AT ONCE IS A DIFFERENT QUESTION AND IT HAS ITS OWN +// ANSWER (internal/provider's maxArms); a purse that also counted them would be +// two mechanisms for one shape. type Purse interface { Allows(usd float64, now time.Time) bool } @@ -302,9 +317,37 @@ type Plan struct { // [TurnGiveUp]). A zero moment is NO deadline, which is the honest reading // for a plan somebody built by hand and the reason [Plan.Spent] asks. Deadline time.Time - // SpendUSD is what this whole call may cost, zero being unbounded. It is - // the second half of one budget: the purse below bounds one ACT, and this - // bounds the call that keeps taking them. + // SpendUSD is what this call may spend on RESCUING ITSELF, zero being + // unbounded. It is the second half of one budget: the deadline above bounds + // how long, and this bounds how much. + // + // IT IS DERIVED AND IT IS NOT A KNOB. The money a call may spend buying its + // wait back is what that wait is worth to whoever is waiting through it, and + // that is two factors: how long it may still be, and what one second of it is + // worth to them. + // + // SpendUSD = GiveUp × (λ / AttentionValue) / AttentionValue + // + // [Plan.Lambda] is SECONDS PER DOLLAR — a dollar buys `AttentionValue` + // seconds of somebody's attention back — so the rate at the top of the scale + // is `1 / AttentionValue` dollars a second, and `λ / AttentionValue` is how + // much less this role's seconds are worth than that. A wait nobody is sitting + // through is worth a quarter of one somebody is, which is what + // `lane.UnattendedValue` says and all this reads off it. + // + // AND THE DISCOUNT MUST BE A MULTIPLIER, WHICH IS THE BUG IT WAS SHIPPED + // WITH. `GiveUp / λ` reads the same ratio upside down: the smaller λ of an + // unwatched call makes its seconds four times DEARER, so a conversation a + // person was reading got $1.00 to rescue itself and a background task node + // got $12.00 — the one rail in front of a rescue sized in the opposite order + // to who is waiting. `lane.spendable` carries the derivation and + // `lane.worth` is the arithmetic; the law that no watched second may be worth + // less than an unwatched one is beside them. + // + // Nothing else in this build states a per-call dollar ceiling, and a figure + // invented here would be a number nobody measured deciding what somebody's + // time is worth. A plan built by hand carries zero, which is unbounded — the + // honest reading for a plan nobody priced. SpendUSD float64 // Moves is what this question has already tried, shared by every arm of it. // A nil log is empty and decides nothing. diff --git a/internal/lane/control/hazard.go b/internal/lane/control/hazard.go index c6f4497e57..050ef59923 100644 --- a/internal/lane/control/hazard.go +++ b/internal/lane/control/hazard.go @@ -207,6 +207,10 @@ type hazard struct { // this request: polling it every beat would be asking the same question of // the same numbers. refused bool + // entitled is the pace this question was SENT EXPECTING — see THE PACE A + // STREAM IS HELD TO below — and it is the one belief on this type that + // [Controller.Serving] may not overwrite. + entitled Survival // z is the abnormality gate's threshold, in standard deviations of the // governing survival's own log-normal. It is derived once from the shape of // this request — see the derivation at the top of this file — because k is @@ -225,6 +229,7 @@ func New(plan Plan) Controller { } return &hazard{ plan: plan, + entitled: plan.Gap, now: plan.Began, progress: plan.Began, delta: plan.Began, @@ -292,7 +297,7 @@ func (h *hazard) Note(reading Reading) Act { // The belief is per token, while one event may contain a whole // batch. Normalize its interval so healthy batching keeps credit. perToken := gap.Seconds() / float64(reading.Visible) - surprise := (math.Log(perToken) - h.plan.Gap.Mu) / h.plan.Gap.Sigma + surprise := (math.Log(perToken) - h.pace()) / h.plan.Gap.Sigma h.drift = h.drift*decay + surprise h.weight = h.weight*decay + 1 } @@ -311,6 +316,52 @@ func (h *hazard) Note(reading Reading) Act { return h.verdict() } +// ── THE PACE A STREAM IS HELD TO IS NOT THE PACE OF WHOEVER ANSWERED ─────── +// +// A MACHINE MAY NOT SET ITS OWN BAR, and until 2026-09-11 it did. The aggregate +// credit test below measures each gap against [Plan.Gap], which +// [Controller.Serving] re-points to the SERVING lane's own belief the moment the +// stream says who is really answering — so a machine that has been collapsing all +// afternoon is judged against the collapse. That is a closed circle: the more +// this process learned about a slow machine, the less any stream it served could +// ever read as slow. Measured that day, twice. One endpoint's believed rate had +// already fallen to about six tokens a second; it then wrote 604 tokens in 86 +// seconds and 5,793 in 363, and both were "keeping up". +// +// THE BAR IS THE PACE THE QUESTION WAS SENT EXPECTING: the gap belief the plan +// was BUILT with, which is the machine the choice named and the reason this +// request went out at all. `provider.order` is advisory once fallbacks are on +// (#850), so the machine that answers is routinely not the machine that was +// asked for — and a router that substitutes a machine ten times slower has +// handed the person something they were never offered. That is what this notices +// and it is the whole of what it notices. +// +// IT IS NOT A RULE ABOUT SLOWNESS. Where the machine that answered IS the machine +// that was asked for, the bar and the belief are the same number and nothing at +// all changes — which is every ordinary request. A healthy stream on a machine a +// third slower than the fleet's best is not touched, because nobody was promised +// the fleet's best; they were promised this machine. +// +// AND A QUESTION SENT BLIND IS ENTITLED TO NOTHING IN PARTICULAR. With no belief +// behind the head of the order there is no expectation to hold anybody to, so the +// first belief that arrives becomes the bar ([hazard.Serving]) — which is the +// behaviour this test has always had for a cold pair. + +// pace is that bar, as the log of the seconds between two visible tokens: the +// same unit and sign convention [Plan.Gap] is in, so the surprise above is +// unchanged arithmetic with a centre that cannot be moved by the machine being +// judged. +// +// THE SPREAD IS STILL THE SERVING LANE'S, deliberately: how variable ONE gap is +// is a property of the machine we are on, while where the gap OUGHT to sit is a +// property of what this question was sent expecting. +func (h *hazard) pace() float64 { + if h.entitled.Known() { + return h.entitled.Mu + } + return h.plan.Gap.Mu +} + // keepingUp reports whether the measured visible rate still earns progress. // An unknown gap belief decides nothing, and the first visible token has no // preceding visible gap — its interval belongs to the first-token clock. @@ -337,6 +388,13 @@ func (h *hazard) Serving(lane string, first, gap Survival, now time.Time) { } if gap.Known() { h.plan.Gap = gap + // AND THE BAR IS SET ONCE. A question sent with no belief at all is + // entitled to whatever the machine that answered it turns out to be; one + // sent expecting a pace keeps that expectation whoever answers. See THE + // PACE A STREAM IS HELD TO. + if !h.entitled.Known() { + h.entitled = gap + } } if first.Known() && h.phase == PhaseSilent { h.plan.First = first @@ -633,10 +691,12 @@ func (h *hazard) costAlt() (Alternative, bool) { // act is the ladder: which of the six this moment is, in the order // docs/ARCHITECTURE.md already sets. // -// The purse is asked LAST and only when a hedge is really about to go out, -// because asking it is spending it: [internal/lane.Budget.Allow] counts the arm -// it allows, and a controller that polled it while deciding to report something -// else would spend somebody's allowance on a decision nobody acted on. +// The purse is asked LAST and only when a hedge is really about to go out. +// It no longer SPENDS anything by being asked — there is no rolling allowance +// left for a poll to drain ([internal/lane.Spending] is a reading of this +// plan's own budget) — and it is still asked last, because a controller that +// priced an arm while deciding to report something else would be pricing a +// decision nobody took. // AND AT THE CEILING λ STOPS DECIDING. What a second is worth is what makes a // wait worth money, and for a role nobody is watching it is nothing — but the // ceiling is not about money at all. It is the promise that no call this build diff --git a/internal/lane/control/ratedrift_test.go b/internal/lane/control/ratedrift_test.go index 62a33c902d..8d2b082d6f 100644 --- a/internal/lane/control/ratedrift_test.go +++ b/internal/lane/control/ratedrift_test.go @@ -229,3 +229,85 @@ func TestHealthyBatchedTokensKeepTheirProgressCredit(t *testing.T) { } } } + +// ── A STREAM FROM A MACHINE NOBODY ASKED FOR ─────────────────────────────── + +// TestARouterSubstitutingATenTimesSlowerMachineIsNoticed is the 2026-09-11 +// shape, in the numbers it happened in. The question went out expecting the +// machine the choice named — believed at sixty tokens a second — and the router +// answered from one believed at seven. Every gap is then perfectly ordinary FOR +// THE MACHINE THAT ANSWERED, which is exactly why nothing used to notice: the +// bar moved to whoever picked up. +func TestARouterSubstitutingATenTimesSlowerMachineIsNoticed(t *testing.T) { + p := plan() + // What the choice named, and the reason this request went out at all. + p.Gap = logNormal(1/60.0, 0.6) + p.Expected = 1200 + p.Ceiling = 10 * time.Second + p.Lambda = 0 + + watch := New(p) + // And the machine that really answered, with its own — correct, learned — + // belief that it writes at seven a second. + watch.Serving("other", Survival{}, logNormal(1/7.3, 0.6), epoch) + + var acted Act + for token := 1; token <= 604; token++ { + act := watch.Note(Reading{At: at(token * 137), Visible: 1}) + if act.Kind != None && acted.Kind == None { + acted = act + } + } + if acted.Kind == None { + t.Fatal("a stream ten times slower than the machine the question asked for was never acted on") + } + if acted.Reason != RateReason { + t.Fatalf("reason = %q, want %q", acted.Reason, RateReason) + } + // The whole wait the person really paid was 86 seconds, because a stream + // that is writing is not a silence and nothing read it as one. The promise + // is the role's own ceiling, counted from the last credited progress, and + // one beat's slack over it is the reading the stream happened to land on. + if acted.Silence > p.Ceiling+time.Second { + t.Fatalf("acted after %s of silence, past the %s ceiling", acted.Silence, p.Ceiling) + } +} + +// TestAMachineAnsweringAtTheRateItWasAskedForIsNeverJudgedSlow is the other half +// and the false-positive bound. The same collapsed machine, asked for on +// purpose: nobody was promised anything faster, so nothing here is abnormal and +// the bar and the belief are one number. +func TestAMachineAnsweringAtTheRateItWasAskedForIsNeverJudgedSlow(t *testing.T) { + p := plan() + p.Gap = logNormal(1/7.3, 0.6) + p.Expected = 1200 + p.Ceiling = 10 * time.Second + p.Lambda = 0 + + watch := New(p) + watch.Serving("head", Survival{}, logNormal(1/7.3, 0.6), epoch) + for token := 1; token <= 604; token++ { + if act := watch.Note(Reading{At: at(token * 137), Visible: 1}); act.Reason == RateReason { + t.Fatalf("a machine answering at exactly the rate it was asked for was called collapsed at token %d: %+v", token, act) + } + } +} + +// TestAQuestionSentBlindIsEntitledToNothingInParticular is the cold pair, where +// there is no expectation to hold anybody to: the first belief that arrives +// becomes the bar, and a machine matching its own belief is left alone. +func TestAQuestionSentBlindIsEntitledToNothingInParticular(t *testing.T) { + p := plan() + p.Gap = Survival{} + p.Expected = 1200 + p.Ceiling = 10 * time.Second + p.Lambda = 0 + + watch := New(p) + watch.Serving("other", Survival{}, logNormal(1/7.3, 0.6), epoch) + for token := 1; token <= 604; token++ { + if act := watch.Note(Reading{At: at(token * 137), Visible: 1}); act.Reason == RateReason { + t.Fatalf("a question sent with no belief judged its answer slow at token %d: %+v", token, act) + } + } +} diff --git a/internal/lane/e2e_real_test.go b/internal/lane/e2e_real_test.go index ee0d32eaeb..f1fe16f0f3 100644 --- a/internal/lane/e2e_real_test.go +++ b/internal/lane/e2e_real_test.go @@ -65,6 +65,7 @@ import ( "github.com/Agent-Field/aforge-v2/internal/config" "github.com/Agent-Field/aforge-v2/internal/home" "github.com/Agent-Field/aforge-v2/internal/lane" + "github.com/Agent-Field/aforge-v2/internal/lane/control" "github.com/Agent-Field/aforge-v2/internal/provider" "github.com/Agent-Field/agentfield/sdk/go/ai" ) @@ -297,12 +298,12 @@ func TestRealRouterServesASheetAndHonoursAPreference(t *testing.T) { pin.Only, pin.Order, pin.Ignore = []string{slow.ID.Lane}, nil, nil belief, _ := ledger.Belief(slow.ID) watch := lane.NewWatch(pin, belief, time.Now()) - // The shipped budget, so that this test is measuring the router somebody - // gets rather than an allowance written here: two hedges in any twenty - // requests and a tenth of recent spend. One request needs one of them. - budget := lane.DefaultBudget() + // The shipped rail, so that this test is measuring the router somebody gets + // rather than an allowance written here: what one call may spend rescuing + // itself, which is its own patience converted through λ ([lane.PlanFor]). + purse := lane.Spending(lane.PlanFor(lane.Choice{}, lane.Pace{}, lane.RoleTalk, time.Now())) - rescue, err := streamPinned(ctx, key, pin, watch, budget, ledger) + rescue, err := streamPinned(ctx, key, pin, watch, purse, ledger) if err != nil { t.Fatalf("the pinned call: %v", err) } @@ -321,9 +322,8 @@ func TestRealRouterServesASheetAndHonoursAPreference(t *testing.T) { "a rescue that does not beat the wait is a second bill for nothing", rescue.wall.Round(time.Millisecond), expected.Round(time.Millisecond)) } - case !budget.Allow(time.Now(), 0.001): - t.Logf("no rescue was attempted: the hedge budget refuses everything " + - "(lane watch and hedge budget not landed yet, wave 1 L-C)") + case !purse.Allows(0.001, time.Now()): + t.Logf("no rescue was attempted: this call's own budget refuses everything") case rescue.wall <= expected: t.Logf("no rescue was needed: %s answered in %v against the %v its own sheet row "+ "predicted, so the watch was right not to spend a second request on it", @@ -333,7 +333,7 @@ func TestRealRouterServesASheetAndHonoursAPreference(t *testing.T) { // slower than its own sheet row predicted and no second request went // out — which is the watch missing exactly the case it exists for. It // is logged rather than failed because on a real router it is also what - // a budget that has already been spent looks like, and a live test that + // a call with nothing to spend looks like, and a live test that // failed on somebody else's traffic would be a test nobody could run. // The deterministic version of this claim is S2 in e2e_test.go, which // does fail. @@ -504,12 +504,12 @@ type realRescue struct { } // streamPinned sends the pinned request, drives [lane.Watch] over it, and -// spends at most one hedge under the budget. +// spends at most one hedge under the call's own budget. // // It talks to the router directly rather than through `internal/provider` // because the thing being tested is the WATCH — the contract in this package — // and the transport's own half of it is tested where the transport lives. -func streamPinned(ctx context.Context, key string, choice lane.Choice, watch *lane.Watch, budget *lane.Budget, ledger lane.Ledger) (realRescue, error) { +func streamPinned(ctx context.Context, key string, choice lane.Choice, watch *lane.Watch, purse control.Purse, ledger lane.Ledger) (realRescue, error) { streamCtx, cancel := context.WithCancel(ctx) defer cancel() @@ -582,7 +582,7 @@ func streamPinned(ctx context.Context, key string, choice lane.Choice, watch *la if verdict := watch.Silence(time.Now()); !verdict.Hedge { continue } - if !budget.Allow(time.Now(), 0.001) { + if !purse.Allows(0.001, time.Now()) { continue } hedged = true diff --git a/internal/lane/e2e_test.go b/internal/lane/e2e_test.go index d1c971111d..1d09a582a4 100644 --- a/internal/lane/e2e_test.go +++ b/internal/lane/e2e_test.go @@ -67,6 +67,7 @@ import ( "time" "github.com/Agent-Field/aforge-v2/internal/home" + "github.com/Agent-Field/aforge-v2/internal/lane/control" "github.com/Agent-Field/aforge-v2/internal/lane/lanestub" ) @@ -709,7 +710,7 @@ func TestS5NoFetchOnTheSendPath(t *testing.T) { type e2eRouter struct { stub *lanestub.Server ledger Ledger - budget *Budget + purse control.Purse client *http.Client // at is the moment in the WORLD, which advances by what each request // actually took plus the time somebody spends reading the answer. The @@ -737,11 +738,12 @@ func newE2ERouter(stub *lanestub.Server, ledger Ledger) *e2eRouter { return &e2eRouter{ stub: stub, ledger: ledger, - // THE DESIGN'S OWN BUDGET, and it is the shipped one rather than a - // figure written here: two hedges in any twenty requests and a tenth of - // recent spend ([DefaultBudget]). A scenario that picked its own - // allowance would be a scenario measuring a router nobody ships. - budget: DefaultBudget(), + // THE DESIGN'S OWN RAIL, and it is the shipped one rather than a figure + // written here: what a call may spend rescuing itself is its own patience + // converted through λ, which [PlanFor] derives and [Spending] reads. A + // scenario that picked its own allowance would be a scenario measuring a + // router nobody ships. + purse: Spending(PlanFor(Choice{}, Pace{}, RoleTalk, e2eMoment)), client: &http.Client{}, at: e2eMoment, } @@ -790,14 +792,6 @@ func (r *e2eRouter) send(t *testing.T, pinned string, pin bool) { if err != nil { t.Fatalf("send: %v", err) } - // THE BUDGET'S DENOMINATOR IS REQUESTS, so it is told about every one of - // them and not only about the ones that needed rescuing. That is the whole - // of the correction the simulator forced (Part III, C4): a bucket counted in - // minutes refills itself while a slow batch runs, and the same policy then - // hedges one request in a hundred on one seed and nineteen in twenty on - // another. - r.budget.NoteRequest(r.at) - r.budget.NoteSpend(answer.cost, r.at) r.answers = append(r.answers, answer.total) r.firsts = append(r.firsts, answer.first) if answer.hedged { @@ -873,16 +867,10 @@ func (r *e2eRouter) race(ctx context.Context, choice Choice, request Request) (e // The estimate a hedge is judged against: what a second whole // answer on the alternative would cost. estimate := e2eEstimate(rescue, request) - if !r.budget.Allow(r.at, estimate) { + if r.purse != nil && !r.purse.Allows(estimate, r.at) { continue } answer.hedged = true - // A hedge is charged when it is FIRED and at the estimate it was - // allowed on, because the stream that loses is cancelled and never - // reports what it cost. Counting only the waste that was observed - // would let the mechanism spend without limit as long as it kept - // being right. - r.budget.NoteHedge(estimate, r.at) alternate = r.start(ctx, nil, []string{rescue}, nil, request) case result := <-primary.done: @@ -1333,12 +1321,13 @@ func TestTheInstrumentInThisFileWorksBeforeAnySeamDoes(t *testing.T) { Default().SetSheet(e2eServeSheet(stub)) router := newE2ERouter(stub, ledger) - // AND NO ALLOWANCE, so that eight requests are eight asks. Every call is + // AND NOTHING TO SPEND, so that eight requests are eight asks. Every call is // watched now — a plan with a ceiling is built whether or not the chooser // had an opinion — and this test is about the instrument rather than about - // the rescue: a budget of nothing is how the second request is switched off - // (hedge.go), and it keeps the count below an assertion about plumbing. - router.budget = NewBudget(0, 0) + // the rescue: a purse that refuses everything is how the second request is + // switched off (hedge.go), and it keeps the count below an assertion about + // plumbing. + router.purse = NoSpending() router.run(t, 8, "Cloudflare", false) diff --git a/internal/lane/frontier.go b/internal/lane/frontier.go index 0c55e5b690..19e03ca19f 100644 --- a/internal/lane/frontier.go +++ b/internal/lane/frontier.go @@ -75,14 +75,16 @@ const ( // ahead of the lanes it does not — the one draw that can disprove a published // claim, because a lane that is never asked can never prove the sheet wrong. // - // IT IS THE HEDGE PURSE'S OWN RATE, WRITTEN AS A PERIOD. [DefaultBudget] - // allows two rescues in any twenty requests, and a probe is bought for - // exactly the same reason a rescue is: one request in ten pays a little to - // find out something every request after it spends better. The purse itself - // cannot be asked here — it is spent in DOLLARS and answers against a clock, - // and this file may read neither (see the note in choose.go) — so the rate is - // stated and the draw is the chooser's own sampler, which is seeded per - // request and therefore reproducible. + // IT IS AN EXPLORATION RATE AND IT NOW STANDS ON ITS OWN. It used to be + // written as the hedge purse's own rate — two rescues in any twenty requests + // — and that purse is deleted (hedge.go): what a call may spend rescuing + // itself is its own budget now, and there is no per-window count anywhere to + // borrow. What is left is the plain statement: one request in ten pays a + // little to find out something every request after it spends better, which is + // enough draws that a doubted lane is re-asked several times in an afternoon + // and few enough that nine people in ten never meet one. It is a prior to be + // fitted in the simulator, like the change-point's own two. The draw is the + // chooser's own sampler, seeded per request and therefore reproducible. probeInEvery = 10 // PriceCeilingMultiple is how far above the cheapest acceptable lane's own diff --git a/internal/lane/hedge.go b/internal/lane/hedge.go index 009cc32bb3..2639df014a 100644 --- a/internal/lane/hedge.go +++ b/internal/lane/hedge.go @@ -1,263 +1,97 @@ package lane import ( - "sync" "time" + + "github.com/Agent-Field/aforge-v2/internal/lane/control" ) -// ── THE BUDGET: WHAT KEEPS A RESCUE FROM BECOMING A SECOND BILL ───────────── +// ── THE PURSE: WHAT KEEPS A RESCUE FROM BECOMING A SECOND BILL ────────────── // // A hedge is the one mechanism in this package that can cost real money by -// working exactly as designed. Every other part of it is arithmetic over -// numbers somebody else already paid for; this one sends a second request. So -// the budget is not a safety net bolted on afterwards — it is the reason the -// mechanism is allowed to exist at all, and the empty budget refuses -// everything, which is the correct behaviour of an un-wired router. +// working exactly as designed. Every other part of it is arithmetic over numbers +// somebody else already paid for; this one sends a second request. So there is a +// rail in front of it, and what the rail is made of is the whole argument. // -// TWO LIMITS, BECAUSE THERE ARE TWO FAILURE MODES AND ONE LIMIT CANNOT SEE -// BOTH. +// ── IT USED TO BE A WINDOW, AND THE WINDOW WAS THE DEFECT ─────────────────── // -// - A SHARE OF REQUESTS bounds a storm: at most two hedges in any twenty -// requests, which is one in ten once a session is going and two at the very -// start, when there is no history to judge from. +// Until 2026-09-11 it was a rolling process-wide allowance: at most two hedges +// in any twenty requests, and at most a tenth of the last hour's bill. Both +// numbers were literals, and both answered the wrong question. A count per +// window cannot tell the one request that needs a rescue from the nineteen that +// do not, so it refuses by arrival order — and the measured case is what that +// costs. On the owner's task that day the controller reached its ceiling and +// asked for a second machine; the allowance said no; one endpoint then wrote 604 +// tokens in 86 seconds with a person watching an empty line, and the row carried +// `action report reason ceiling refused budget`. The rescue it refused would have +// cost about two cents against a wait worth a hundred times that. // -// IT COUNTS REQUESTS AND NOT MINUTES, and that correction came from the -// simulator. A per-minute bucket sounds equivalent and is not: a slow batch -// of `work` requests refills its own allowance while it runs, so the same -// policy hedged one per cent of requests on one seed and ninety-five per -// cent on another. The thing being rationed is REQUESTS, so the window has -// to be measured in them. -// - A SHARE OF RECENT SPEND bounds the slow leak. A hedge that fires a little -// too eagerly is invisible on any one request and obvious at the end of the -// month, and no per-request check can see it: the question is what fraction -// of the last hour's bill this mechanism added, and only a rolling window -// can answer that. This one IS in wall-clock time, because a bill is. +// ── WHAT IT IS INSTEAD: THIS CALL'S OWN BUDGET ────────────────────────────── // -// The windows are the CALLER'S: the budget is told when a request finished -// ([Budget.NoteRequest]), what a call cost ([Budget.NoteSpend]) and what a hedge -// cost ([Budget.NoteHedge]) by whoever billed them, and holds no clock, no -// ledger and no opinion about prices. It is safe to use from every goroutine -// that is streaming, because that is exactly how it will be used. +// What a rescue costs is a fact about ONE CALL — the prompt at this model's +// price plus whatever the second arm writes, which the frontier has already +// priced on every candidate machine ([control.Alternative.Extra]) — and what it +// may cost is [control.Plan.SpendUSD], which is that call's patience converted +// through λ ([PlanFor]). Both halves belong to the plan, so the purse is a +// reading of the plan and holds no state, no clock and no history. // -// WITH NOTHING SPENT YET THE SHARE CANNOT BE EXCEEDED, so a fresh session -// hedges under the bucket alone. The alternative — refusing until some spending -// has accumulated — would switch the mechanism off for precisely the first -// answers a person waits through, which are the ones they remember. - -// spendWindow is how far back the SPEND share is measured. An hour is long -// enough that one expensive turn does not licence a spree and short enough that -// a session which has changed what it is doing is judged on what it is doing -// now. -const spendWindow = time.Hour - -// requestWindow is how many requests back the RATE share is measured. Twenty -// is short enough that a bad minute is bounded while it is happening and long -// enough that the ratio it holds — one hedge in ten — is a ratio rather than a -// coin toss. -const requestWindow = 20 +// HOW MANY ARMS ONE QUESTION MAY RUN AT ONCE IS A DIFFERENT QUESTION AND IT IS +// ANSWERED ELSEWHERE (internal/provider's `maxArms`, four: the original and +// three rescues). A purse that counted them as well would be two mechanisms for +// one shape, which is exactly how the window came to overrule the controller. -// billed is one amount at the moment it was charged. -type billed struct { - usd float64 - at time.Time -} - -// Budget is what stops a rescue mechanism from becoming a second bill. +// Spending is one call's own budget as the rail the controller asks before it +// acts. // -// Two limits, because the two failure modes are different. A TOKEN BUCKET -// bounds a storm — one bad minute in which every stream stalls must not double -// every request in flight. A SHARE OF SPEND bounds the slow leak, where a -// hedge that fires a little too eagerly is invisible per request and obvious at -// the end of the month. -type Budget struct { - mu sync.Mutex - // perTwenty is how many hedges are allowed in any twenty requests. - perTwenty int - // share is the fraction of recent spend hedging may add, 0 to 1. - share float64 - // requests is how many have finished, ever, and marks is the request each - // allowed hedge was counted at. Marks are pruned as the window slides, so - // what is held is bounded by perTwenty rather than by the session's length. - requests int - marks []int - // calls and hedges are the rolling spend windows, oldest first. They are - // slices rather than running totals because a total cannot be un-added when - // it ages out, and a window that only grows is a budget that loosens - // forever. - calls []billed - hedges []billed -} - -// NewBudget states a budget: how many hedges are allowed in any twenty -// requests, and what share of recent spending they may add. A zero or negative -// allowance is a budget that allows nothing, which is how hedging is switched -// off. +// It is a small adapter and not a method on [control.Plan] because the direction +// of the dependency matters: the controller may not know what this package is, +// and the plan may not grow a method whose answer depends on prices. A zero +// allowance is UNBOUNDED and not empty — see [control.Plan.SpendUSD] — because a +// plan somebody built by hand priced nothing, and reading "nobody said" as +// "nothing may be spent" is how the window's refusal would come back by the +// other door. +func Spending(plan control.Plan) control.Purse { return purse{allowance: plan.SpendUSD} } + +// NoSpending is the purse of a call that may spend NOTHING on rescuing itself. // -// THE FIRST ARGUMENT IS PER TWENTY REQUESTS AND NOT PER MINUTE. It was per -// minute until the simulator showed what that does — see the law at the top of -// this file — and the signature is unchanged because the shape of the answer is -// the same: a small integer allowance and a share. -func NewBudget(perTwenty int, share float64) *Budget { - return &Budget{perTwenty: perTwenty, share: share} +// It is a person's own switch and not an arithmetic answer: `SetLaneGuard(false)` +// is somebody saying "do not spend extra to keep an answer moving", and a call +// carrying this purse has no rescue to be refused rather than one it could not +// afford. It is spelled as its own constructor because zero dollars already +// means UNBOUNDED on a plan nobody priced (see [control.Plan.SpendUSD]), and one +// figure cannot honestly mean both. +func NoSpending() control.Purse { return purse{refuses: true} } + +// purse is that reading. It is a value rather than a pointer because there is +// nothing in it to mutate: the question "can this call pay for one more arm" has +// the same answer however many times it is asked, which is what lets the +// controller ask it while deciding and the race ask it again at the moment the +// arm really goes out. +type purse struct { + allowance float64 + refuses bool } -// DefaultBudget is the budget the design argues for: two hedges in any twenty -// requests, a tenth of recent spend. It is stated once here so that a caller -// wiring the transport does not get to pick its own idea of "not very many". -func DefaultBudget() *Budget { return NewBudget(2, 0.10) } - -// Share is what fraction of recent spending hedging may add: the ceiling the -// slow-leak half of this budget enforces, as a fraction of one. +// Allows reports whether this call can pay for one arm costing usd. // -// IT IS EXPORTED SO THAT A BENCH GRADING A RUN AGAINST THE PURSE READS THE -// PURSE'S OWN FIGURE. `bench/lanelab/gosim` reports what hedging cost against -// what the budget allows it to cost, and a bench that restated the tenth would -// be a second copy of a number this package owns — which is the one thing this -// codebase's own law about a single source of truth forbids. -func (b *Budget) Share() float64 { - if b == nil { - return 0 - } - b.mu.Lock() - defer b.mu.Unlock() - return b.share -} - -// NoteRequest records that one request has finished. It is the denominator of -// the rate limit and it is called once per request, hedged or not. -func (b *Budget) NoteRequest(_ time.Time) { - if b == nil { - return - } - b.mu.Lock() - defer b.mu.Unlock() - b.requests++ - b.prune() -} - -// NoteSpend records what one ordinary call cost. It is the denominator of the -// share and nothing else. -func (b *Budget) NoteSpend(usd float64, now time.Time) { - if b == nil || usd <= 0 { - return - } - b.mu.Lock() - defer b.mu.Unlock() - b.calls = append(b.calls, billed{usd: usd, at: now}) - b.forget(now) -} - -// NoteHedge records what one hedge cost — the extra request, whether it won or -// was cancelled — which is the numerator. -// -// A hedge that won is still spending: the answer arrived, the loser was -// cancelled, and the bill for the turn is one stream plus however much of the -// other one was generated before the cancel landed. Counting only the waste -// would let the mechanism spend without limit as long as it kept being right. -func (b *Budget) NoteHedge(usd float64, now time.Time) { - if b == nil || usd <= 0 { - return - } - b.mu.Lock() - defer b.mu.Unlock() - b.hedges = append(b.hedges, billed{usd: usd, at: now}) - b.forget(now) -} - -// Allow reports whether one more hedge, expected to cost costEstimate dollars, -// may be sent now — and COUNTS IT when it says yes. +// IT ASKS AND IT DOES NOT SPEND, which is the one property the two callers both +// depend on. The controller asks while it is still deciding — a reading, and one +// it may take several times over one silence — and the surface asks again before +// it draws a countdown over a rescue ([hedgeRace.affordableAlt]); a rail that +// counted either of those would charge a question for arms it never started. +// What is really spent is bounded by the allowance above and by how many arms a +// question may have at once, and neither of those is a tally kept here. // -// It is a decision and not a question on purpose: two goroutines asking at the -// same moment must not both be told yes on the strength of one allowance, and a -// caller that had to confirm afterwards would be a caller that could forget to. -func (b *Budget) Allow(now time.Time, costEstimate float64) bool { - if b == nil { +// The moment is unused and is kept in the signature because [control.Purse] is +// asked by a package that holds no clock: every question in it takes the moment +// as an argument, and a rail that dropped it would be the one seam in that +// design that could not be given a scripted one. +func (p purse) Allows(usd float64, _ time.Time) bool { + if p.refuses { return false } - b.mu.Lock() - defer b.mu.Unlock() - if !b.affordableLocked(now, costEstimate) { - return false - } - b.marks = append(b.marks, b.requests) - return true -} - -// Affordable is the same question ASKED WITHOUT SPENDING, for the one caller -// that has to know the answer before there is anything to decide: a surface -// drawing "if this does not answer by 4.4s it switches to parasail" is making a -// promise, and a promise the budget was always going to refuse is the surface -// lying about the machinery. It is a reading and not a reservation — the -// decision is still [Budget.Allow]'s, taken at the moment the hedge is really -// wanted — which is the right way round: the promise may be withdrawn between -// the drawing and the deadline, and a person who is never promised anything is -// never let down. -func (b *Budget) Affordable(now time.Time, costEstimate float64) bool { - if b == nil { - return false - } - b.mu.Lock() - defer b.mu.Unlock() - return b.affordableLocked(now, costEstimate) -} - -// affordableLocked is the arithmetic both of them read, so that the question -// and the decision can never drift apart. It runs with the lock held and it -// prunes, which is a read that ages rather than a write that spends. -func (b *Budget) affordableLocked(now time.Time, costEstimate float64) bool { - if b.perTwenty <= 0 { - return false - } - b.prune() - if len(b.marks) >= b.perTwenty { - return false - } - b.forget(now) - if b.share > 0 { - var total, hedged float64 - for _, one := range b.calls { - total += one.usd - } - for _, one := range b.hedges { - hedged += one.usd - } - // Nothing spent is nothing to take a share of. See the law at the top: - // the bucket alone governs a session that has not billed anything yet. - if total > 0 && hedged+costEstimate > b.share*total { - return false - } - } - return true -} - -// prune drops the hedges that have slid out of the request window. -func (b *Budget) prune() { - oldest := b.requests - requestWindow - keep := b.marks[:0] - for _, mark := range b.marks { - if mark > oldest { - keep = append(keep, mark) - } - } - b.marks = keep -} - -// forget drops what has aged out of the window. It is called from every write -// so that a session which stops spending stops being judged on what it spent. -func (b *Budget) forget(now time.Time) { - cut := now.Add(-spendWindow) - b.calls = sinceCut(b.calls, cut) - b.hedges = sinceCut(b.hedges, cut) -} - -// sinceCut is the window's tail, copied in place rather than resliced so that a -// long session does not hold the whole hour's backing array forever. -func sinceCut(window []billed, cut time.Time) []billed { - keep := window[:0] - for _, one := range window { - if one.at.After(cut) { - keep = append(keep, one) - } + if p.allowance <= 0 { + return true } - return keep + return usd <= p.allowance } diff --git a/internal/lane/lane_test.go b/internal/lane/lane_test.go index b41c309f00..1b65684a6b 100644 --- a/internal/lane/lane_test.go +++ b/internal/lane/lane_test.go @@ -231,16 +231,16 @@ func TestEverySeamAnswersAndNoneOfThemInventsANumber(t *testing.T) { } } -// TestABudgetThatCannotCountRefuses is the one implementation that could cost +// TestAPurseThatSpendsNothingRefuses is the one implementation that could cost // somebody money if it guessed. // -// It said so by refusing everything while lane L-C had not built the counting. -// Now that it counts, the law it holds is the one that survives: a budget with -// no allowance allows nothing, which is how hedging is switched off, and -// hedge.go carries the rest of its behaviour. -func TestABudgetThatCannotCountRefuses(t *testing.T) { - if NewBudget(0, 0.1).Allow(time.Now(), 0.0001) { - t.Fatal("a budget with no allowance allowed a hedge") +// It used to be a budget with no allowance — the counting rail this build no +// longer has — and the law that survived the deletion is the one a person's own +// switch depends on: `SetLaneGuard(false)` is a purse that refuses everything, +// and hedge.go carries the rest of its behaviour. +func TestAPurseThatSpendsNothingRefuses(t *testing.T) { + if NoSpending().Allows(0.0001, time.Now()) { + t.Fatal("a purse that spends nothing allowed a rescue") } } diff --git a/internal/lane/watch.go b/internal/lane/watch.go index 1be12caab0..db790ef541 100644 --- a/internal/lane/watch.go +++ b/internal/lane/watch.go @@ -114,7 +114,7 @@ type Pace struct { // and not twice. func PlanFor(choice Choice, pace Pace, role Role, now time.Time) control.Plan { head := HeadOf(choice) - return control.Plan{ + plan := control.Plan{ Lane: head, Ceiling: role.Ceiling(), Floor: ActionFloor, @@ -135,7 +135,71 @@ func PlanFor(choice Choice, pace Pace, role Role, now time.Time) control.Plan { Deadline: now.Add(role.GiveUp()), Role: string(role), Moves: control.NewMoveLog(), + // AND THE MONEY IS THE SAME SENTENCE SAID IN DOLLARS: the longest wait + // still in front of this call, priced at what one second of it is worth + // to whoever is waiting ([spendable]). It is derived here, once, for the + // reason the deadline is: a second place that named a figure would be a + // second answer to "what may this call spend rescuing itself" (see + // [control.Plan.SpendUSD]). + SpendUSD: spendable(role), } + plan.Purse = Spending(plan) + return plan +} + +// spendable is [control.Plan.SpendUSD] for one role: what the wait this call may +// still be kept in is worth to whoever is waiting through it, in dollars. +// +// ── TWO FACTORS, AND THE SECOND ONE USED TO POINT THE WRONG WAY ───────────── +// +// HOW LONG is [Role.GiveUp]: the longest wait still in front of this call, which +// is the most a rescue could ever buy back. +// +// WHAT A SECOND OF IT IS WORTH is the other factor, and it is the one this got +// backwards until 2026-09-11. [AttentionValue] is stated in SECONDS PER DOLLAR — +// a dollar buys ninety seconds of somebody's attention back — so a second of a +// watched wait is worth `1 / AttentionValue` dollars, about a hundredth of a +// cent, and that is the rate at the TOP of the scale. [UnattendedValue] is a +// quarter of it because a second nobody is sitting through is worth a quarter of +// one somebody is ([Lambda]'s own account of why zero was measured wrong). So +// the discount for being unwatched is the RATIO `λ / AttentionValue`, and a +// second's worth is: +// +// dollars per second = (λ / AttentionValue) / AttentionValue +// +// Dividing the wait by λ, which is what this did, used that ratio upside down: +// the smaller λ of an unwatched call made its seconds four times DEARER than a +// watched person's, so a conversation somebody was reading was allowed $1.00 to +// rescue itself and a background task node $12.00 — the one rail in front of a +// rescue, sized in the opposite order to who is waiting. It is the shape the +// deleted process-wide allowance had (refusing the request that needed it) with +// the priority order reversed rather than removed. +// +// WHAT THIS DOES AND DOES NOT ORDER. It orders the RATE: no second of a watched +// wait is ever worth less than a second of an unwatched one, which is the law +// beside this in watch_test.go. It does not order the total, and must not — +// a standing pass really does tolerate nine minutes where a conversation +// tolerates ninety seconds, and six times the wait at a quarter the rate is +// worth more than one times the wait at the full rate. What the wait is worth is +// the product; what nobody is allowed to say is that an unwatched SECOND is +// worth more. +// +// A ROLE NOBODY WAITS ON SPENDS NOTHING, and that is λ at zero rather than a +// case here — the controller already refuses to act for such a role below its +// ceiling ([hazard.reachable]), and saying it again would be the same rule +// written twice. Today no role answers zero ([Lambda]'s floor is +// [UnattendedValue]), so the guard is against a plan somebody builds by hand. +func spendable(role Role) float64 { + return worth(role.GiveUp(), role.Lambda()) +} + +// worth is a wait in dollars: how long, times what a second of it is worth to +// whoever is waiting. It is the one place that arithmetic is written. +func worth(wait time.Duration, lambda float64) float64 { + if lambda <= 0 { + return 0 + } + return wait.Seconds() * (lambda / AttentionValue) / AttentionValue } // PaceFor is what THIS PROCESS believes about one pair right now. @@ -294,28 +358,6 @@ func alternatives(choice Choice, head string) []control.Alternative { return alts } -// Spending is a budget as the rail the controller asks before it acts. -// -// It is a small adapter and not a method on [Budget] because the direction of -// the dependency matters: the controller may not know what a budget is, and the -// budget may not know what a controller is. A nil budget refuses, which is how -// hedging is switched off and has always been. -func Spending(budget *Budget) control.Purse { return purse{budget} } - -type purse struct{ budget *Budget } - -// Allows ASKS AND DOES NOT SPEND, which is [Budget.Affordable] and deliberately -// not [Budget.Allow]. -// -// The two halves of a rescue are two different moments. The controller asks -// whether an arm is affordable while it is still deciding — a reading, and one -// it may take several times over one silence — and the race takes the allowance -// at the instant the arm really goes out, which is the decision. A controller -// that reserved would leave allowances held by every request that recovered on -// its own, and one that counted here as well would charge the budget twice for -// one arm. -func (p purse) Allows(usd float64, now time.Time) bool { return p.budget.Affordable(now, usd) } - // ── DRIVING IT ────────────────────────────────────────────────────────────── // controller is the one this watch was built with, from the one factory. diff --git a/internal/lane/watch_test.go b/internal/lane/watch_test.go index 5ac8863293..35a3cb6440 100644 --- a/internal/lane/watch_test.go +++ b/internal/lane/watch_test.go @@ -590,85 +590,114 @@ func TestTheDeadlineIsNeverInThePastAndNeverPastTheCeiling(t *testing.T) { } } -// ── THE BUDGET ────────────────────────────────────────────────────────────── +// ── THE PURSE ─────────────────────────────────────────────────────────────── -func TestTheAllowanceIsCountedInRequestsAndNotInMinutes(t *testing.T) { +// TestTheCallsOwnPatienceIsWhatItMaySpendRescuingItself is the derivation in +// [control.Plan.SpendUSD], read back off a plan the shipped door built. The +// figure is never written here: a test that restated it would pass while the +// build's own conversion drifted away from it. +func TestTheCallsOwnPatienceIsWhatItMaySpendRescuingItself(t *testing.T) { now := time.Now() - budget := NewBudget(2, 0) - if !budget.Allow(now, 0) || !budget.Allow(now, 0) { - t.Fatalf("a fresh budget refused one of its first two hedges") - } - if budget.Allow(now, 0) { - t.Fatalf("a third hedge went out inside the same twenty requests") - } - // TIME ALONE BUYS NOTHING. This is the correction the simulator forced: a - // slow batch of requests must not refill its own allowance while it runs. - if budget.Allow(now.Add(time.Hour), 0) { - t.Fatalf("an hour of waiting refilled an allowance that is counted in requests") - } - // Requests do. Nineteen of them still hold the first two hedges inside the - // window; the twentieth slides the first one out. - for range 19 { - budget.NoteRequest(now) - } - if budget.Allow(now, 0) { - t.Fatalf("a hedge went out with two still inside the last twenty requests") - } - budget.NoteRequest(now) - if !budget.Allow(now, 0) { - t.Fatalf("the oldest hedge never slid out of the window") + for _, role := range []Role{RoleTalk, RoleLeafUnattended, RoleStanding} { + plan := PlanFor(Choice{}, Pace{}, role, now) + want := role.GiveUp().Seconds() * (role.Lambda() / AttentionValue) / AttentionValue + if math.Abs(plan.SpendUSD-want) > 1e-12 { + t.Fatalf("%s may spend $%.4f rescuing itself, want $%.4f", role, plan.SpendUSD, want) + } + if plan.Purse == nil { + t.Fatalf("%s built a plan with no purse on it", role) + } + // The rescue that was refused on 2026-09-11 cost about two cents. Every + // role this build has is asked whether it could have paid for it, because + // the answer that mattered was the same for all of them. + if !plan.Purse.Allows(0.02, now) { + t.Fatalf("%s refused a two-cent rescue against a $%.4f allowance", role, plan.SpendUSD) + } } } -func TestTheAllowanceHoldsAtAboutOneHedgeInTen(t *testing.T) { +// TestNoWatchedSecondIsWorthLessThanAnUnwatchedOne is the ordering the shipped +// derivation got backwards, and it is stated PER SECOND because that is where it +// is true. +// +// WHAT WAS WRONG. `SpendUSD = GiveUp / λ` divided by a figure that is smaller +// for a call nobody is watching, so an unwatched second came out four times +// dearer than a watched one: `talk` was allowed $1.00 to rescue itself and +// `leaf.unattended` $12.00, `standing` $24.00. The one rail in front of a rescue +// was sized in the opposite order to who is waiting, which is the deleted +// window's own defect — refusing the request that needed it — with the priority +// reversed rather than removed. +// +// WHY THE LAW IS NOT "EVERY WATCHED ROLE AFFORDS MORE THAN EVERY UNWATCHED +// ONE". Patience varies thirty-fold WITHIN each group — `recall` waits 18 +// seconds and `talk` ninety, `probe` 45 and `standing` nine minutes — so no +// per-second price can make the totals order that way, and a price that did +// would be saying a nine-minute wait is worth less than an eighteen-second one. +// What a wait is worth is how long it is TIMES what a second of it is worth; the +// second factor is the one exposure decides, and it is the one asserted here, +// across every role in the table rather than the three that were noticed. +func TestNoWatchedSecondIsWorthLessThanAnUnwatchedOne(t *testing.T) { now := time.Now() - budget := NewBudget(2, 0) - hedges := 0 - for range 200 { - if budget.Allow(now, 0) { - hedges++ + cheapestWatched, dearestUnwatched := math.Inf(1), 0.0 + var cheapest, dearest Role + for role := range roles { + if !role.Known() { + continue + } + plan := PlanFor(Choice{}, Pace{}, role, now) + seconds := role.GiveUp().Seconds() + if seconds <= 0 { + t.Fatalf("%s gives up after no time at all", role) + } + perSecond := plan.SpendUSD / seconds + if role.Facts().Interactive { + if perSecond < cheapestWatched { + cheapestWatched, cheapest = perSecond, role + } + continue + } + if perSecond > dearestUnwatched { + dearestUnwatched, dearest = perSecond, role } - budget.NoteRequest(now) - } - // Two in the first twenty and one in every ten after them. - if hedges < 18 || hedges > 22 { - t.Fatalf("%d hedges in two hundred requests, want about one in ten", hedges) } -} - -func TestAZeroAllowanceIsHowHedgingIsSwitchedOff(t *testing.T) { - if NewBudget(0, 0.5).Allow(time.Now(), 0) { - t.Fatalf("a budget of nothing allowed a hedge") + if cheapestWatched < dearestUnwatched { + t.Fatalf("a second of %s (watched) is worth $%.6f and a second of %s (unwatched) $%.6f — the rail is sized in the opposite order to who is waiting", + cheapest, cheapestWatched, dearest, dearestUnwatched) } - var nothing *Budget - if nothing.Allow(time.Now(), 0) { - t.Fatalf("a nil budget allowed a hedge") + // AND THE PAIR THAT WAS MEASURED WRONG, named so a reader can see it: a + // conversation somebody is reading may never buy less of its own wait back + // than a task node nobody is sitting in front of. + talk := PlanFor(Choice{}, Pace{}, RoleTalk, now) + leaf := PlanFor(Choice{}, Pace{}, RoleLeafUnattended, now) + if talk.SpendUSD/RoleTalk.GiveUp().Seconds() < leaf.SpendUSD/RoleLeafUnattended.GiveUp().Seconds() { + t.Fatalf("talk values a second of its own wait at $%.6f and leaf.unattended at $%.6f", + talk.SpendUSD/RoleTalk.GiveUp().Seconds(), leaf.SpendUSD/RoleLeafUnattended.GiveUp().Seconds()) } } -func TestTheShareRefusesOnceHedgingHasHadItsTenthOfTheBill(t *testing.T) { +// TestAnArmDearerThanTheWholeWaitIsWorthIsRefused is the other side of the same +// figure: the rail is money and it still says no to something. +func TestAnArmDearerThanTheWholeWaitIsWorthIsRefused(t *testing.T) { now := time.Now() - budget := NewBudget(60, 0.10) - budget.NoteSpend(1.00, now) - if !budget.Allow(now, 0.05) { - t.Fatalf("refused a five-cent hedge against a dollar of spending") - } - budget.NoteHedge(0.09, now) - if budget.Allow(now, 0.05) { - t.Fatalf("allowed a hedge that would take the share past a tenth") + plan := PlanFor(Choice{}, Pace{}, RoleTalk, now) + if plan.Purse.Allows(plan.SpendUSD*2, now) { + t.Fatalf("an arm costing twice the call's whole allowance was afforded") } - // And an hour later the window has rolled: neither the spending nor the - // hedging that was in it is judged any more. - later := now.Add(2 * time.Hour) - if !budget.Allow(later, 0.05) { - t.Fatalf("the share was still judging an hour-old bill") +} + +// TestAPlanNobodyPricedIsUnboundedAndNotEmpty is the law a zero has to keep. +// Reading "nobody said" as "nothing may be spent" is how the deleted window's +// refusal would come back by the other door. +func TestAPlanNobodyPricedIsUnboundedAndNotEmpty(t *testing.T) { + if !Spending(control.Plan{}).Allows(1000, time.Now()) { + t.Fatalf("a plan nobody priced refused a rescue") } } -func TestWithNothingSpentTheRequestAllowanceAloneGoverns(t *testing.T) { - now := time.Now() - budget := NewBudget(2, 0.10) - if !budget.Allow(now, 0.02) { - t.Fatalf("refused the first hedge of a session that has billed nothing yet") +// TestTheSpeedGuardOffIsAPurseThatRefusesEverything is the one switch a person +// has, and it is spelled as a purse rather than as a flag the race consults. +func TestTheSpeedGuardOffIsAPurseThatRefusesEverything(t *testing.T) { + if NoSpending().Allows(0, time.Now()) { + t.Fatalf("a purse that spends nothing allowed a free rescue") } } diff --git a/internal/manual/chat/lanes.md b/internal/manual/chat/lanes.md index fd0ff99158..aa7130330e 100644 --- a/internal/manual/chat/lanes.md +++ b/internal/manual/chat/lanes.md @@ -131,7 +131,7 @@ Text arriving in a batch earns progress for its approximate token count, so a provider that sends whole phrases is not judged as though each phrase were one token. Tool-only replies also teach the first-token and generation clocks. -When a watched request fails and its recovery allowance can fund another +When a watched request fails and this call's own budget can pay for another endpoint, that endpoint is tried before repeating the failed request. Rate limits still respect their retry delay. Without an affordable alternative, the existing bounded retries and wait reporting remain. @@ -298,15 +298,29 @@ and a second request costs money. They are never silent either — the same sentence is on their row. **Ten seconds always does something, even when a second request is too -expensive.** aforge only runs a small number of rescues — a second request to -another machine costs real money, so it keeps a small allowance and spends it -where it helps most. When the allowance is gone and the machine has sent nothing -at all, not one byte, aforge stops that attempt instead and asks somewhere else. -Before 2026-09-10 it did neither: four tasks that evening sat on one machine for -six and seven minutes after the ten seconds were up, because the only way to act -was the one aforge could not afford. If the machine IS sending something — the -router is talking, or the model is writing where you cannot see it — nothing is -stopped, because nine such calls in ten turn out to be seconds from an answer. +expensive.** A second request to another machine costs real money, so every +rescue is priced before it goes out — against what THIS call may spend, which is +how long it is allowed to keep trying converted into money at what a second of +your waiting is worth. A rescue costing a couple of cents against a minute and a +half of your time is afforded; one costing more than the whole wait is worth is +not. When a rescue is refused and the machine has sent nothing at all, not one +byte, aforge stops that attempt instead and asks somewhere else. Before +2026-09-10 it did neither: four tasks that evening sat on one machine for six and +seven minutes after the ten seconds were up, because the only way to act was the +one aforge could not afford. If the machine IS sending something — the router is +talking, or the model is writing where you cannot see it — nothing is stopped, +because nine such calls in ten turn out to be seconds from an answer. + +**There is no per-session rescue quota.** There used to be: at most two rescues +in any twenty requests, and at most a tenth of the last hour's bill, shared by +everything running in aforge at once. That is gone, and it is gone because it +answered the wrong question — a count spread over twenty requests cannot tell the +one that needs rescuing from the nineteen that do not, so it refused whichever +asked last. On 2026-09-11 that is exactly what happened: a machine wrote 604 +words in 86 seconds with somebody watching, the rescue was called for, and the +quota said no on behalf of requests that had already finished. What bounds a +rescue now is this call's own budget and how many requests one question may have +running at once, which is four. **A conversation turn gives up after ninety seconds** of not reaching any model at all, and tells you so in one line. It is the point where every model in the @@ -484,6 +498,35 @@ countdown, and there is nothing behind it about when the answer will come: it is that a line which cannot promise you anything can at least be honest about the size of what it is asking you to sit through. +## When the answer is arriving too slowly to read + +This is not the same thing as the line above, and it took a real afternoon to +learn the difference. `all lanes slow · still waiting` is about a **silence** — +nothing is arriving. Sometimes words ARE arriving and the wait is just as real, +because they are arriving at a crawl: + +``` +answering slowly · nowhere faster · 1m 26s +``` + +That line means aforge measured the words appearing against the pace the machine +it asked for was expected to write at, found the stream far under it, and has +nowhere better to send the question — every other machine has been tried, or +this call cannot pay for a second request. The answer is still coming and the +words still appear as they arrive; the line is there so that the wait has a name. + +On 2026-09-11 the same call showed one nudge and then nothing for eighty-six +seconds, because neither of the two lines aforge had was true: it was not +writing at any speed a person would call writing, and it was not silent either. + +**A machine is judged against the pace its question was sent expecting**, not +against its own recent form. That distinction is the whole fix: as aforge learned +that one machine had slowed to a seventh of its usual speed, every stream it +served started to look normal *for that machine*, and the guard quietly stopped +firing. What it is held to now is the machine the routing choice named — the +reason the request went out at all — so a router that quietly hands your question +to something ten times slower is noticed. + ## What the status line is telling you | what you see | what happened | @@ -497,6 +540,7 @@ it is asking you to sit through. | `coreweave is slow · switch to auto? (y)` | your pinned machine is quiet, and you can end the wait | | `coreweave cannot serve this model; routing on auto for this model until you pin again` | the machine you pinned said no, so the pin is retired for this model | | `all lanes slow · still waiting · 12s` | everywhere is slow; nothing to be done but tell you, and how long you have waited | +| `answering slowly · nowhere faster · 1m 26s` | words ARE arriving, too slowly to be worth reading, and there is no faster machine to move to | ## When a machine refuses to serve the model diff --git a/internal/provider/accountset_test.go b/internal/provider/accountset_test.go index 4e86fa1811..11e9ca370b 100644 --- a/internal/provider/accountset_test.go +++ b/internal/provider/accountset_test.go @@ -55,7 +55,6 @@ func cheapestExcludedRig(t *testing.T, name string, offered ...lanestub.Lane) *l rig := newLaneRigWithPrice(t, name, func(string) (float64, float64, bool) { return firstPartyIn, firstPartyOut, true }, offered...) - SetHedgeBudget(lanes.NewBudget(6, 0)) if !lanes.WireSheet(rig.server.URL(), "", sheetFetcher{}, false) { t.Fatal("the sheet would not take the stub as its base") } diff --git a/internal/provider/armwatch.go b/internal/provider/armwatch.go index fbe7ec9d74..b6e7dfc148 100644 --- a/internal/provider/armwatch.go +++ b/internal/provider/armwatch.go @@ -24,7 +24,10 @@ type streamWatch struct { race *hedgeRace arm int - mu sync.Mutex + mu sync.Mutex + // plan is what this arm's controller is built from, kept so that EVERY + // ATTEMPT can be given one ([streamWatch.attempt]). + plan control.Plan control control.Controller // deadline is the moment the controller last said it wanted waking at. The // beat sleeps until it rather than polling, so it is compared on every @@ -112,17 +115,18 @@ type streamWatch struct { // ── THE CEILING PICKS ONE OR THE OTHER, NEVER NEITHER ─────────────────────── // -// A hedge is the PAID way to act on a silence — a second request, out of a purse -// that is deliberately small (the waiting design's §B: two rescues per twenty -// calls). A cut is the FREE way: end this attempt and let the layer above ask -// another machine at once, paying the prompt again and nothing else. +// A hedge is the PAID way to act on a silence — a second request, priced against +// what this call may spend rescuing itself ([control.Plan.SpendUSD]). A cut is +// the FREE way: end this attempt and let the layer above ask another machine at +// once, paying the prompt again and nothing else. // -// UNTIL THIS WAVE THE CEILING COULD CHOOSE NEITHER, and that is a hole rather +// UNTIL THAT WAVE THE CEILING COULD CHOOSE NEITHER, and that is a hole rather // than a trade-off. `docs/design/waiting/DESIGN.md` §A clause 1 says the role's -// ceiling is hard "regardless of belief"; it has to be hard regardless of PURSE -// too, or the clause means "regardless of belief, when we happen to be able to -// afford it". Measured on 2026-09-10: 2,186 attempts fired the ceiling, had the -// purse refuse the arm, and then had NOTHING act — 648 of them went on for more +// ceiling is hard "regardless of belief"; it has to be hard regardless of what +// this call can afford too, or the clause means "regardless of belief, when we +// happen to be able to afford it". Measured on 2026-09-10: 2,186 attempts fired +// the ceiling, had the purse refuse the arm, and then had NOTHING act — 648 of +// them went on for more // than six times the silence that had just been refused, to a ninety-ninth // percentile of 272 seconds and a worst case of 938. Four quick tasks that // evening waited on one machine for six and seven MINUTES before its first @@ -178,6 +182,60 @@ func streamWatchFrom(ctx context.Context) *streamWatch { return watch } +// ── EVERY ATTEMPT IS WATCHED, AND IT IS THE SAME CONTROLLER EACH TIME ─────── +// +// THE LAW: ONE CONTROLLER PER ATTEMPT OF EVERY REQUEST — the primary's, a +// rescue arm's, and every re-ask under either of them. A watch belongs to an +// ARM and an arm can send many times: the refusal loop under it re-encodes and +// re-sends ([Client.send]), and until 2026-09-11 all of those attempts shared +// one controller built at the moment the arm started. That is not a watch, it is +// a watch of the first attempt with the rest of them inside its shadow. +// +// THE MEASURED CASE (2026-09-11 14:32, the owner's task). Attempt 1 drew a +// relayed refusal; attempt 2 asked GMICloud, was served by another machine, and +// ran 363 SECONDS — and its row carried no action, no reason and no silence at +// all, while the 86-second attempt before it had at least said `ceiling`. Three +// of this type's once-only rules are why, and each of them is right for one +// attempt and wrong for a call: [streamWatch.acted] keeps the FIRST act, so the +// row of a later attempt can never be written; the controller's own `acted` fires +// a report once per controller; and a purse refusal is final per controller. So +// the second attempt of a recovered call was, by construction, unwatched. +// +// Re-arming is not forgetting. WHAT BELONGS TO THE QUESTION STAYS — the deadline +// and the move log are the plan's and the plan is not rebuilt — and what belongs +// to ONE SEND is what starts again: when this send went out, what it has written, +// who answered it, and what was done about its wait. + +// attempt re-arms this watch for one send of its arm, and it is called by the +// send loop before every one of them ([Client.send]). +// +// IT IS UNCONDITIONAL AND HAS NO ATTEMPT NUMBER IN IT. A branch on "is this the +// first" would be the same rule written twice — the first send is a send like +// any other — and the first controller [hedgeRace.startArm] built is replaced by +// one dated from the moment the bytes really leave, which is the moment the +// person's wait actually starts. +func (w *streamWatch) attempt(now time.Time) { + if w == nil || w.race == nil || w.race.build == nil { + return + } + plan := w.plan + plan.Began = now + plan.Alts = w.race.untriedAlts() + w.mu.Lock() + w.control = w.race.build(plan) + w.began, w.first, w.last, w.gap = now, time.Time{}, time.Time{}, 0 + w.tokens, w.visible, w.beats = 0, 0, 0 + w.served = "" + w.acted, w.silence, w.fault = control.Act{}, 0, false + w.applied, w.appliedWord = 0, "" + w.deadline = w.control.Deadline() + if !w.deadline.IsZero() { + w.armed = w.deadline.Sub(now) + } + w.mu.Unlock() + w.race.rearm() +} + // note folds one moment of the stream into the controller and does whatever it // says. // diff --git a/internal/provider/armwatch_test.go b/internal/provider/armwatch_test.go index 6e8d1f6fbd..0328754c6e 100644 --- a/internal/provider/armwatch_test.go +++ b/internal/provider/armwatch_test.go @@ -119,7 +119,7 @@ func (s scriptedController) Acted(control.Kind) bool // THE SHAPE: four quick tasks, one machine, first tokens at 260, 370, 375 and // 428 seconds. On every one the hazard fired at its ten-second ceiling with // `no heartbeat` — not a token, not a router comment — and the row says -// `refused: budget`. The purse (two rescues per twenty calls) declined the arm, +// `refused: plan cannot pay`. The purse declined the arm, // and then NOTHING acted for another six minutes. Acting meant hedging, hedging // was purse-gated, and so the waiting design's §A clause 1 — the ceiling is hard // "regardless of belief" — quietly meant "when we can afford it". diff --git a/internal/provider/callprogress_test.go b/internal/provider/callprogress_test.go index 46214aeab4..902f6505d6 100644 --- a/internal/provider/callprogress_test.go +++ b/internal/provider/callprogress_test.go @@ -6,7 +6,6 @@ import ( "testing" "time" - lanes "github.com/Agent-Field/aforge-v2/internal/lane" "github.com/Agent-Field/aforge-v2/internal/lane/lanestub" ) @@ -210,7 +209,6 @@ func TestEachArmOfARaceIsWatchedAndTheLoserIsNotAFailure(t *testing.T) { // shape of a belief: ordinary jitter is never a surprise and a long silence // is nothing else. rig.believes("A", 2, 250) - SetHedgeBudget(lanes.NewBudget(1, 0)) log := &progressLog{} report := &HedgeReport{} diff --git a/internal/provider/ceiling_race_test.go b/internal/provider/ceiling_race_test.go index 43f6bb4880..9bc72cbcca 100644 --- a/internal/provider/ceiling_race_test.go +++ b/internal/provider/ceiling_race_test.go @@ -54,7 +54,6 @@ func ceilingRaceLanes(excludeFirstRescue bool) []lanestub.Lane { func TestARaceWalkDoesNotMistakeAnEndpointRefusalForAPriceRefusal(t *testing.T) { rig := newPricedLaneRig(t, "ceiling/walk", ceilingRaceLanes(false)...) - SetHedgeBudget(lanes.NewBudget(6, 0)) ctx := WithLaneChoice(talking(), ceilingRaceChoice(rig.model)) response, err := rig.client.CompleteWithMessages(ctx, userMessages("hello")) @@ -95,7 +94,7 @@ func TestARaceWalkDoesNotMistakeAnEndpointRefusalForAPriceRefusal(t *testing.T) func TestARefusedCeilingClimbsTheLadderWhenThePurseFundsNoWalk(t *testing.T) { rig := newPricedLaneRig(t, "ceiling/empty-purse", ceilingRaceLanes(false)...) - SetHedgeBudget(lanes.NewBudget(0, 0)) + noRescues(t) var notices []string ctx := noticeContext(WithLaneChoice(talking(), ceilingRaceChoice(rig.model)), ¬ices) @@ -124,30 +123,41 @@ func TestARefusedCeilingClimbsTheLadderWhenThePurseFundsNoWalk(t *testing.T) { } } -func TestARefusedRescueClimbsTheLadderWhenThePurseFundsNoSecondWalk(t *testing.T) { +// TestAWalkKeepsWalkingWhileTheModelHasAnotherMachine is what the deleted +// purse used to hide. +// +// IT USED TO STOP AT ONE, AND THAT WAS A COUNT AND NOT A REASON. The walk was +// bounded by a rolling allowance of two rescues in any twenty requests, so the +// second machine a refusal earned was refused for a number about other requests +// — and the call climbed the relaxation ladder while machines that had said +// nothing at all were still standing. What bounds a walk now is the serving set +// and [maxArms], which are facts about THIS question. +func TestAWalkKeepsWalkingWhileTheModelHasAnotherMachine(t *testing.T) { rig := newPricedLaneRig(t, "ceiling/one-walk", ceilingRaceLanes(true)...) - SetHedgeBudget(lanes.NewBudget(1, 0)) ctx := WithLaneChoice(talking(), ceilingRaceChoice(rig.model)) response, err := rig.client.CompleteWithMessages(ctx, userMessages("hello")) if err != nil { - t.Fatalf("the relaxed rescue did not answer: %v", err) + t.Fatalf("the walk did not answer: %v", err) } if got := answerTokens(response); got != 19 { t.Fatalf("the answer is %d tokens, want Together's 19", got) } asks := rig.server.Asks() if len(asks) != 3 { - t.Fatalf("%d requests went out, want the capped primary, one walk, and one relaxed retry", len(asks)) + t.Fatalf("%d requests went out, want the capped primary and two walks", len(asks)) } if asks[0].MaxPrice == nil { t.Fatal("the refused primary carried no ceiling, so this test proves nothing") } + // THE FIRST WALK GOES TO THE MACHINE THE FRONTIER NAMED, and it is refused + // because the sheet is all that offers it; the second goes to the next one, + // which answers. Neither carries the ceiling the primary was refused over. if !demandedOnly(asks[1], "Fireworks") || asks[1].MaxPrice != nil { - t.Fatalf("the walk carried only=%v max_price=%+v, want an uncapped Fireworks demand", asks[1].Only, asks[1].MaxPrice) + t.Fatalf("the first walk carried only=%v max_price=%+v, want an uncapped Fireworks demand", asks[1].Only, asks[1].MaxPrice) } - if len(asks[2].Only) != 0 || asks[2].MaxPrice != nil { - t.Fatalf("the relaxed retry carried only=%v max_price=%+v", asks[2].Only, asks[2].MaxPrice) + if !demandedOnly(asks[2], "Together") || asks[2].MaxPrice != nil { + t.Fatalf("the second walk carried only=%v max_price=%+v, want an uncapped Together demand", asks[2].Only, asks[2].MaxPrice) } for _, ask := range asks { if ask.Model != rig.model { diff --git a/internal/provider/dispatch.go b/internal/provider/dispatch.go index c6b4d6a034..ab7801dc88 100644 --- a/internal/provider/dispatch.go +++ b/internal/provider/dispatch.go @@ -579,6 +579,11 @@ func (c *Client) send(ctx context.Context, request *ai.Request, knobs callKnobs, cancelAttempt() return nil, err } + // AND THIS ATTEMPT GETS ITS OWN CONTROLLER. Every send of every request is + // watched, and it is watched from the moment its own bytes leave — see + // EVERY ATTEMPT IS WATCHED in armwatch.go for the 363-second attempt whose + // row said nothing because its arm's first attempt had already spoken. + streamWatchFrom(ctx).attempt(dispatchNow()) attemptBegan := logNow() // THE ROW THAT SAYS A CALL IS IN FLIGHT, written before the wait rather // than after it. Without it a planning call four minutes into a diff --git a/internal/provider/dispatch_e2e_test.go b/internal/provider/dispatch_e2e_test.go index 5fe288f524..77e61f6034 100644 --- a/internal/provider/dispatch_e2e_test.go +++ b/internal/provider/dispatch_e2e_test.go @@ -176,9 +176,12 @@ func TestOneDeadlineBoundsEverything(t *testing.T) { // limits, and a healthy fourth the sheet doubts. // // WHAT IT ASSERTS IS WHAT THE PERSON GOT. Four sends, each machine asked once, -// an answer, an ordinal they could count in, and NOT ONE router sentence on the -// screen. On the binary this design was written against it was nine identical -// sends to one machine over three minutes and then a turn that ended. +// an answer, a `switching` line NAMING the machine it is walking to, and NOT ONE +// router sentence on the screen. The ordinal belongs to the retry loop, which is +// not what moved here: a walk between MACHINES says where it is going. +// +// On the binary this design was written against it was nine identical sends to +// one machine over three minutes and then a turn that ended. func TestTheScreenshotScenarioAnswersThroughTheFourthMachine(t *testing.T) { pool := &scriptedPool{ roster: []string{"DeepInfra", "Io Net", "Fireworks", "GMICloud"}, @@ -236,13 +239,22 @@ func TestTheScreenshotScenarioAnswersThroughTheFourthMachine(t *testing.T) { t.Errorf("the walk paid %s of waiting for three moves to other machines: %v", believed, asked) } - // AND THE PERSON READ AN ORDINAL AND NO ROUTER SENTENCE. - moving, ok := told.find(PhaseRetrying) + // AND THE PERSON READ THE MOVE AND NO ROUTER SENTENCE. + // + // IT IS THE WALK'S OWN SENTENCE AND NOT THE RETRY LOOP'S. This test asked for + // [PhaseRetrying] and its ordinal until 2026-09-11, and never reached the + // assertion: the call it stages died on the first machine's 404 (the walk was + // bounded by a rolling allowance that had nothing to do with this question), + // so nobody noticed that a walk between MACHINES says [PhaseSwitching] and + // names the one it is going to. Naming the machine is the better sentence of + // the two and it is what ships; the ordinal belongs to the retry loop, which + // is not what moved here. + moving, ok := told.find(PhaseSwitching) if !ok { t.Fatal("a call that walked three refusing machines told the person nothing") } - if !strings.Contains(moving.Detail, " of ") { - t.Fatalf("the phase read %q, want an ordinal a person can count in", moving.Detail) + if moving.Then == "" { + t.Fatalf("the phase named no machine to switch to: %+v", moving) } for _, news := range told.all() { for _, machinery := range []string{"429", "404", "API error", "No allowed providers"} { diff --git a/internal/provider/e2e_account_real_test.go b/internal/provider/e2e_account_real_test.go index b45c3ecc10..120e9b2a8b 100644 --- a/internal/provider/e2e_account_real_test.go +++ b/internal/provider/e2e_account_real_test.go @@ -157,8 +157,6 @@ func TestRealRouterAccountExclusionIsPaidOnceForEveryModel(t *testing.T) { // the race of 2026-09-10, a turn twenty-five messages into a conversation // with a spending history. Six rescues in twenty and no share limit is what // the stub rig states for the same reason (hedge_test.go's newLaneRig). - provider.SetHedgeBudget(lanes.NewBudget(6, 0)) - t.Cleanup(func() { provider.SetHedgeBudget(nil) }) tally := &accountTally{inner: http.DefaultTransport} newClient := func() *provider.Client { @@ -283,8 +281,6 @@ func TestRealRouterTheMeasuredRaceLandsOnTheFirstAttempt(t *testing.T) { } lanes.ForgetRefusals() t.Cleanup(lanes.ForgetRefusals) - provider.SetHedgeBudget(lanes.NewBudget(6, 0)) - t.Cleanup(func() { provider.SetHedgeBudget(nil) }) tally := &accountTally{inner: http.DefaultTransport} client, err := provider.NewClient(provider.Config{ diff --git a/internal/provider/foldedladder_test.go b/internal/provider/foldedladder_test.go index 100355c083..b19c5429dc 100644 --- a/internal/provider/foldedladder_test.go +++ b/internal/provider/foldedladder_test.go @@ -135,7 +135,7 @@ func TestASecondArmCannotDemandAMachineTheFirstOneTried(t *testing.T) { plan := lanes.PlanFor(lanes.Choice{}, lanes.Pace{}, lanes.RoleTalk, time.Now()) plan.Lane = "DeepInfra" plan.Alts = []control.Alternative{{Lane: "Fireworks"}, {Lane: "GMICloud"}} - race := &hedgeRace{model: "openrouter/pool", winner: -1, plan: plan, budget: lanes.DefaultBudget()} + race := &hedgeRace{model: "openrouter/pool", winner: -1, plan: plan} // The first arm is on Fireworks — written down by the dispatcher under it, // which is the only writer the race can see and the point of a shared log. diff --git a/internal/provider/hedge.go b/internal/provider/hedge.go index 6ee3dfc0fb..e511057bef 100644 --- a/internal/provider/hedge.go +++ b/internal/provider/hedge.go @@ -33,9 +33,12 @@ import ( // alternative. A call with nowhere to go still has a ceiling, still reports // and still writes down what it did. // 2. MANY ARMS, ONE PURSE. A question may become more than two requests. What -// bounds it is money — [lanes.Budget], asked before every arm — and -// [maxArms] as the absolute cap on one question, never a boolean that says -// a rescue has already been spent. +// bounds it is money — THIS CALL'S OWN ([control.Plan.SpendUSD], asked +// through [hedgeRace.affordsLocked] before every arm) — and [maxArms] as the +// absolute cap on one question, never a boolean that says a rescue has +// already been spent. It was a rolling process-wide allowance until +// 2026-09-11 and that is the shape the purse must never take again: a count +// spread over twenty requests refuses by arrival order. // 3. THE PERSON HEARS ONE VOICE. Every arm streams, and one of them is ever // the SPEAKER. The others' deltas are held, and are replayed only if one of // them wins — after a plain notice, because text that was on the screen and @@ -168,7 +171,6 @@ type hedgeRace struct { observer StreamObserver report *HedgeReport phase *phaseClock - budget *lanes.Budget base context.Context messages []ai.Message options []ai.Option @@ -272,7 +274,6 @@ func (c *Client) raceFor(ctx context.Context, observer StreamObserver, build con observer: observer, report: HedgeReportFrom(ctx), phase: phaseClockFrom(ctx), - budget: currentHedgeBudget(), speaker: 0, winner: -1, held: map[int][]StreamEvent{}, @@ -469,9 +470,6 @@ func (r *hedgeRace) abandon(ctx context.Context, seen map[int]armResult) (*ai.Re // than the race actually spent. func (r *hedgeRace) accountForTheAbandoned(seen map[int]armResult) { r.drainArms(seen) - now := waitNow() - r.budget.NoteRequest(now) - r.budget.NoteSpend(r.spent(seen), now) } // drainArms collects what the arms report as they stop, for at most @@ -516,7 +514,7 @@ func (r *hedgeRace) startArm(index int, lane string, ladder []byte) { plan.Pinned = false } plan.Alts = r.untriedAlts() - watch := &streamWatch{race: r, arm: index, control: r.build(plan), began: now} + watch := &streamWatch{race: r, arm: index, plan: plan, control: r.build(plan), began: now} watch.deadline = watch.control.Deadline() if !watch.deadline.IsZero() { watch.armed = watch.deadline.Sub(now) @@ -670,10 +668,16 @@ func (r *hedgeRace) act(from int, act control.Act) { if r.rescueOnStall(from, act) { return } + // AND A REPORT IS STILL A MOVE. Nothing could be started for this + // question; the next one need not meet the same machine. + r.exhaust(from, act) // A FIRST PROMPT STILL OWES THE DOOR even when no second arm can // start. Saying nothing here is the 90s hang: the stream guard is - // the next thing that acts, and `/model` is never named. + // the next thing that acts, and `/model` is never named. The door is + // the NOTICE; the phase is [hedgeRace.theWait]'s either way, because a + // person reads the last word said and not the first. if r.tellFirstPrompt("", quietWords(act.Silence), false) { + r.theWait(act) return } r.tellTheWait(act) @@ -688,8 +692,123 @@ func (r *hedgeRace) act(from int, act control.Act) { } } -// hedge is another arm: budgeted, to a machine nobody has asked yet, and never -// past [maxArms]. +// exhaust is the move a live wire below the pace still has when no second +// request can be started, and the sentence that goes with it. +// +// ── A REPORT USED TO BE THE END OF THE STORY, AND IT IS NOT ───────────────── +// +// [control.Report] means "nothing can be started about this wait". Until +// 2026-09-11 that is all it did: the row said `action report`, and the question +// went on being answered by the machine that had earned the report, at the pace +// that earned it. The measured case is one endpoint writing 604 tokens in 86 +// seconds with a person watching a single nudge and nothing else. +// +// Two things are still true when nothing can be started, and this is both of +// them. +// +// THE MACHINE IS SPENT FOR THIS QUESTION. [control.Next] is the one move +// generator and it never hands back a machine already on the plan's move log, so +// writing the SERVING machine down there is the whole of "no arm and no attempt +// of this question goes back to it" — the same law and the same list a refusal +// already writes into (dispatch.go), asked of a machine that answered instead of +// one that refused. It is asked of [control.Next] rather than decided here: a +// question with nowhere else to go gets [control.MoveNone] and nothing is +// written, because taking away the only machine there is would be worse than the +// pace. +// +// THE PERSON IS TOLD SOMEWHERE ELSE. [PhaseBelowPace] is the word for a stream +// writing too slowly to read, and it is [hedgeRace.theWait]'s to say: this +// function used to say it too and was then overwritten one statement later by +// the phase [hedgeRace.tellTheWait] sets, so the person read the wrong sentence. +// One mechanism per shape — the move is this function's, the word is not. +func (r *hedgeRace) exhaust(from int, act control.Act) { + if r == nil || act.Reason != control.RateReason { + return + } + served := r.armServed(from) + if served == "" { + return + } + if move := control.Next(r.plan, r.plan.Moves.List()); move.Kind == control.MoveMachine { + r.claimMove(served) + } +} + +// armServed is the machine one arm's stream said was answering it, empty while +// nothing has named one — the attribution law, kept here as everywhere else. +func (r *hedgeRace) armServed(index int) string { + r.mu.Lock() + arm := r.armAt(index) + r.mu.Unlock() + if arm == nil { + return "" + } + return arm.watch.lane() +} + +// planCannotPay is what a row says when the one thing that stopped a rescue was +// the CALL'S OWN BUDGET: what this question may spend buying its wait back, +// which is its patience converted through λ ([control.Plan.SpendUSD]). +// +// IT REPLACED THE WORD `budget`, WHICH NAMED THE WRONG THING. That word meant a +// rolling process-wide allowance — two rescues in any twenty requests — so a row +// carrying it said "some other request spent this one's rescue", which is a fact +// about arrival order and not about this call at all. There is no such +// allowance any more (internal/lane's hedge.go), and a refusal that reaches a +// row now names the only rail there is. +const planCannotPay = "plan cannot pay" + +// affords reports whether this call's own budget can pay for one more arm on +// this machine. +// +// IT IS THE ONE DOOR, asked by the controller's hedge, by the stall rescue, by +// the walk past a refusal and by the surface before it draws a countdown over a +// rescue — because a promise the rail was always going to refuse is the surface +// lying about the machinery ([hedgeRace.affordableAlt]). It ASKS AND DOES NOT +// SPEND ([lane.Spending]), so asking it twice over one silence costs nothing and +// no question is charged for an arm it never started. +func (r *hedgeRace) affords(alt string) bool { + r.mu.Lock() + defer r.mu.Unlock() + return r.affordsLocked(alt) +} + +// affordsLocked is the same reading for a caller already holding the lock. A +// plan with no purse is UNBOUNDED and not empty: nobody priced it, which is the +// state of every plan a test or a bench builds by hand. +func (r *hedgeRace) affordsLocked(alt string) bool { + if r.plan.Purse == nil { + return true + } + return r.plan.Purse.Allows(r.altPrice(alt), waitNow()) +} + +// altPrice is what THE PLAN says one more arm on this machine adds to the bill. +// +// IT IS THE NUMBER THE CONTROLLER ASKS THE PURSE WITH, and that is the whole +// reason it is read off the plan rather than worked out again here. +// [control.Alternative.Extra] is priced once, where the plan is built +// (internal/lane's watch.go), from the frontier entry this race would send to; +// [hazard.reachable] asks the purse with it before it decides whether there is a +// move, and [hedgeRace.affordsLocked] asks the purse with it before the arm +// leaves. One rail, one price, two readers — and two estimates for one bound is +// how a rail and the decision in front of it come to disagree about what this +// question can afford, which is the shape the deleted allowance had. +// +// A MACHINE THE PLAN NEVER NAMED HAS NO PRICE ON IT, and the frontier's own +// estimate is the honest answer there rather than a refusal: that is the walk +// past a refusal, which sends somewhere the controller was never offered. +func (r *hedgeRace) altPrice(alt string) float64 { + for _, candidate := range r.plan.Alts { + if equalLane(candidate.Lane, alt) { + return candidate.Extra + } + } + return r.estimateLocked(alt, r.expected) +} + +// hedge is another arm: paid for out of the call's own budget, to a machine +// nobody has asked yet, and never past [maxArms]. func (r *hedgeRace) hedge(from int, act control.Act, alt string) { var why string wanted := alt @@ -699,13 +818,13 @@ func (r *hedgeRace) hedge(from int, act control.Act, alt string) { r.recordHedge(wanted, why, false) return } - if !r.budget.Allow(waitNow(), r.estimate(alt, r.expected)) { + if !r.affords(alt) { r.mu.Lock() r.refused = true - r.rememberRefusalLocked("budget") + r.rememberRefusalLocked(planCannotPay) r.releaseMove(alt) r.mu.Unlock() - r.recordHedge(alt, "budget", false) + r.recordHedge(alt, planCannotPay, false) return } r.mu.Lock() @@ -762,11 +881,11 @@ func (r *hedgeRace) recordHedge(alt, reason string, fired bool) { // rescueOnStall starts the second arm a stall is owed, without waiting for // the primary to die. // -// THE PURSE STILL GATES THIS. A wait at the ceiling is the role's bound, but -// answering it with another request is spending and follows the same law as -// every other hedge. claim is asked `past` so an earlier refusal may be -// reconsidered if the rolling budget has since opened; [lanes.Budget.Allow] -// makes the decision again at the moment the rescue would start. +// THE CALL'S OWN BUDGET STILL GATES THIS. A wait at the ceiling is the role's +// bound, but answering it with another request is spending and follows the same +// law as every other hedge. claim is asked `past` so an earlier refusal may be +// reconsidered — a machine that has sat past the ceiling without dying must not +// go unanswered because a slowness hedge was refused a moment earlier. func (r *hedgeRace) rescueOnStall(from int, act control.Act) bool { if r == nil || r.base == nil || r.base.Err() != nil { return false @@ -776,10 +895,10 @@ func (r *hedgeRace) rescueOnStall(from int, act control.Act) bool { r.rememberRefusal(why) return false } - if !r.budget.Allow(waitNow(), r.estimate(alt, r.expected)) { + if !r.affords(alt) { r.mu.Lock() r.refused = true - r.rememberRefusalLocked("budget") + r.rememberRefusalLocked(planCannotPay) r.releaseMove(alt) r.mu.Unlock() return false @@ -844,9 +963,13 @@ func (r *hedgeRace) tellFirstPrompt(alt, quiet string, rescuing bool) bool { if strings.TrimSpace(alt) == "" { r.phase.switching("/model", quiet) } - } else { - r.phase.allSlow("") } + // AND IT DOES NOT SAY THE WAIT'S OWN WORD. It used to set [PhaseAllSlow] + // here on the non-rescuing branch, which is the same word + // [hedgeRace.theWait] is about to choose and, whenever the wait was a live + // wire below its pace, the WRONG one — said second, so it is the one left on + // the screen. One decision, one place: this function owes the notice and the + // `/model` door, and the phase belongs to whoever knows why the wait exists. if r.observer != nil { r.observer(StreamEvent{Kind: StreamNotice, Delta: notice, Session: session}) } @@ -889,7 +1012,7 @@ func (r *hedgeRace) claim(preferred string, past bool) (lane, why string) { // caller stops at a refusal, which is what keeps one refusal from // becoming a poll. if r.refused && !past { - return "", "budget" + return "", planCannotPay } // AND THE RESERVATION IS A MOVE ON THE PLAN. [control.MoveLog.Add] answers // false for a machine this question has already taken, so the claim and the @@ -1051,14 +1174,45 @@ func (r *hedgeRace) tellTheWait(act control.Act) { already := r.reported r.reported = true r.mu.Unlock() - if already { + // THE ROW IS WRITTEN ONCE AND THE WORD IS NOT. What may not be repeated is + // the finish row's account of the FIRST thing that acted on this question — + // a second report overwriting it would lose the one the person waited + // through. The sentence on the screen is the opposite: it describes the wait + // that is happening NOW, so a wait whose character changes owes a new word. + // Behind the latch it was neither said again nor said at all, which is how a + // rate report that arrived after any earlier report left the person reading + // whatever the earlier one had said. + if !already { + r.report.note(func(report *HedgeReport) { + if report.action == "" { + report.action, report.silence, report.reason = actionWord(act.Kind), act.Silence, act.Reason + } + }) + } + r.theWait(act) +} + +// theWait is THE ONE PLACE THAT CHOOSES THE WORD for a wait nothing can be done +// about, and there are two of them because there are two waits. +// +// A person reads the LAST thing said, not every thing said. Until 2026-09-11 +// this decision was made twice on one path — [hedgeRace.exhaust] said +// [PhaseBelowPace] and then [hedgeRace.tellFirstPrompt] or [hedgeRace.tellTheWait] +// said [PhaseAllSlow] over the top of it a statement later — so the sentence +// left on the screen was `all lanes slow · still waiting` for a stream that was +// visibly writing. That is the sentence [PhaseBelowPace] exists to replace, and +// a test that scans every phase ever emitted cannot tell the two apart. +// +// THE REASON PICKS IT, because the reason is the difference. A rate report is a +// wire that IS answering and answering too slowly to read, and there is text on +// the screen to prove it; everything else that reaches here — a drift report, an +// escalate, a silence with no lane left — is a wait with nothing arriving, which +// is what `all lanes slow · still waiting` describes. +func (r *hedgeRace) theWait(act control.Act) { + if act.Kind == control.Report && act.Reason == control.RateReason { + r.phase.belowPace("") return } - r.report.note(func(report *HedgeReport) { - if report.action == "" { - report.action, report.silence, report.reason = actionWord(act.Kind), act.Silence, act.Reason - } - }) r.phase.allSlow(r.waitWords(act)) } @@ -1111,7 +1265,7 @@ func (r *hedgeRace) affordableAlt() (string, bool) { if alt == "" || refused || full { return "", false } - return alt, r.budget.Affordable(waitNow(), r.estimate(alt, r.expected)) + return alt, r.affords(alt) } // walk is the second rung: the rescue we sent was refused or broke, nobody has @@ -1122,7 +1276,8 @@ func (r *hedgeRace) affordableAlt() (string, bool) { // by the controller; this is a response to a lane that has FAILED, and the // request it replaces is already gone. That is why it is not gated by the // purse's earlier refusal — one refusal must not spend the rescue a slow lane -// is still owed — and why it is gated by the budget and by [maxArms] instead. +// is still owed — and why it is gated by the call's own budget and by [maxArms] +// instead. // // THE PRIMARY IS NEVER WALKED FROM. An arm that is still streaming has not // failed, and the controller is the only thing allowed to give up on it. @@ -1158,7 +1313,7 @@ func (r *hedgeRace) walk(from int, cause error) { r.mu.Lock() index := len(r.arms) r.mu.Unlock() - if !r.budget.Allow(waitNow(), r.estimate(alt, r.expected)) { + if !r.affords(alt) { return } r.report.started(refusal.news(alt)) @@ -1414,9 +1569,12 @@ func (r *hedgeRace) emit(arm int, event StreamEvent) { // // IT READS THE SAME LANE THE WALK WILL CLAIM. A name in the frontier is not a // walk when the wire has since struck that lane, the arm cap is full, or the -// purse will refuse its estimated cost. Affordable is only a reading here; -// [hedgeRace.walk] still reserves the spend through [lanes.Budget.Allow] at the -// moment it starts the arm. +// purse will refuse its price. IT IS ONLY A READING, AND NOTHING RESERVES +// ANYTHING. The purse asks and never spends ([lane.Spending]), so this reading +// and the one [hedgeRace.walk] takes when it really starts the arm are the same +// question asked twice at no cost — which is the single property the whole rail +// now rests on, and the reason no question is ever charged for an arm it did +// not send. // // THE WALK DELIBERATELY IGNORES r.refused. A refusal to fund a hedge against // slowness does not deny the rescue owed after a lane actually fails, which is @@ -1432,7 +1590,7 @@ func (r *hedgeRace) walkAvailable() bool { if lane == "" || r.plan.Moves.Tried(lane) || !lanes.Serves(r.model, lane) { continue } - return r.budget.Affordable(waitNow(), r.estimateLocked(lane, r.expected)) + return r.affordsLocked(lane) } return false } @@ -1501,7 +1659,7 @@ func (r *hedgeRace) takeRefusal(body []byte) bool { if lane == "" || r.plan.Moves.Tried(lane) || !lanes.Serves(r.model, lane) { continue } - if !r.budget.Affordable(waitNow(), r.estimateLocked(lane, r.expected)) { + if !r.affordsLocked(lane) { return false } r.ladderOwed = true @@ -1674,7 +1832,6 @@ func (r *hedgeRace) settle(result armResult, seen map[int]armResult) (*ai.Respon r.withdraw() r.mu.Lock() arms := append([]*hedgeArm(nil), r.arms...) - hedged := len(arms) > 1 model := r.model r.mu.Unlock() // The model as the ANSWER spelled it, because a router may pin a variant @@ -1717,16 +1874,15 @@ func (r *hedgeRace) settle(result armResult, seen map[int]armResult) (*ai.Respon lanes.Default().Ledger().Note(sighting) } } - // The two denominators of the budget: every dollar this adapter is seen to - // spend, and every request it is seen to make. Both are noted on every - // watched call, hedged or not — the rate limit is counted in REQUESTS rather - // than in minutes, so a request that never reports leaves the allowance - // looking emptier than it is. - now := waitNow() - r.budget.NoteRequest(now) - r.budget.NoteSpend(r.spent(seen), now) + // AND NOTHING IS BILLED TO A WINDOW. There were two rolling denominators here + // — every dollar this adapter was seen to spend and every request it was seen + // to make — and both belonged to a process-wide allowance that no longer + // exists (internal/lane's hedge.go). What a call may spend is its own budget, + // asked before an arm goes out and never tallied afterwards, so the only money + // this loop still adds up is what the ROW says: the waste, which is a figure a + // person reads and not a rail anything is decided on. winner, loser, primary := "", "", "" - var waste, second float64 + var waste float64 for _, arm := range arms { if arm.index == result.index { winner = laneOf(arm) @@ -1743,16 +1899,6 @@ func (r *hedgeRace) settle(result armResult, seen map[int]armResult) (*ai.Respon if arm.index == 0 { primary = laneOf(arm) } - if arm.index > 0 { - // EVERY RESCUE IS CHARGED, not only the first one. The walk can put - // a third and a fourth request on the wire, and a budget that only - // ever saw the second would let a walk spend the session's whole - // share while believing it had spent one arm's. - second += r.armCost(seen, arm) - } - } - if hedged { - r.budget.NoteHedge(second, now) } r.report.note(func(report *HedgeReport) { report.winner, report.loser, report.primary = winner, loser, primary diff --git a/internal/provider/hedge_abandon_test.go b/internal/provider/hedge_abandon_test.go index 95a25cd568..77fea92a4d 100644 --- a/internal/provider/hedge_abandon_test.go +++ b/internal/provider/hedge_abandon_test.go @@ -19,8 +19,6 @@ import ( "errors" "testing" "time" - - lanes "github.com/Agent-Field/aforge-v2/internal/lane" ) func TestACancelledRaceAnswersAtOnceAndDrainsItsArmsBehindTheCaller(t *testing.T) { @@ -29,7 +27,6 @@ func TestACancelledRaceAnswersAtOnceAndDrainsItsArmsBehindTheCaller(t *testing.T // the whole second — which is exactly the arm [abandonGrace] was written // for, and exactly the one a caller must not be behind. race := &hedgeRace{ - budget: lanes.DefaultBudget(), results: make(chan armResult, 2), arms: []*hedgeArm{{index: 0}, {index: 1}}, winner: -1, diff --git a/internal/provider/hedge_firstprompt_test.go b/internal/provider/hedge_firstprompt_test.go index c11da8ce98..caf83bfd72 100644 --- a/internal/provider/hedge_firstprompt_test.go +++ b/internal/provider/hedge_firstprompt_test.go @@ -5,7 +5,6 @@ import ( "testing" "time" - lanes "github.com/Agent-Field/aforge-v2/internal/lane" "github.com/Agent-Field/aforge-v2/internal/lane/lanestub" ) @@ -26,7 +25,6 @@ func TestAStalledFirstPromptLaneShowsARescueSwitch(t *testing.T) { ) // THE LEDGER IS LEFT EMPTY. That is a first prompt: nothing believed, the // shipped default still on the talk slot. - SetHedgeBudget(lanes.NewBudget(1, 0)) rig.patience(t, 150*time.Millisecond) watched := ¬ices{} diff --git a/internal/provider/hedge_refusal_test.go b/internal/provider/hedge_refusal_test.go index 2221040eba..fa5e5429e4 100644 --- a/internal/provider/hedge_refusal_test.go +++ b/internal/provider/hedge_refusal_test.go @@ -4,15 +4,17 @@ import ( "testing" "time" - lanes "github.com/Agent-Field/aforge-v2/internal/lane" "github.com/Agent-Field/aforge-v2/internal/lane/lanestub" ) -// TestARescueTheAllowanceRefusedSaysSoOnTheRow is C4: the row records both -// that the wait was reported and that the second request the ceiling still -// owed was refused for money. Those are different facts, which is why the -// refusal has its own field. -func TestARescueTheAllowanceRefusedSaysSoOnTheRow(t *testing.T) { +// TestARescueTheCallCouldNotPayForSaysSoOnTheRow is C4: the row records both +// that the wait was reported and that the second request the ceiling still owed +// never went out. Those are different facts, which is why the refusal has its +// own field — and the WORD is the rail that really refused. It used to be +// `budget`, which named a rolling process-wide allowance and therefore said +// "some other request spent this one's rescue"; [planCannotPay] names the only +// rail there is. +func TestARescueTheCallCouldNotPayForSaysSoOnTheRow(t *testing.T) { read := loggingTo(t) rig := newLaneRig(t, "row/budget-refusal", lanestub.Lane{Name: "A", Profile: lanestub.Profile{TTFT: 60 * time.Millisecond, Rate: 2000, Tokens: 24}}, @@ -20,7 +22,7 @@ func TestARescueTheAllowanceRefusedSaysSoOnTheRow(t *testing.T) { ) rig.believes("A", 20, 2000) rig.patience(t, 50*time.Millisecond) - SetHedgeBudget(lanes.NewBudget(0, 0)) + noRescues(t) ctx := WithLaneChoice(talking(), choiceFor(rig.model, 12*time.Millisecond)) if _, err := rig.client.CompleteWithMessages(ctx, userMessages("hello")); err != nil { @@ -37,8 +39,8 @@ func TestARescueTheAllowanceRefusedSaysSoOnTheRow(t *testing.T) { if got := rows[0].Action; got != "report" { t.Fatalf("action = %q, want the controller's report", got) } - if got := rows[0].Refused; got != "budget" { - t.Fatalf("refused = %q, want budget", got) + if got := rows[0].Refused; got != planCannotPay { + t.Fatalf("refused = %q, want %q", got, planCannotPay) } } diff --git a/internal/provider/hedge_stall_test.go b/internal/provider/hedge_stall_test.go index 45fff1901a..85e5d0a91c 100644 --- a/internal/provider/hedge_stall_test.go +++ b/internal/provider/hedge_stall_test.go @@ -1,6 +1,7 @@ package provider import ( + "context" "testing" "time" @@ -27,7 +28,6 @@ func TestAStalledLaneArmsARescueWithoutATerminalError(t *testing.T) { ) // THE PURSE ALLOWS ONE RESCUE, so keepalives cannot turn a live but // stalled stream into an unbounded wait. - SetHedgeBudget(lanes.NewBudget(1, 0)) rig.patience(t, 150*time.Millisecond) report := &HedgeReport{} @@ -55,3 +55,104 @@ func TestAStalledLaneArmsARescueWithoutATerminalError(t *testing.T) { } waitFor(t, func() bool { return rig.server.Cancels("A") == 1 }) } + +// ── A WIRE THAT IS WRITING AND STILL LOSING THE PERSON'S AFTERNOON ────────── + +// slowWireRig stages the 2026-09-11 call on the wire: a machine the choice +// expected sixty tokens a second from, delivering seven, beside one that is +// believed fast and is. +func slowWireRig(t *testing.T, name string) *laneRig { + t.Helper() + rig := newLaneRig(t, name, + lanestub.Lane{Name: "A", Profile: lanestub.Profile{ + TTFT: 40 * time.Millisecond, Rate: 7, Tokens: 40, Heartbeats: true, + }}, + lanestub.Lane{Name: "B", Profile: lanestub.Profile{TTFT: 5 * time.Millisecond, Rate: 2000, Tokens: 24}}, + ) + // What the choice expected of the machine it named, which is the bar the + // stream is held to. + rig.believes("A", 40, 60) + rig.believes("B", 5, 2000) + rig.patience(t, 150*time.Millisecond) + return rig +} + +// unattended is the role of a task node nobody is sitting in front of: it still +// has a λ ([lane.UnattendedValue]), because a task still ends when its slowest +// call ends. +func unattended() context.Context { + return WithRole(context.Background(), lanes.RoleLeafUnattended) +} + +// TestAWireBelowItsPromisedPaceIsRescuedOnAnUnattendedTask is the owner's own +// call, staged. The machine writes — there is no silence and no error — and it +// writes an eighth as fast as the machine the question was sent to. The rescue +// costs about what the call was going to cost anyway, and the call's own budget +// affords it; under the deleted window it did not, and a person watched 604 +// tokens arrive over 86 seconds. +func TestAWireBelowItsPromisedPaceIsRescuedOnAnUnattendedTask(t *testing.T) { + rig := slowWireRig(t, "slowwire/rescued") + + report := &HedgeReport{} + ctx := WithHedgeReport(unattended(), report) + ctx = WithLaneChoice(ctx, choiceFor(rig.model, 0)) + + response, err := rig.client.CompleteWithMessages(ctx, userMessages("hello")) + if err != nil { + t.Fatal(err) + } + if !report.Hedged() { + t.Fatal("a wire writing at an eighth of the pace it was asked for was never rescued") + } + if winner, _ := report.Lanes(); winner != "B" { + t.Fatalf("winner = %q, want the answer from the machine that kept the pace", winner) + } + if tokens := answerTokens(response); tokens != 24 { + t.Fatalf("the answer is %d tokens, want B's 24", tokens) + } + if got := rig.server.Requests("B"); got != 1 { + t.Fatalf("Requests(B) = %d, want the one rescue the pace earned", got) + } +} + +// TestAWireBelowItsPaceWithNothingToSpendSaysSoAndIsSpent is the other half of +// the same call: nothing can be started, and a report is still a move. +// +// THREE THINGS ARE ASSERTED BECAUSE THREE THINGS USED TO BE MISSING. The row +// names the rail that really refused — the call's own budget, not some other +// request's allowance. The person is told the answer is arriving too slowly to +// read, which neither `writing` nor `all lanes slow` could honestly say. And the +// machine is written down as spent for this question, so no arm and no attempt +// of it goes back to the machine that earned the report. +func TestAWireBelowItsPaceWithNothingToSpendSaysSoAndIsSpent(t *testing.T) { + read := loggingTo(t) + rig := slowWireRig(t, "slowwire/nothing-to-spend") + noRescues(t) + told := listen(t) + + ctx := WithLaneChoice(unattended(), choiceFor(rig.model, 0)) + if _, err := rig.client.CompleteWithMessages(ctx, userMessages("hello")); err != nil { + t.Fatal(err) + } + if got := rig.server.Requests("B"); got != 0 { + t.Fatalf("Requests(B) = %d, want no rescue from a call that may spend nothing", got) + } + rows := ended(read()) + if len(rows) != 1 { + t.Fatalf("ended rows = %d, want the one call's row", len(rows)) + } + if got := rows[0].Refused; got != planCannotPay { + t.Fatalf("refused = %q, want %q — the rail that really refused", got, planCannotPay) + } + // AND IT IS THE LAST WORD THAT IS ASSERTED, not one somewhere in the list. + // A phase said and then overwritten is a phase nobody read: this assertion + // was `told.find(PhaseBelowPace)` and it passed while [hedgeRace.exhaust] + // said `answering slowly` and the next statement said `all lanes slow` over + // the top of it. See [heard.settled]. + if got := told.settled(); got != PhaseBelowPace { + for _, news := range told.all() { + t.Logf("PHASE %q detail=%q", news.Phase, news.Detail) + } + t.Fatalf("a person watching an answer crawl was left reading %q, want %q", got, PhaseBelowPace) + } +} diff --git a/internal/provider/hedge_terminal_test.go b/internal/provider/hedge_terminal_test.go index 6e4b96dc58..b04ff6068d 100644 --- a/internal/provider/hedge_terminal_test.go +++ b/internal/provider/hedge_terminal_test.go @@ -30,7 +30,6 @@ func TestALaterAcceptedStreamCutOutranksThePrimaryRoutingRefusal(t *testing.T) { PriceIn: 1.32e-6, PriceOut: 3.96e-6, }}, ) - SetHedgeBudget(lanes.NewBudget(6, 0)) choice := lanes.Choice{ Order: []string{"A", "B", "C"}, Frontier: []lanes.Scored{ diff --git a/internal/provider/hedge_test.go b/internal/provider/hedge_test.go index c9976319f6..b034e09efd 100644 --- a/internal/provider/hedge_test.go +++ b/internal/provider/hedge_test.go @@ -141,7 +141,6 @@ func newLaneRigWithPrice( ledger := &scriptedLedger{beliefs: map[lanes.ID]lanes.Belief{}} registry := lanes.Default() registry.SetLedger(ledger) - SetHedgeBudget(lanes.NewBudget(6, 0)) // THE CONTROLLER IS INSTALLED THROUGH THE ONE SEAM, exactly as a shipped // build installs it, so that what these tests exercise is the wiring and // not a second arrangement built for them. @@ -172,7 +171,6 @@ func newLaneRigWithPrice( // all. lanes.SetController(shipped) rigLanes.Delete(model) - SetHedgeBudget(nil) // The package's learners go back too, beside the registry and the // controller: see resetSharedLearners for why they are a restoration // and not an extra. @@ -576,7 +574,7 @@ func TestAnExhaustedBudgetRefusesTheRescueAndTheAnswerArrivesLate(t *testing.T) ) rig.believes("A", 20, 2000) // A budget with no bucket is how hedging is switched off. - SetHedgeBudget(lanes.NewBudget(0, 0)) + noRescues(t) report := &HedgeReport{} ctx := WithHedgeReport(talking(), report) @@ -1335,3 +1333,17 @@ func TestARefusalOnlyNamesWhatTheQuestionWasLeftWithout(t *testing.T) { } }) } + +// noRescues switches the second request off for one test and puts it back +// afterwards. +// +// IT IS THE PERSON'S OWN SWITCH and not a rail of its own ([SetLaneGuard]), +// because there is no process-wide allowance left to zero: what a call may spend +// rescuing itself is its own budget, and the only thing that can refuse every +// rescue outright is somebody saying so. +func noRescues(t *testing.T) { + t.Helper() + before := LaneGuardOn() + t.Cleanup(func() { SetLaneGuard(before) }) + SetLaneGuard(false) +} diff --git a/internal/provider/lanepin.go b/internal/provider/lanepin.go index 029d73ca08..96a7ff45d5 100644 --- a/internal/provider/lanepin.go +++ b/internal/provider/lanepin.go @@ -15,11 +15,13 @@ import ( // `lane.` row and the `lane.guard` row (internal/config's settings.go), // resolved once by the surface and handed down. // -// IT IS A PROCESS-WIDE KNOB AND NOT A FIELD ON Config, for [SetHedgeBudget]'s -// reason and one more. The reason it shares: the row is about a SESSION and not -// about an adapter, and two clients in one process — the conversation's and a -// tool loop's own — must not hold two different answers to "which machine did -// they ask for". The reason of its own: the row is written WHILE THE PROCESS IS +// IT IS A PROCESS-WIDE KNOB AND NOT A FIELD ON Config, for two reasons. The row +// is about a SESSION and not about an adapter, and two clients in one process — +// the conversation's and a tool loop's own — must not hold two different answers +// to "which machine did they ask for". (The other process-wide knob this used to +// cite, `SetHedgeBudget`, is deleted: what a call may spend rescuing itself is a +// figure on its own plan now and not a switch anybody throws.) And the row is +// written WHILE THE PROCESS IS // RUNNING, by the picker (internal/tui3's pinLane), and a pin that only took // effect at the next launch would be a promise this build did not keep. A // settings read per request would keep it too — at the cost of a disk read in @@ -156,25 +158,21 @@ func PinnedFor(model string) string { } // SetLaneGuard turns the speed guard on or off, and it is the ONE switch: it -// moves the hedge budget and the probe together, because both are the same -// promise to a person — that this build may spend a little extra to keep an -// answer moving — and a row that turned off half of it would be a row nobody -// could reason about. +// moves the rescue and the probe together, because both are the same promise to +// a person — that this build may spend a little extra to keep an answer moving — +// and a row that turned off half of it would be a row nobody could reason about. // -// OFF IS A BUDGET THAT ALLOWS NOTHING rather than a flag the race consults. The -// budget is already the one gate every hedge passes through ([hedgeRace.hedge]), -// so a zero allowance is the whole of "do not rescue" with no second path to -// keep in step. The probe reads the flag directly, because a probe is not -// budgeted in dollars — it is gated on whether anybody is waiting. +// OFF IS A PURSE THAT REFUSES EVERYTHING rather than a flag the race consults. +// The purse is already the one gate every rescue passes through +// ([hedgeRace.affords]), so a purse that says no is the whole of "do not rescue" +// with no second path to keep in step — and it is written onto the plan where +// every arm of the question reads it (waitplan.go). The probe reads the flag +// directly, because a probe is not priced in dollars: it is gated on whether +// anybody is waiting. func SetLaneGuard(on bool) { lanePinMu.Lock() laneGuard = on lanePinMu.Unlock() - if on { - SetHedgeBudget(nil) - return - } - SetHedgeBudget(lanes.NewBudget(0, 0)) } // LaneGuardOn reports whether the speed guard is on. diff --git a/internal/provider/lanepin_test.go b/internal/provider/lanepin_test.go index a25fc40e68..68773e1002 100644 --- a/internal/provider/lanepin_test.go +++ b/internal/provider/lanepin_test.go @@ -186,8 +186,8 @@ func TestTheGuardOffRefusesEveryHedgeAndEveryProbe(t *testing.T) { t.Cleanup(func() { SetLaneGuard(before) }) SetLaneGuard(false) - if currentHedgeBudget().Allow(client.clock(), 0.0001) { - t.Fatal("the budget allowed a rescue with the guard off") + if plan := client.planFor(context.Background(), lanes.Choice{}, model, 0); plan.Purse.Allows(0.0001, client.clock()) { + t.Fatal("the purse allowed a rescue with the guard off") } if !InstallLaneProber(client, nil) { t.Fatal("the prober refused a router client") @@ -198,10 +198,10 @@ func TestTheGuardOffRefusesEveryHedgeAndEveryProbe(t *testing.T) { t.Fatalf("%d requests went out with the guard off, want none", got) } - // And back on, the same budget answers the same question the other way. + // And back on, the same plan answers the same question the other way. SetLaneGuard(true) - if !currentHedgeBudget().Allow(client.clock(), 0.0001) { - t.Fatal("the budget refused a rescue with the guard on") + if plan := client.planFor(context.Background(), lanes.Choice{}, model, 0); !plan.Purse.Allows(0.0001, client.clock()) { + t.Fatal("the purse refused a rescue with the guard on") } } diff --git a/internal/provider/phase.go b/internal/provider/phase.go index c9602e3ce1..14c510cdc1 100644 --- a/internal/provider/phase.go +++ b/internal/provider/phase.go @@ -105,6 +105,23 @@ const ( // exactly where it was and the surface goes on counting up from the moment // the wait began. PhaseAllSlow Phase = "all lanes slow" + // PhaseBelowPace is the other half of [control.Report], and it is the half + // this build used to say nothing about: the endpoint IS writing, and it is + // writing too slowly to be worth reading, and no second machine can be + // started to fix it. + // + // IT IS NOT [PhaseAllSlow] AND IT IS NOT [PhaseWriting]. A person told "all + // lanes slow · still waiting" while words are appearing is being told about a + // silence they can see is not happening; a person told "writing" while 604 + // tokens take 86 seconds is being told about a stream that is technically + // alive and practically stopped. On 2026-09-11 that exact call showed a + // person one nudge and then nothing for eighty-six seconds, because neither + // of the two existing words was true and the honest third one did not exist. + // + // IT IS NOT A WAIT IN [PhaseNews.Waiting]'s sense, deliberately: the answer is + // arriving, so the surface goes on drawing it and this word sits beside it + // rather than in place of it. + PhaseBelowPace Phase = "below pace" // PhaseSwitchingModel is the LAST rung of the ladder and the only one that // changes what a person asked for: every lane of the model has been tried // and a fallback model is being asked instead. Then names it. @@ -392,6 +409,27 @@ func (p *phaseClock) enter(phase Phase, detail string) { if p.phase == phase { return } + // AND THE WRITING MAY NOT TAKE THE WORD BACK FROM [PhaseBelowPace], because + // they are the same state and one of them is the true sentence about it. + // + // A stream below its pace IS writing — that is what makes it different from + // a silence — so the next token it produces arrives here and, without this, + // re-enters [PhaseWriting] and the judgement is gone. Measured while fixing + // #924: `answering slowly · nowhere faster` lasted exactly ONE token of the + // eighty-six seconds it was about, and the person went back to reading + // `writing` for the rest of it. + // + // IT LASTS AS LONG AS THE JUDGEMENT DOES, WHICH IS THIS ATTEMPT. The + // controller reports a below-pace wire once per controller and there is one + // controller per attempt ([streamWatch.attempt]), because nothing that + // happens inside an attempt makes "this wire is slow and there is nowhere + // faster" untrue. The next attempt opens on [PhaseConnecting] or + // [PhaseFirstWord] and clears it through this same door, as does a thought, + // a switch, a silence and the end of the answer — every phase except the one + // this stream was already in. + if p.phase == PhaseBelowPace && phase == PhaseWriting { + return + } now := p.now() p.phase, p.since, p.tokens = phase, now, 0 // A CONSEQUENCE BELONGS TO THE PHASE THAT EARNED IT. The hedge deadline is @@ -613,6 +651,22 @@ func (p *phaseClock) allSlow(detail string) { p.say(detail, now) } +// belowPace says this stream is writing too slowly to be worth reading and +// nothing can be started about it ([PhaseBelowPace]). +// +// It keeps the moment for the same reason [phaseClock.allSlow] does: nothing has +// changed about what the endpoint is doing, so the count a person is reading goes +// on counting from where it was. +func (p *phaseClock) belowPace(detail string) { + if p == nil { + return + } + p.mu.Lock() + defer p.mu.Unlock() + p.phase = PhaseBelowPace + p.say(detail, p.now()) +} + type phaseClockContextKey struct{} // withPhaseClock puts one request's clock where every layer of it can find the diff --git a/internal/provider/phase_test.go b/internal/provider/phase_test.go index ac461a8e41..6a82d57d21 100644 --- a/internal/provider/phase_test.go +++ b/internal/provider/phase_test.go @@ -53,6 +53,28 @@ func (h *heard) story() []Phase { return out } +// settled is the LAST phase a person was left reading, empty when nothing was +// said at all. +// +// IT IS A DIFFERENT QUESTION FROM [heard.find], and the difference is the whole +// point of having both. `find` answers "was this ever said", which a phase set +// and then overwritten a microsecond later satisfies perfectly — that is exactly +// how `answering slowly` passed a test while every person saw `all lanes slow` +// (the 2026-09-11 review of #924). A claim about what somebody READ has to be a +// claim about the last word. +// The empty phase [phaseClock.done] ends every story with is not a sentence +// anybody reads — it is the clock stopping — so the last WORD is the last +// non-empty one. +func (h *heard) settled() Phase { + story := h.story() + for i := len(story) - 1; i >= 0; i-- { + if story[i] != "" { + return story[i] + } + } + return "" +} + // find is the first news in a phase, and whether there was one. func (h *heard) find(phase Phase) (PhaseNews, bool) { for _, news := range h.all() { @@ -294,7 +316,7 @@ func TestNoCountdownIsDrawnOverARescueNobodyCanAfford(t *testing.T) { ) rig.believes("A", 20, 2000) // A budget with no bucket is how the speed guard is switched off. - SetHedgeBudget(lanes.NewBudget(0, 0)) + noRescues(t) ctx := WithLaneChoice(talking(), choiceFor(rig.model, 12*time.Millisecond)) if _, err := rig.client.CompleteWithMessages(ctx, userMessages("hello")); err != nil { diff --git a/internal/provider/purse_law_test.go b/internal/provider/purse_law_test.go new file mode 100644 index 0000000000..2ac29b6fbd --- /dev/null +++ b/internal/provider/purse_law_test.go @@ -0,0 +1,270 @@ +package provider + +import ( + "go/ast" + "go/parser" + "go/token" + "os" + "path/filepath" + "strings" + "testing" + "time" + + lanes "github.com/Agent-Field/aforge-v2/internal/lane" + "github.com/Agent-Field/aforge-v2/internal/lane/control" +) + +// TestThePurseIsAskedThroughOneDoor is the law the deleted allowance broke. +// +// THERE IS ONE RAIL IN FRONT OF A SECOND REQUEST, ONE PRICE, AND TWO PLACES +// THAT ASK IT — one per side of the decision, and no more. +// +// What made the process-wide budget able to overrule the controller was that it +// was asked from five places under three different names — the controller's +// hedge, the stall rescue, the walk, the countdown's reading and the ladder's — +// each with its own idea of what a refusal meant. Two doors survive that, and +// they are not two mechanisms: +// +// - [hazard.reachable] asks BEFORE DECIDING. A move the purse will refuse is +// not a move, so a controller that did not ask would hand back a hedge that +// cannot happen and the request would sit through a rescue nobody could pay +// for — see THE PURSE IS ASKED LAST in hazard.go's act. +// - [hedgeRace.affordsLocked] asks BEFORE SENDING, because the controller is +// not the only thing that starts an arm: the stall rescue, the walk past a +// refusal and the surface's own countdown all reach the wire through here. +// +// AND THEY ASK WITH THE SAME NUMBER, which is the half that makes them one +// mechanism rather than two. [Alternative.Extra] is priced once where the plan +// is built and both sides read it ([hedgeRace.altPrice]); two estimates for one +// bound is how the rail and the decision come to disagree about what this +// question can afford. +// +// This walks the source of both packages and fails on any third asker. +func TestThePurseIsAskedThroughOneDoor(t *testing.T) { + doors := map[string]string{ + "affordsLocked": "asks before sending", + "reachable": "asks before deciding", + } + for _, dir := range []string{".", "../lane/control"} { + set := token.NewFileSet() + pkg, err := parser.ParseDir(set, dir, func(info os.FileInfo) bool { + return !strings.HasSuffix(info.Name(), "_test.go") + }, 0) + if err != nil { + t.Fatal(err) + } + for _, files := range pkg { + for _, file := range files.Files { + for _, decl := range file.Decls { + fn, ok := decl.(*ast.FuncDecl) + if !ok { + continue + } + if _, allowed := doors[fn.Name.Name]; allowed { + continue + } + ast.Inspect(fn, func(node ast.Node) bool { + call, ok := node.(*ast.CallExpr) + if !ok { + return true + } + if selector, ok := call.Fun.(*ast.SelectorExpr); ok && selector.Sel.Name == "Allows" { + t.Errorf("%s: %s asks the purse; only affordsLocked (before sending) and reachable (before deciding) may — see THE ONE DOOR in hedge.go", + set.Position(call.Pos()), fn.Name.Name) + } + return true + }) + } + } + } + } +} + +// TestTheTwoDoorsAskThePurseWithOnePrice is the other half of the law above: +// one rail asked with two estimates is two rails. +// +// The number is [Alternative.Extra], priced once from the frontier entry this +// race would send to, where the plan is built. The controller asks with it +// directly; the race asks with it through [hedgeRace.altPrice]. This drives the +// shipped door with a plan whose alternative is priced differently from the +// frontier beside it, and fails if the race quietly answers from the frontier. +func TestTheTwoDoorsAskThePurseWithOnePrice(t *testing.T) { + const dear = 4.0 + asked := []float64{} + plan := control.Plan{ + Lane: "A", + Alts: []control.Alternative{{Lane: "B", Extra: dear}}, + Purse: recordingPurse{seen: &asked}, + } + race := &hedgeRace{model: "openrouter/oneprice", winner: -1, plan: plan, expected: 100} + // The frontier prices the same machine at a tenth of what the plan does, so + // an answer taken from here instead of from the plan is visible. + race.choice.Frontier = []lanes.Scored{{ID: lanes.ID{Model: "openrouter/oneprice", Lane: "B"}, Price: dear / 10}} + race.affords("B") + if len(asked) != 1 { + t.Fatalf("the purse was asked %d times for one arm, want once", len(asked)) + } + if asked[0] != dear { + t.Fatalf("the race asked the purse for $%.2f and the plan prices this arm at $%.2f; two estimates for one bound", + asked[0], dear) + } +} + +// recordingPurse allows everything and remembers what it was asked for. +type recordingPurse struct{ seen *[]float64 } + +func (p recordingPurse) Allows(usd float64, _ time.Time) bool { + *p.seen = append(*p.seen, usd) + return true +} + +// TestEveryAttemptGetsItsOwnController is the law the 363-second attempt broke. +// +// A watch belongs to an ARM and an arm sends many times. Every once-only rule on +// this type — the act the row carries, the controller's own report, a purse +// refusal — is right for ONE SEND and wrong for a call, so a second attempt used +// to run inside the first one's answers and could not report at all. Re-arming +// is what makes each send a send; the deadline and the move log belong to the +// question and are untouched, which is the other half of the law. +func TestEveryAttemptGetsItsOwnController(t *testing.T) { + began := time.Now() + plan := control.Plan{ + Lane: "A", Ceiling: 10 * time.Millisecond, Began: began, + Moves: control.NewMoveLog(), Deadline: began.Add(time.Hour), + } + race := &hedgeRace{model: "openrouter/attempts", winner: -1, plan: plan, build: control.New} + watch := &streamWatch{race: race, arm: 0, plan: plan, control: control.New(plan), began: began} + race.arms = []*hedgeArm{{index: 0, watch: watch}} + + // The first attempt runs out its ceiling and is acted on, which is what + // writes the row. + watch.mu.Lock() + watch.after(watch.control.Quiet(began.Add(time.Second)), began.Add(time.Second)) + watch.mu.Unlock() + if watch.acted.Kind == control.None { + t.Fatal("the first attempt was never acted on, so this test proves nothing") + } + if watch.plan.Deadline != plan.Deadline { + t.Fatal("the watch did not keep the question's own deadline") + } + + // And the second attempt is a send like any other. + second := began.Add(time.Minute) + watch.attempt(second) + if watch.acted.Kind != control.None { + t.Fatalf("the second attempt inherited the first one's act (%v), so its row can never say what it did", watch.acted.Kind) + } + if !watch.began.Equal(second) { + t.Fatalf("the second attempt is dated %s rather than the moment its own bytes left (%s)", watch.began, second) + } + if watch.tokens != 0 || watch.visible != 0 || watch.beats != 0 || watch.served != "" { + t.Fatalf("the second attempt started holding the first one's stream: %d tokens, %d visible, %d beats, served by %q", + watch.tokens, watch.visible, watch.beats, watch.served) + } + if watch.deadline.Before(second) { + t.Fatalf("the second attempt's deadline %s is already past at %s", watch.deadline, second) + } + // WHAT BELONGS TO THE QUESTION IS UNTOUCHED. A rescue does not buy the + // question more time and a re-ask does not forget where it has been. + if watch.plan.Deadline != plan.Deadline { + t.Fatal("re-arming moved the question's deadline") + } + if race.plan.Moves != plan.Moves { + t.Fatal("re-arming replaced the question's move log") + } +} + +// TestTheDeletedAllowanceIsNotStillDescribed is bar §6 with teeth: delete what +// you replace, INCLUDING the sentences that describe it. +// +// WHY A GREP IS THE RIGHT SHAPE HERE. `go build` deletes a dead identifier from +// the code the moment nothing calls it, and then leaves it standing in every +// comment and every design page that named it — where it is worse than dead, +// because a comment is what the next person reads INSTEAD of the code. The +// review of #924 found six: this file's own top-of-file law still said money was +// bounded by `lanes.Budget`, and `docs/ARCHITECTURE.md`'s clock table still +// published a tenth of the hour's spend over twenty requests as the rail in +// front of a rescue. Two of them asserted the OPPOSITE of the law that replaced +// them. +// +// A DEAD NAME MAY STILL BE SAID IN THE PAST TENSE, and that is the whole of the +// exception: "it was a rolling allowance until 2026-09-11" is the sentence that +// stops somebody re-inventing it. So a line carrying one of these names has to +// carry a word that puts it in the past as well. +func TestTheDeletedAllowanceIsNotStillDescribed(t *testing.T) { + dead := []string{ + "lanes.Budget", "lane.Budget", "NewBudget", "DefaultBudget", + "SetHedgeBudget", "currentHedgeBudget", "requestWindow", "spendWindow", + } + // A name in the past tense is a record and not a claim. These are the words + // this codebase uses to say so. + past := []string{ + "deleted", "no longer", "used to", "until 2026-09-11", "was written against", + "stopped", "gone", "replaced", "does not exist", "gets deleted", + } + // The live prose: the two packages that owned the rail and the two documents + // that publish what bounds a wait. `docs/changes` and `bench/lanelab/REPORT.md` + // are records of what was true on a day and are deliberately not read. + for _, where := range []string{"../provider", "../lane", "../../docs/ARCHITECTURE.md", "../../docs/design/waiting/DESIGN.md"} { + for _, file := range prose(t, where) { + // THE LAW IS ALLOWED TO SAY THE NAMES IT BANS. Every dead identifier + // is written out in this file, twice — once in the list and once in + // the account of where they were found — and a law that failed on + // its own statement of itself would be unwritable. + if strings.HasSuffix(file, "purse_law_test.go") { + continue + } + body, err := os.ReadFile(file) + if err != nil { + t.Fatal(err) + } + for number, line := range strings.Split(string(body), "\n") { + lower := strings.ToLower(line) + for _, name := range dead { + if !strings.Contains(line, name) { + continue + } + buried := false + for _, word := range past { + if strings.Contains(lower, word) { + buried = true + break + } + } + if !buried { + t.Errorf("%s:%d still describes %s as though it existed: %s", + file, number+1, name, strings.TrimSpace(line)) + } + } + } + } + } +} + +// prose is every Go source file and Markdown page under one path, the path +// itself when it names a file. Test sources are read too: a law stated in a +// test's own comment is read by exactly the person this is for. +func prose(t *testing.T, where string) []string { + t.Helper() + info, err := os.Stat(where) + if err != nil { + t.Fatal(err) + } + if !info.IsDir() { + return []string{where} + } + entries, err := os.ReadDir(where) + if err != nil { + t.Fatal(err) + } + var out []string + for _, entry := range entries { + if entry.IsDir() { + continue + } + if name := entry.Name(); strings.HasSuffix(name, ".go") || strings.HasSuffix(name, ".md") { + out = append(out, filepath.Join(where, name)) + } + } + return out +} diff --git a/internal/provider/reasoninghedge_test.go b/internal/provider/reasoninghedge_test.go index 44c8aecef1..1fcfbf89cc 100644 --- a/internal/provider/reasoninghedge_test.go +++ b/internal/provider/reasoninghedge_test.go @@ -158,7 +158,7 @@ func TestAThinkingStallStillObeysTheBudget(t *testing.T) { lanestub.Lane{Name: "B", Profile: lanestub.Profile{TTFT: 5 * time.Millisecond, Rate: 2000, Tokens: 24}}, ) rig.believes("A", 2, 250) - SetHedgeBudget(lanes.NewBudget(0, 0)) + noRescues(t) report := &HedgeReport{} ctx := WithHedgeReport(talking(), report) diff --git a/internal/provider/recovery_test.go b/internal/provider/recovery_test.go index 5cc0887cba..198fdb39e1 100644 --- a/internal/provider/recovery_test.go +++ b/internal/provider/recovery_test.go @@ -76,7 +76,6 @@ func vetoEverythingBut(rig *laneRig, vetoed ...string) { // never hands the caller the 404. func TestTheMeasuredRefusalRaceEndsInAnAnswerOnTheFirstAttempt(t *testing.T) { rig := newLaneRig(t, "recovery/measured", measuredLanes(true)...) - SetHedgeBudget(lanes.NewBudget(6, 0)) vetoEverythingBut(rig, "DeepSeek", "Io Net", "Novita") told := ¬ices{} @@ -126,7 +125,6 @@ func TestTheMeasuredRefusalRaceEndsInAnAnswerOnTheFirstAttempt(t *testing.T) { // routing 404 that the walk had already acted on. func TestAnExhaustedRaceSettlesOnTheMostActionableError(t *testing.T) { rig := newLaneRig(t, "recovery/actionable", measuredLanes(false)...) - SetHedgeBudget(lanes.NewBudget(6, 0)) vetoEverythingBut(rig, "DeepSeek", "Io Net") ctx := WithLaneChoice(talking(), measuredChoice(rig.model)) @@ -198,7 +196,6 @@ func TestAnAccountExclusionIsLearnedOnceForEveryModel(t *testing.T) { } } rig := newLaneRig(t, "recovery/account-a", serving()...) - SetHedgeBudget(lanes.NewBudget(6, 0)) modelB := "openrouter/recovery/account-b" rig.server.Model(modelB, serving()...) @@ -271,7 +268,6 @@ func TestARescueOnAFullPoolGoesBackToTheRaceAtOnce(t *testing.T) { lanestub.Lane{Name: "Fireworks", Profile: lanestub.Profile{Paced: true}}, lanestub.Lane{Name: "Novita", Profile: lanestub.Profile{TTFT: 2 * time.Millisecond, Rate: 2000, Tokens: 19}}, ) - SetHedgeBudget(lanes.NewBudget(6, 0)) vetoEverythingBut(rig, "Fireworks", "Novita") choice := lanes.Choice{Order: []string{"DeepSeek"}} for _, lane := range []string{"DeepSeek", "Fireworks"} { diff --git a/internal/provider/routing_audit_test.go b/internal/provider/routing_audit_test.go index 5ac98c5af3..b8c2d5602b 100644 --- a/internal/provider/routing_audit_test.go +++ b/internal/provider/routing_audit_test.go @@ -98,8 +98,6 @@ func TestWatchedFaultUsesFundedAlternativeBeforeRepeatingTheFailedRequest(t *tes fmt.Fprint(w, "data: {\"provider\":\"B\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\"The replacement answered normally.\"}}]}\n\ndata: {\"choices\":[{\"index\":0,\"delta\":{},\"finish_reason\":\"stop\"}]}\n\ndata: [DONE]\n\n") })) client.wait = func(context.Context, time.Duration) error { waits.Add(1); return nil } - SetHedgeBudget(lanes.NewBudget(1, 0)) - t.Cleanup(func() { SetHedgeBudget(nil) }) // This request overrides the client's default model. A refusal for that // default must not take away an alternative serving the requested model. const model = "vendor/override" @@ -119,8 +117,6 @@ func TestStrictPinDoesNotWalkToAnotherEndpointAfterTransportFailure(t *testing.T client, recorded := pinningClient(t, StaticRouting(RoutingLatency), 0, 0, false, answering(reply{status: 503, body: `{"error":{"message":"temporarily unavailable","metadata":{"provider_name":"A"}}}`})) client.wait = func(context.Context, time.Duration) error { return nil } - SetHedgeBudget(lanes.NewBudget(1, 0)) - t.Cleanup(func() { SetHedgeBudget(nil) }) choice := choiceFor(client.config.Model, time.Second) choice.Only, choice.Order = []string{"A"}, nil ctx := WithLaneChoice(WithStreamObserver(talking(), func(StreamEvent) {}), choice) diff --git a/internal/provider/waitplan.go b/internal/provider/waitplan.go index 4c93129ee4..2c437aa412 100644 --- a/internal/provider/waitplan.go +++ b/internal/provider/waitplan.go @@ -3,7 +3,6 @@ package provider import ( "context" "strings" - "sync" lanes "github.com/Agent-Field/aforge-v2/internal/lane" "github.com/Agent-Field/aforge-v2/internal/lane/control" @@ -12,42 +11,21 @@ import ( // ── WHAT ONE CALL IS WATCHED WITH ─────────────────────────────────────────── // // A plan is not a decision. It is what is BELIEVED about the machine expected -// to serve, what the role will wait, where acting could go and what the process -// can afford — assembled here because this is the only place all four are known +// to serve, what the role will wait, where acting could go and what this call may +// spend rescuing itself — assembled here because this is the only place all four are known // at once, and handed to `internal/lane/control`, which is the only thing in // this build that decides when a wait has gone on long enough. -// ── THE PURSE THIS PROCESS ACTS UNDER ─────────────────────────────────────── - -// hedgeBudget is the one budget the whole process acts under. +// ── THE PURSE IS THE CALL'S OWN, AND THE PROCESS HAS NONE ─────────────────── // -// It is a package variable rather than a field on the Client because the limit -// it holds is about a SESSION and not about an adapter: two clients — the chat -// and a tool loop's own — must not each be allowed six rescues a minute and a -// tenth of the bill. -var ( - hedgeBudgetMu sync.RWMutex - hedgeBudget = lanes.DefaultBudget() -) - -// currentHedgeBudget is the budget in force. -func currentHedgeBudget() *lanes.Budget { - hedgeBudgetMu.RLock() - defer hedgeBudgetMu.RUnlock() - return hedgeBudget -} - -// SetHedgeBudget replaces the process's budget. A nil argument restores the -// default rather than leaving a hole; a budget of zero rescues is how the -// second request is switched off outright. -func SetHedgeBudget(budget *lanes.Budget) { - hedgeBudgetMu.Lock() - defer hedgeBudgetMu.Unlock() - if budget == nil { - budget = lanes.DefaultBudget() - } - hedgeBudget = budget -} +// There was a package-wide budget here until 2026-09-11 — two rescues in any +// twenty requests and a tenth of the last hour's bill, shared by every client — +// and it is deleted. A window cannot tell the request that needs a rescue from +// the nineteen that do not, so it refused by arrival order and overruled the one +// thing in this build that decides when a wait has gone on long enough +// ([internal/lane]'s hedge.go carries the measurement). What a call may spend +// rescuing itself is its own patience converted through λ +// ([control.Plan.SpendUSD]), and it is built with the rest of the plan. // ── WHAT ONE CALL IS WATCHED WITH ─────────────────────────────────────────── @@ -55,9 +33,10 @@ func SetHedgeBudget(budget *lanes.Budget) { // // It is assembled here because this is the only place all of it is known at // once: the role, from the context; the belief, from the ledger; the lanes -// worth going to, from the choice; and the purse, from the process. None of it -// is a decision — the plan says what is believed and what is allowed, and -// `internal/lane/control` is the only thing that decides. +// worth going to, from the choice; and what it may spend rescuing itself, from +// the plan the role already carries. None of it is a decision — the plan says +// what is believed and what is allowed, and `internal/lane/control` is the only +// thing that decides. func (c *Client) planFor(ctx context.Context, choice lanes.Choice, model string, expected int) control.Plan { now := waitNow() // THE PLAN IS `internal/lane`'s ARITHMETIC AND IT IS BUILT THERE: the head @@ -68,14 +47,21 @@ func (c *Client) planFor(ctx context.Context, choice lanes.Choice, model string, // ── and then the four things only the transport knows // // A STRICT PIN IS WHAT `Only` MEANS: the person named the machine, so the - // act is a question rather than a rescue (offer.go). The purse is the - // process's. How long the answer will be is the caller's own hint, and it - // is what makes the commitment half of the inequality computable. And how - // long this model deliberates is a fact about the model and the rung it was - // asked at, which is a knob only this layer resolves. + // act is a question rather than a rescue (offer.go). How long the answer will + // be is the caller's own hint, and it is what makes the commitment half of the + // inequality computable. And how long this model deliberates is a fact about + // the model and the rung it was asked at, which is a knob only this layer + // resolves. THE PURSE IS NOT AMONG THEM ANY MORE: it is the plan's own budget + // and `lane.PlanFor` fills it in, which is why nothing here names money. plan.Pinned = len(choice.Only) > 0 - plan.Purse = lanes.Spending(currentHedgeBudget()) plan.Expected = expected + // AND A PERSON MAY TURN RESCUING OFF OUTRIGHT (lanepin.go's [SetLaneGuard]). + // It is the one thing about a plan that is neither a belief nor a role, so it + // is written where every arm of the question reads it: a purse that refuses + // everything, which the race's one affordability door already asks. + if !LaneGuardOn() { + plan.Purse = lanes.NoSpending() + } plan.Think = lanes.Thinks(model, c.recordedEffort(model, knobsFrom(ctx)), now) // AND WHERE A SECOND REQUEST COULD NOT BE DEMANDED THERE ARE NO // ALTERNATIVES — an endpoint that is not a router, or a person who asked diff --git a/internal/tui3/phase.go b/internal/tui3/phase.go index f7efaf524b..bc0422eae9 100644 --- a/internal/tui3/phase.go +++ b/internal/tui3/phase.go @@ -595,6 +595,13 @@ func phaseFields(news PhaseNews, now time.Time) []rowField { // what this surface did before — is a person watching a line that says // nothing while a real wait runs. return []rowField{rowSay("all lanes slow"), rowSay("still waiting"), rowSay(countUpWord(since))} + case provider.PhaseBelowPace: + // THE ANSWER IS ARRIVING AND IT IS TOO SLOW TO READ, and there is no + // faster machine to move it to. It is a different sentence from the one + // above because it is a different fact — words ARE appearing — and a + // person told "still waiting" while watching text arrive would stop + // believing this row. The clock is the wait's own and it counts up. + return []rowField{rowSay("answering slowly"), rowSay("nowhere faster"), rowSay(countUpWord(since))} case provider.PhaseChecking, provider.PhaseTidying, provider.PhaseTakingStock: // THREE WORDS WITH ONE SHAPE: a stage named by nothing but itself, and // the clock a person is reading it against. `taking stock` shares the arm