diff --git a/cmd/codeaf/chat.go b/cmd/codeaf/chat.go index ec201ac6e2..be70ad176b 100644 --- a/cmd/codeaf/chat.go +++ b/cmd/codeaf/chat.go @@ -545,9 +545,14 @@ func buildBrain(w *chatWindow, session string, opts brainOptions) (*chatBrain, e fanIn = store.DependencyFanIn{} } // Ordinary leaves retain the byte-identical headless envelope, raised by - // exactly what their own fan-in measures — see gatheringGrant. Reflexes use - // the deliberately tiny rung budget and a seconds-scale watchdog. + // exactly what their own fan-in measures — see gatheringGrant. A leaf + // re-dispatched after running out is granted more than the attempt that + // ran out, or it would buy the same truncated ending again — see + // regrantAfterRunningOut; the wall below follows the larger grant because + // it is arithmetic over the same number. Reflexes use the deliberately + // tiny rung budget and a seconds-scale watchdog. turns, tokens := gatheringGrant(chatLeafTurns, chatLeafTokens, fanIn) + tokens = regrantAfterRunningOut(tokens, int(node.Attempt)) leafRoom := exec.SubharnessFor(subharness) wallLeft := remainingWall(ctx) deadline := leafRoom.DeadlineWithin(tokens, wallLeft) diff --git a/cmd/codeaf/context_budget_test.go b/cmd/codeaf/context_budget_test.go index 883632683f..7c7cc00348 100644 --- a/cmd/codeaf/context_budget_test.go +++ b/cmd/codeaf/context_budget_test.go @@ -208,3 +208,60 @@ func TestAFoldsGrantIsSizedForThePassItActuallyMakes(t *testing.T) { t.Fatalf("an unsized window granted %d tokens, want the process-wide %d", unknown, want) } } + +// A RE-DISPATCH IS GRANTED MORE ROOM THAN THE ATTEMPT THAT RAN OUT, or the +// identical budget buys the identical truncated ending. The runner's overrun +// settle releases a leaf that ran out back to pending with its banked turns, +// and while the grant was attempt-blind the next claim re-entered the same +// brief under the same meter and stopped at the same place — one truncation, +// paid for as many times as the round cap allowed. +func TestARedispatchIsGrantedMoreRoomThanTheAttemptThatRanOut(t *testing.T) { + // Attempt zero keeps the grant byte for byte: a leaf on its first claim + // has run out of nothing yet. + if room := regrantAfterRunningOut(chatLeafTokens, 0); room != chatLeafTokens { + t.Fatalf("a first claim was granted %d tokens, want the flat %d it has always had", + room, chatLeafTokens) + } + + // Half again per re-dispatch, compounding, and strictly more room each + // time — the growth is the whole point of sending a node round again. + previous := chatLeafTokens + for attempt := 1; attempt <= 3; attempt++ { + room := regrantAfterRunningOut(chatLeafTokens, attempt) + if room <= previous { + t.Fatalf("re-dispatch %d was granted %d tokens, no more than the %d before it", + attempt, room, previous) + } + previous = room + } + // The exact arithmetic, in the integers the grant is stated in: three + // halves, applied per attempt, never rounded up by a shortcut. + if got, want := regrantAfterRunningOut(chatLeafTokens, 2), chatLeafTokens*3/2*3/2; got != want { + t.Fatalf("attempt 2 was granted %d tokens, want the twice-grown %d", got, want) + } + + // The ceiling holds at attempts no run can reach, and the ladder stops + // there rather than stepping over it. + for _, attempt := range []int{10, 100} { + if room := regrantAfterRunningOut(chatLeafTokens, attempt); room != overrunGrantCeiling { + t.Fatalf("attempt %d was granted %d tokens, want the ceiling %d", + attempt, room, overrunGrantCeiling) + } + } + + // A leaf whose fan-in already measured more than the ceiling keeps every + // token it measured. The bound is on what the ladder adds, and a + // re-dispatch handed less room than its first attempt would be the + // identical-budget defect back by another door. + _, measured := gatheringGrant(chatLeafTurns, chatLeafTokens, + store.DependencyFanIn{Count: 8, Bytes: 8 << 20}) + if measured <= overrunGrantCeiling { + t.Fatalf("the wide join measured %d tokens, too small to ask the question", measured) + } + for _, attempt := range []int{1, 3, 100} { + if room := regrantAfterRunningOut(measured, attempt); room != measured { + t.Fatalf("a join measured at %d tokens was re-dispatched at %d on attempt %d", + measured, room, attempt) + } + } +} diff --git a/cmd/codeaf/do.go b/cmd/codeaf/do.go index 9ce388f972..d54f67e2df 100644 --- a/cmd/codeaf/do.go +++ b/cmd/codeaf/do.go @@ -116,14 +116,16 @@ type headlessOutcome struct { // unexported because the sentence leaves through Deliverable, while Settled // is already the machine signal and the JSON contract needs no second key. unfinishedTree string - // Run, Calls and Rounds are what a person went to `calls.jsonl` to - // reconstruct: which run this was, how many model calls it made, and how - // many times it bought more work after looking at what it had. They are - // unexported spellings of the envelope's own keys — the receipt reaches a - // caller through [errandEnvelope] and nowhere else. - run string - calls int - rounds int + // Run, Calls, Rounds and Redispatches are what a person went to + // `calls.jsonl` to reconstruct: which run this was, how many model calls it + // made, how many times it bought more work after looking at what it had, + // and how many times it sent a node round again in place after the node ran + // out of its room. They are unexported spellings of the envelope's own keys + // — the receipt reaches a caller through [errandEnvelope] and nowhere else. + run string + calls int + rounds int + redispatches int // tokensIn and tokensOut are the token half of the bill, summed out of the // same journal read that priced the run. They are unexported because they // reach a caller only through the envelope's `tokens` field, which is the @@ -656,6 +658,11 @@ func priceErrand(graph *store.Store, session string, openedAt int64, outcome *he // work is read from what was written down, for the reason the bill is: a // counter in this process could not see a round a resident spliced. outcome.rounds = errandRounds(graph, session) + // AND THE RE-DISPATCHES OFF IT TOO. How many times this run sent a node + // round again in place is read from the releases that handed work on, for + // the same reason: the release is the record of a re-dispatch, and a + // counter in this process could not see one a resident made. + outcome.redispatches = errandRedispatches(graph, session) } // errandRounds is how many times this errand bought MORE WORK: every growth @@ -685,6 +692,31 @@ func errandRounds(graph *store.Store, session string) int { return rounds } +// errandRedispatches is how many times one of this errand's nodes was sent +// round again in place after running out of the room it was granted: every +// hand-on release its nodes journaled (store.NodeRedispatches). +// +// It asks per node rather than across the store because a release is journaled +// against the node that was re-dispatched, and the rule is the bill's and the +// rounds': a run sharing a durable store with another session must not count +// that session's re-dispatches as its own. A read that fails leaves the count +// at zero rather than at a guess. +func errandRedispatches(graph *store.Store, session string) int { + nodes, err := graph.SessionMemberNodes(session) + if err != nil { + return 0 + } + redispatches := 0 + for _, node := range nodes { + counted, err := graph.NodeRedispatches(node.ID) + if err != nil { + continue + } + redispatches += counted + } + return redispatches +} + // headlessBrain builds and returns the brain this process will run, or nothing // at all when another process is already the brain for this store. // @@ -2970,20 +3002,21 @@ func reportErrand(request doRequest, outcome headlessOutcome) error { // `exec` and `run` from publishing three different objects again. func errandEnvelope(outcome headlessOutcome) resultEnvelope { return buildResultEnvelope(runResult{ - Stop: outcome.resolvedStop(), - Answer: outcome.Deliverable, - Files: outcome.Artifacts, - Error: outcome.Error, - SpendUSD: outcome.Spend, - TokensIn: outcome.tokensIn, - TokensOut: outcome.tokensOut, - Seconds: outcome.Seconds, - Model: outcome.Model, - Steps: outcome.Nodes, - Run: outcome.run, - Calls: outcome.calls, - Rounds: outcome.rounds, - Extra: legacyErrandFields(outcome), + Stop: outcome.resolvedStop(), + Answer: outcome.Deliverable, + Files: outcome.Artifacts, + Error: outcome.Error, + SpendUSD: outcome.Spend, + TokensIn: outcome.tokensIn, + TokensOut: outcome.tokensOut, + Seconds: outcome.Seconds, + Model: outcome.Model, + Steps: outcome.Nodes, + Run: outcome.run, + Calls: outcome.calls, + Rounds: outcome.rounds, + Redispatches: outcome.redispatches, + Extra: legacyErrandFields(outcome), }) } diff --git a/cmd/codeaf/do_test.go b/cmd/codeaf/do_test.go index 852ae14f68..73595abce3 100644 --- a/cmd/codeaf/do_test.go +++ b/cmd/codeaf/do_test.go @@ -2330,3 +2330,70 @@ func TestAnErrandCountsTheRoundsItsOwnJobsBought(t *testing.T) { "a neighbour's rounds as its own", got) } } + +// TestAnErrandCountsTheRedispatchesItsOwnNodesBought is the figure beside the +// rounds, and the reason it comes off the journal rather than a counter. +// +// `do --json` said how many times a run went back for MORE but never how many +// times it sent a node round again IN PLACE — a leaf that ran out of its room +// going back on the queue to be carried on from, which is the other half of +// what a bill of re-runs is made of. +// +// IT IS COUNTED PER NODE AND NOT ACROSS THE STORE, for the reason the rounds +// are: a run sharing a durable store with another session must not report that +// session's re-dispatches as its own. +func TestAnErrandCountsTheRedispatchesItsOwnNodesBought(t *testing.T) { + path := filepath.Join(t.TempDir(), "redispatches.db") + graph, err := store.Open(path) + if err != nil { + t.Fatal(err) + } + defer graph.Close() + + const mine, theirs = "session-mine", "session-theirs" + for _, job := range []struct{ id, session string }{ + {"job-a", mine}, {"job-b", mine}, {"job-c", theirs}, + } { + if err := graph.Splice(store.RootID, store.Subtree{Nodes: []store.NodeSpec{{ + ID: job.id, Brief: "a job", Stage: 1, + }}}, store.Provenance{Origin: store.OriginUser, Intent: "a job", SessionID: job.session}); err != nil { + t.Fatal(err) + } + } + // Two hand-ons on one of this errand's nodes, one on another, a hand-back + // that carried nothing on — which is a claim taken off a worker, not a + // re-dispatch — and one on a node belonging to somebody else entirely. + for _, handed := range []struct { + node string + reason string + recorded int + }{ + {"job-a", "it was still working when it ran out of its token budget", 3}, + {"job-a", "it was still working when it ran out of its token budget", 3}, + {"job-b", "it was still working when it ran out of its token budget", 2}, + {"job-b", "no sign of life for the claim", 0}, + {"job-c", "it was still working when it ran out of its token budget", 1}, + } { + claim, ok, err := graph.Claim(handed.node, "chat-runner") + if err != nil || !ok { + t.Fatalf("claim %s: %v", handed.node, err) + } + if handed.recorded > 0 { + if err := graph.ReleaseWithRecord(claim, handed.reason, handed.recorded); err != nil { + t.Fatal(err) + } + continue + } + if err := graph.ReleaseWithReason(claim, handed.reason); err != nil { + t.Fatal(err) + } + } + + if got := errandRedispatches(graph, mine); got != 3 { + t.Errorf("this errand's nodes were sent round again three times between them and it reports %d", got) + } + if got := errandRedispatches(graph, theirs); got != 1 { + t.Errorf("the other session re-dispatched once and it reports %d — a run may not "+ + "count a neighbour's re-dispatches as its own", got) + } +} diff --git a/cmd/codeaf/envelope.go b/cmd/codeaf/envelope.go index f191944547..d78f434732 100644 --- a/cmd/codeaf/envelope.go +++ b/cmd/codeaf/envelope.go @@ -333,6 +333,19 @@ type resultEnvelope struct { // both because a caller reaching for a key that vanished is a caller // crashing. Rounds int `json:"rounds"` + // Redispatches is how many times one of this run's nodes was re-dispatched + // IN PLACE after running out of the room it was granted — sent round again + // to carry on from what it had banked, rather than grown around. It is + // counted off the journal the way Rounds is (store.NodeRedispatches), for + // the same reason: a run sharing a durable store with another session must + // not count that session's re-dispatches as its own. + // + // It is the one contract field that is ABSENT rather than zero when it never + // happened, which is what `omitempty` buys and why it wears it where Rounds + // does not: a caller of this one is asking whether the run had to send + // anything round again, and "never" is the key not being there. `exec` and + // a saved program never set it, so neither ever carries it. + Redispatches int `json:"redispatches,omitempty"` // extra is what one verb carries beyond the contract, and it is two things: // the OLD field names, kept readable for one release so that a tool written @@ -384,19 +397,20 @@ func (e resultEnvelope) MarshalJSON() ([]byte, error) { // result type, and that mapping is the only place a verb's private shape and // the public contract meet. type runResult struct { - Stop stopReason - Answer string - Files []string - Error string - SpendUSD float64 - TokensIn int - TokensOut int - Seconds float64 - Model string - Steps int - Run string - Calls int - Rounds int + Stop stopReason + Answer string + Files []string + Error string + SpendUSD float64 + TokensIn int + TokensOut int + Seconds float64 + Model string + Steps int + Run string + Calls int + Rounds int + Redispatches int // Extra is this verb's own fields: its old spellings, and whatever it knows // that the contract has no room for. Nil for a verb with neither. Extra map[string]any @@ -421,20 +435,21 @@ func buildResultEnvelope(result runResult) resultEnvelope { files = []string{} } return resultEnvelope{ - OK: exitFor(stop) == exitDone, - Stop: stop, - Answer: result.Answer, - Files: files, - Error: result.Error, - SpendUSD: result.SpendUSD, - Tokens: envelopeTokens{In: result.TokensIn, Out: result.TokensOut}, - Seconds: result.Seconds, - Model: result.Model, - Steps: result.Steps, - Run: result.Run, - Calls: result.Calls, - Rounds: result.Rounds, - extra: result.Extra, + OK: exitFor(stop) == exitDone, + Stop: stop, + Answer: result.Answer, + Files: files, + Error: result.Error, + SpendUSD: result.SpendUSD, + Tokens: envelopeTokens{In: result.TokensIn, Out: result.TokensOut}, + Seconds: result.Seconds, + Model: result.Model, + Steps: result.Steps, + Run: result.Run, + Calls: result.Calls, + Rounds: result.Rounds, + Redispatches: result.Redispatches, + extra: result.Extra, } } diff --git a/cmd/codeaf/envelope_test.go b/cmd/codeaf/envelope_test.go index 531137b2cb..de4bd0d3f8 100644 --- a/cmd/codeaf/envelope_test.go +++ b/cmd/codeaf/envelope_test.go @@ -399,4 +399,19 @@ func TestTheEnvelopeNamesItsRunAndCountsItsCallsAndRounds(t *testing.T) { if fields["rounds"] != float64(8) { t.Errorf("rounds = %v, want the 8 rounds of work it bought", fields["rounds"]) } + // `redispatches` carries its count the same way, and is the one key that is + // absent rather than zero when it never happened — a caller of it is asking + // whether the run had to send anything round again, and "never" is the key + // not being there. + if _, present := fields["redispatches"]; present { + t.Errorf("redispatches = %v on a run that re-dispatched nothing; the key belongs "+ + "to the runs it happened to", fields["redispatches"]) + } + fields = envelopeFields(t, buildResultEnvelope(runResult{ + Run: "0123456789abcdef", Calls: 422, Rounds: 8, Redispatches: 1, + })) + if fields["redispatches"] != float64(1) { + t.Errorf("redispatches = %v, want the one node the run sent round again in place", + fields["redispatches"]) + } } diff --git a/cmd/codeaf/subharness.go b/cmd/codeaf/subharness.go index 170529bed5..5701ab2f1d 100644 --- a/cmd/codeaf/subharness.go +++ b/cmd/codeaf/subharness.go @@ -161,6 +161,50 @@ func gatheringGrant(turns, tokens int, fanIn store.DependencyFanIn) (int, int) { return turns + fanIn.Count, tokens + 2*landed } +// The re-dispatch grant: what a leaf that has been sent round again in place +// is given, in the one number the dispatch path states for ctxbudget. +const ( + // overrunRegrantNum over overrunRegrantDen is how much bigger one + // re-dispatch's token grant is than the grant of the attempt that ran + // out: three halves. A re-dispatch exists to finish a truncated tail, and + // half again is enough to finish one, where doubling would buy the whole + // run a second time. + overrunRegrantNum = 3 + overrunRegrantDen = 2 + + // overrunGrantCeiling is where the regrant stops adding: four flat leaf + // grants, which is room to run an ordinary leaf's whole text through twice + // over and therefore more than finishing a tail can ever cost. The bound + // is on what the ladder adds and not on what the leaf was granted — a + // fan-in that measured more than this keeps every token it measured, + // because the one thing a re-dispatch must never do is hand back room. + overrunGrantCeiling = 4 * chatLeafTokens +) + +// regrantAfterRunningOut is the token grant one claim of a leaf that has been +// re-dispatched in place is given, from the grant this claim measured and the +// attempt it is on. Attempt zero is the grant unchanged; every attempt after +// it grows by [overrunRegrantNum] over [overrunRegrantDen] for each re-dispatch +// before it, and the ladder adds nothing beyond [overrunGrantCeiling]. +// Turns are not regrown: a re-dispatch carries its banked turns as inputs +// rather than re-running them, so the turn ceiling was never what ran out. +// +// The growth is the point. A re-dispatch handed the room its predecessor ran +// out of re-runs the same brief to the same truncated ending — the meter stops +// it at the same place — so the runner's release (see resident's overrun +// settle) is worth a claim only if the room moves with it. The wall the leaf +// is given follows, because it is arithmetic over the same number. +func regrantAfterRunningOut(tokens, attempt int) int { + room := tokens + for ; attempt > 0 && room < overrunGrantCeiling; attempt-- { + room = room * overrunRegrantNum / overrunRegrantDen + if room > overrunGrantCeiling { + return overrunGrantCeiling + } + } + return room +} + // foldGrant sizes the whole run of a leaf that is going to make one model call. // // The gathering grant above is the right arithmetic for a node that has to go diff --git a/docs/changes/unreleased/1098-redispatch-room-and-refusal.md b/docs/changes/unreleased/1098-redispatch-room-and-refusal.md new file mode 100644 index 0000000000..89f6a7fe32 --- /dev/null +++ b/docs/changes/unreleased/1098-redispatch-room-and-refusal.md @@ -0,0 +1,15 @@ +--- +kind: changed +title: A re-dispatch after running out is granted more room, and one that banks nothing new is refused +pr: 1098 +surface: [resident, docs] +invalidates: + - "A leaf that ran out of its token budget was re-dispatched with the identical budget and the identical brief, up to the round cap, and bought the same truncated ending each time. The dispatch's grant is attempt-aware now: `regrantAfterRunningOut` in `cmd/codeaf/subharness.go` grows the token grant by three halves per re-dispatch, never above `overrunGrantCeiling` (four flat leaf grants; a fan-in that measured more keeps every token it measured), and the wall the leaf is given follows the larger grant." + - "A leaf that ran out was requeued whenever anything at all was banked, for as many rounds as the cap allowed. A re-dispatch that banks nothing the attempt before it had not banked is failed now, with the refusal named in the node's error; the count it is measured against is the one the hand-on release journaled, read back through `store.ReleasedTurnsFor`." + - "The `--json` envelope had no field for in-place re-dispatches. `redispatches` counts them across the run's nodes off the journal (`store.NodeRedispatches`), and is absent rather than zero when there were none." +--- + +The two halves are one mechanism: the grant moves with the attempt, and the +record says whether moving it moved anything. A re-dispatch is worth buying only +when the room grows and the work follows; either half alone is the old loop with +a bigger bill. diff --git a/internal/manual/chat/running-from-the-terminal.md b/internal/manual/chat/running-from-the-terminal.md index cb6bc70c71..3624f7eb59 100644 --- a/internal/manual/chat/running-from-the-terminal.md +++ b/internal/manual/chat/running-from-the-terminal.md @@ -172,7 +172,8 @@ stdout, always parseable, printed even when the run failed**: "steps": 3, "run": "0123456789abcdef", "calls": 47, - "rounds": 2 + "rounds": 2, + "redispatches": 1 } ``` @@ -191,6 +192,7 @@ stdout, always parseable, printed even when the run failed**: | `run` | this invocation's id. It names the folder `--debug` writes into, and every row this run wrote into `~/.codeaf/logs/calls.jsonl` carries it too — so `codeaf logs --run ` is how you get from this object to the calls behind it. Empty on a verb that opened no run of its own | | `calls` | how many model calls the run made, counted whether or not the call log is switched on. It is the figure you would otherwise count by hand in `calls.jsonl` | | `rounds` | how many times the run went back for **more work** after looking at what it had. One is the ordinary shape; eight is a run that kept finding more to do, and it is the number that explains a bill nothing else here accounts for. `exec` does not plan and a saved program does not grow, so both hold `0` — a measurement nobody took, the way `steps` does | +| `redispatches` | how many times one of the run's nodes was sent round again **in place** after running out of the room it was granted — back on the queue to carry on from what it had already banked, rather than grown around. Present only when it happened: a run that never re-dispatched a node carries no key at all, and neither does a verb with no plan nodes | **Within a release a field is never removed and never changes meaning; new fields may appear.** `stop` is the field to read for *why*; the exit code only says how much is wrong. diff --git a/internal/resident/exhaustedresult_test.go b/internal/resident/exhaustedresult_test.go index 83b7e4fcbe..6f58e1f8c5 100644 --- a/internal/resident/exhaustedresult_test.go +++ b/internal/resident/exhaustedresult_test.go @@ -139,3 +139,65 @@ func TestAnExhaustedResultWithNothingRecordedFails(t *testing.T) { t.Fatalf("the failure %q does not name the exhaustion", final.Error) } } + +// AND A RE-DISPATCH THAT BANKED NOTHING ITS PREDECESSOR HAD NOT IS REFUSED, so +// the node fails rather than buying the same truncated ending a third time. The +// grant moves with the attempt now (see cmd/codeaf's regrantAfterRunningOut), +// which makes the comparison honest: a re-dispatch given more room that +// reaches the same recorded count has been paid to move and did not. +func TestAnExhaustedRedispatchThatBankedNothingNewFails(t *testing.T) { + graph := openRunnerStore(t) + spliceOneLeaf(t, graph, "task-2") + recordTurns(t, graph, "task-2", 3) + + starts := make(chan struct{}, 8) + runner := NewRunner(graph, func(ctx context.Context, node store.Node) (ExecResult, error) { + starts <- struct{}{} + // The same truncated ending both times: nothing new is recorded between + // the two attempts, so the re-dispatch reaches exactly what the attempt + // before it had already banked. + return ExecResult{ + Summary: "All 722 tests pass. Let me verify the dry-run tests specifically:", + Stop: executor.StopBudget, + Meter: executor.Meter{Name: "cost", Reached: 178086, Allowed: 176834, Unit: "tokens of billed work"}, + }, nil + }, "chat-runner", 1) + + ctx, stop := context.WithTimeout(context.Background(), 10*time.Second) + defer stop() + served := make(chan error, 1) + go func() { served <- runner.Serve(ctx) }() + + waitFor(t, starts, "the leaf never started") + waitFor(t, starts, "the re-dispatch never started — the exhausted node was not sent round again") + deadline := time.Now().Add(8 * time.Second) + var final store.Node + for time.Now().Before(deadline) { + node, found, err := graph.Node("task-2") + if err != nil { + t.Fatalf("read the node: %v", err) + } + if found && node.Status == store.Failed { + final = node + break + } + time.Sleep(10 * time.Millisecond) + } + stop() + <-served + runner.Wait() + + if final.Status != store.Failed { + t.Fatalf("a re-dispatch that banked nothing new settled %q, want failed — it must not "+ + "be sent round to buy the same truncated ending again", final.Status) + } + if final.Attempt != 2 { + t.Fatalf("the node was claimed %d times, want the two of a first attempt and one refused re-dispatch", + final.Attempt) + } + for _, want := range []string{"refused", "added nothing"} { + if !contains(final.Error, want) { + t.Fatalf("the failure %q does not say the re-dispatch was refused for adding nothing", final.Error) + } + } +} diff --git a/internal/resident/ranoutwords.go b/internal/resident/ranoutwords.go index e451dfe18f..2876861b8a 100644 --- a/internal/resident/ranoutwords.go +++ b/internal/resident/ranoutwords.go @@ -6,8 +6,8 @@ import ( executor "github.com/Agent-Field/codeaf/internal/exec" ) -// The two sentences a node gets when the leaf holding it ran out. They are in -// one file because they are one account of one event told to two different +// The three sentences a node gets when the leaf holding it ran out. They are in +// one file because they are one account of one event told to three different // endings, and a reader who meets them a day apart must not have to work out // whether they are describing the same thing. // @@ -57,6 +57,18 @@ func outOfRoomFailure(result ExecResult, recorded int) string { return ranOutWords(result.Stop) + tail + meterAside(result.Meter) } +// outOfRoomRefusedFailure is the same fact when the re-dispatch that just ran +// out banked nothing the attempt before it had not banked: more room was the +// only thing a re-dispatch buys, the work did not move with it, and the node is +// failed here rather than paid to reach the same point again. It says the +// refusal and why, because a bare ending would read as a verdict on the work +// rather than on buying a second copy of the same truncation. +func outOfRoomRefusedFailure(result ExecResult, recorded int) string { + return ranOutWords(result.Stop) + " again, " + pluralTurns(recorded) + + " in and no further than the attempt before it, so the re-dispatch was refused: it added nothing" + + meterAside(result.Meter) +} + // A RELEASE REASON THAT HANDS WORK ON ENDS WITH THE SAME CLAUSE, AND IT IS // SPELLED HERE ONCE. // diff --git a/internal/resident/runner.go b/internal/resident/runner.go index 69db0201fd..349678771f 100644 --- a/internal/resident/runner.go +++ b/internal/resident/runner.go @@ -1420,6 +1420,16 @@ func (r *Runner) runOne(ctx context.Context, node store.Node, hold *leafHold) { // with nothing able to continue it has said all it is going to say, and it // is failed with the ending named rather than handed round again for ever. // + // AND A RE-DISPATCH THAT BANKED NOTHING NEW IS REFUSED. More room is the + // only thing a re-dispatch buys — the grant moves with the attempt now (see + // cmd/codeaf's regrantAfterRunningOut) — so a re-dispatch that reaches no + // further than the attempt before it has been paid to move and did not, and + // it is failed with the refusal named rather than sent round to buy the + // same truncated ending again. The count it is measured against is the one + // the hand-on release journaled: that row is the record's own answer to + // what the attempt before banked, and reading it beats keeping a second + // copy on the claim. + // // A PASSED DELIVERY GATE SETTLES THE NODE. Running out is a statement about // RESOURCES — the meter reached its bound while the worker was still going — // and a gate pass is a statement about the WORK, taken by reading the result @@ -1444,6 +1454,15 @@ func (r *Runner) runOne(ctx context.Context, node store.Node, hold *leafHold) { return } _, recorded := BankedRun(r.graph, node.ID) + // A read of the hand-on release that fails answers "nobody could say", + // which is the answer that leaves the claim to the rules below. + if node.Attempt > 0 { + if carried, handed, readErr := r.graph.ReleasedTurnsFor(node.ID); readErr == nil && + handed && recorded > 0 && recorded <= carried { + _ = r.graph.Fail(claim, outOfRoomRefusedFailure(result, recorded)) + return + } + } if node.Attempt < MaxOverrunRounds && recorded > 0 { _ = r.graph.ReleaseWithRecord(claim, outOfRoomClaimReason(result, recorded), recorded) return diff --git a/internal/store/lifecycle.go b/internal/store/lifecycle.go index 8f254d56cf..e15e0c7d56 100644 --- a/internal/store/lifecycle.go +++ b/internal/store/lifecycle.go @@ -2,6 +2,7 @@ package store import ( "database/sql" + "encoding/json" "errors" "fmt" "math" @@ -403,6 +404,80 @@ func (s *Store) release(claim Claim, reason string, recorded int) error { return nil } +// ReleasedTurnsFor is how many recorded turns the newest hand-on release left +// waiting for the next claim, or that no release of this node ever handed work +// on. It exists because the release is where the count the attempt before +// banked is written down — the settle that re-dispatched the node journaled it +// in the same breath as the reason — and the settle that has to COMPARE against +// it (a re-dispatch deciding whether it moved) cannot compose that comparison +// from its own memory across processes. +// +// Releases that took a claim back without handing work on are not it: they say +// nothing about how much the attempt before banked, and the newest count that +// does say is the one a re-dispatch is compared against. +func (s *Store) ReleasedTurnsFor(nodeID string) (int, bool, error) { + rows, err := s.db.Query(` + SELECT payload FROM events + WHERE node_id = ? AND kind = ? + ORDER BY seq DESC`, strings.TrimSpace(nodeID), EventNodeReleased) + if err != nil { + return 0, false, fmt.Errorf("read the handed-on turns: %w", err) + } + defer rows.Close() + for rows.Next() { + var payload string + if err := rows.Scan(&payload); err != nil { + return 0, false, fmt.Errorf("read the handed-on turns: %w", err) + } + var release releasePayload + if err := json.Unmarshal([]byte(payload), &release); err != nil { + return 0, false, fmt.Errorf("read the handed-on turns: %w", err) + } + if release.Recorded > 0 { + return release.Recorded, true, nil + } + } + if err := rows.Err(); err != nil { + return 0, false, fmt.Errorf("read the handed-on turns: %w", err) + } + return 0, false, nil +} + +// NodeRedispatches is how many times this node has been re-dispatched in place +// after running out of the room it was granted: the count of its releases that +// handed recorded turns on. It is read from the journal rather than counted +// anywhere in a process for the same reason every run figure is — a durable +// store outlives the binary that wrote it, and a re-dispatch a resident made is +// as much a re-dispatch as one this process made. +func (s *Store) NodeRedispatches(nodeID string) (int, error) { + rows, err := s.db.Query(` + SELECT payload FROM events + WHERE node_id = ? AND kind = ? + ORDER BY seq`, strings.TrimSpace(nodeID), EventNodeReleased) + if err != nil { + return 0, fmt.Errorf("read the re-dispatches: %w", err) + } + defer rows.Close() + redispatches := 0 + for rows.Next() { + var payload string + if err := rows.Scan(&payload); err != nil { + return 0, fmt.Errorf("read the re-dispatches: %w", err) + } + var release releasePayload + if err := json.Unmarshal([]byte(payload), &release); err != nil { + return 0, fmt.Errorf("read the re-dispatches: %w", err) + } + if release.Recorded > 0 { + redispatches++ + } + } + if err := rows.Err(); err != nil { + return 0, fmt.Errorf("read the re-dispatches: %w", err) + } + return redispatches, nil +} + // ── the claim reaper reads EVIDENCE OF LIFE, never a clock ────────────────── // // THE DEFECT THIS ANSWERS, and it is the same defect twice. This sweep used to