From c5944143ffd1fea35fe54a1af7bddd64aac37575 Mon Sep 17 00:00:00 2001 From: Thomas Kosiewski Date: Mon, 16 Nov 2026 09:10:28 +0000 Subject: [PATCH 1/3] test(faucet): a late failure below the position ends in no gap GitHub writes an attempt down when it ends, so a failure can be listed a few seconds below the position of the run before. A read reaches RedeliveryReadOverlap below that position, but the gap that stands for an unread read starts at the position itself. A late failure in that overlap that no run reads before GitHub deletes it ends in neither a read nor a gap. The new tests list a failure L 3 seconds below the position A, after the run that read A, and check that L ends tracked, stored or inside a logged gap. Each case reaches one of the three places where a gap replaces a read: - the head reads to the end of the list after a cron outage longer than GitHub's retention; - a range left open by a burst expires unread; - a range is read to the end of the list after GitHub deleted L. All three fail on main. requireNoSilentLoss now prints the logged gaps, so a failure shows how far above L each gap starts. --- faucet/redeliver_gaps_test.go | 50 ++++++++++++++++++++++++++++++++++- 1 file changed, 49 insertions(+), 1 deletion(-) diff --git a/faucet/redeliver_gaps_test.go b/faucet/redeliver_gaps_test.go index 2ba3b57..b6deb3f 100644 --- a/faucet/redeliver_gaps_test.go +++ b/faucet/redeliver_gaps_test.go @@ -20,6 +20,10 @@ type gapWindow struct{ from, to time.Time } func (w gapWindow) holds(at time.Time) bool { return !at.Before(w.from) && !at.After(w.to) } +func (w gapWindow) String() string { + return w.from.Format(time.DateTime) + " to " + w.to.Format(time.DateTime) +} + // loggedGaps are the time ranges of the gaps that the runs logged. func (e *redeliveryEnv) loggedGaps() []gapWindow { var gaps []gapWindow @@ -50,7 +54,7 @@ func requireNoSilentLoss(t *testing.T, env *redeliveryEnv, failures []ghAttempt) for _, w := range gaps { covered = covered || w.holds(f.at) } - require.True(t, covered, "%s failed at %s and has no final state: no run tracked it and no gap covers it", f.guid, f.at.Format(time.TimeOnly)) + require.True(t, covered, "%s failed at %s and has no final state: no run tracked it and no gap covers it, gaps %v", f.guid, f.at.Format(time.DateTime), gaps) } } @@ -281,3 +285,47 @@ func TestALateAttemptIsReadWhileARangeStandsAtThePosition(t *testing.T) { } } } + +func TestALateFailureEndsInAGapWhenTheCronIsDownLongerThanGitHubKeepsIt(t *testing.T) { + t.Parallel() + env := newRedeliveryEnv(t) + env.clock.WithLogger(quartz.NoOpLogger) + a := env.gh.record("A", env.gh.now().Add(-time.Second), http200) + env.run() + l := env.gh.record("L", a.at.Add(-3*time.Second), 503) + + env.advance(80 * time.Hour) + env.run() + + requireNoSilentLoss(t, env, []ghAttempt{l}) +} + +func TestALateFailureEndsInAGapWhenGitHubDeletesItBeforeARunReadsTheRangeAboveIt(t *testing.T) { + t.Parallel() + + for name, listStart := range map[string]time.Duration{ + "the range expires unread": 8 * time.Hour, + "the range is read to the end of the list": time.Minute, + } { + t.Run(name, func(t *testing.T) { + t.Parallel() + env := newRedeliveryEnv(t, func(o *envOptions) { o.maxPages = 1 }) + env.clock.WithLogger(quartz.NoOpLogger) + t0 := env.gh.now() + a := env.gh.record("A", t0.Add(-time.Second), http200) + env.run() + l := env.gh.record("L", a.at.Add(-3*time.Second), 503) + for i := range 150 { + env.gh.record(fmt.Sprintf("b-%03d", i), t0.Add(cronEvery-time.Duration(i+1)*2*time.Second), http200) + } + env.advance(cronEvery) + require.Equal(t, 1, env.run().Ranges, "the burst leaves a range down to A") + + env.advance(t0.Add(listStart + faucet.GitHubDeliveryRetention).Sub(env.gh.now())) + env.useMaxPages(faucet.DefaultMaxPagesPerRun) + env.run() + + requireNoSilentLoss(t, env, []ghAttempt{l}) + }) + } +} From 3142e9034b8622cb7475e4e17f857ddda9c551ec Mon Sep 17 00:00:00 2001 From: Thomas Kosiewski Date: Mon, 16 Nov 2026 09:10:56 +0000 Subject: [PATCH 2/3] fix(faucet): start a gap where the read it stands for stops A read of the head or of a range stops RedeliveryReadOverlap below its old end, because GitHub lists an attempt that ended late below one that ended earlier. The gap that a run records in place of that read started at the old end, so a late failure in the overlap that GitHub deleted before any run read it ended in neither a read nor a gap. The three gaps that stand for a read now start at the read's bound: - the head's gap when it reads to the end of the list; - the gap of a range that expires unread; - the gap of a range that is read to the end of the list. The first run's before-tracking gap keeps its start, because that read has no overlap. The gaps that a hook change closes in the stores keep theirs too. TestRedeliverRecordsOnlyTheExpiredPartOfARangeAsAGap now expects the gap to start the overlap below the earlier position. The Redeliver doc comment and the README's `gap` state say that a gap holds the overlap. --- README.md | 2 +- faucet/redeliver.go | 22 ++++++++++------------ faucet/redeliver_test.go | 2 +- 3 files changed, 12 insertions(+), 14 deletions(-) diff --git a/README.md b/README.md index 5b6baee..931b451 100644 --- a/README.md +++ b/README.md @@ -146,7 +146,7 @@ The token, the ID and one of org or repo go together. With none set, `/cron/rede - `recovered`: faucet stored the event, or a later attempt succeeded. - `gave_up` with `limit`: every ask was spent. A delivery is asked for 7 times, at 0, 10, 30, 70, 150, 310 and 630 minutes after the run that first read it. The count is raised before the request is sent, so a crash or a refused request costs an ask, and two runs cannot send the same one. - `gave_up` with `expired`: GitHub keeps a delivery for 72 hours, counted from its first attempt (the earliest attempt that a run read), and the delivery was still not stored. A delivery of a webhook that you replaced ends here too: an attempt ID of one webhook means nothing to another, so a run asks only for the deliveries of the webhook it was configured with. Those deliveries stay pending until faucet stores the event or 72 hours pass. -- `gap`: a time range of attempts that no run read. It names no delivery, because the attempts were never read. A range whose old end left GitHub's retention before a run read it is a gap, and so is the part of GitHub's 72 hours before the first run's lookback of 24 hours, and the open ranges of a webhook that the operator replaced. +- `gap`: a time range of attempts that no run read. It names no delivery, because the attempts were never read. A range whose old end left GitHub's retention before a run read it is a gap, together with the 5 minute overlap below that end, where an attempt that GitHub wrote late is listed. So is the part of GitHub's 72 hours before the first run's lookback of 24 hours, and so are the open ranges of a webhook that the operator replaced. The reply is JSON: `busy`, `pages`, `attempts`, `tracked`, `asked`, `recovered`, `gave_up` (`limit` and `expired`), `gaps`, `pending` and `ranges`. A run that fails answers 500 with the same JSON and an `error` member, so the outcomes that it recorded before it failed are in a reply as well as in the log. The first reply of a new database says `gaps: 1`, for the time before its lookback. Faucet deletes a final delivery and a recorded gap 72 hours after its last attempt, so a give-up is logged once. A pending delivery and an open range stay until they end, and the scan position is one row that stays. diff --git a/faucet/redeliver.go b/faucet/redeliver.go index 9207fc4..8b8f9f1 100644 --- a/faucet/redeliver.go +++ b/faucet/redeliver.go @@ -197,7 +197,8 @@ type RedeliverResult struct { // the ranges, newest first, with what the head read leaves of their page // limit. A cursor that GitHub gave is kept with the range, so a later run // continues right after the last attempt read. A range whose old end leaves -// GitHub's retention before it is read becomes a gap. +// GitHub's retention before it is read becomes a gap, together with the +// RedeliveryReadOverlap below that end, where a late attempt is listed. // // A delivery is asked for at most MaxAsksPerDelivery times, on the schedule // that AskOffsets gives, and at most MaxAsksPerRun times in a run, the delivery @@ -335,19 +336,15 @@ func (p pageRead) step(epoch int64, now time.Time) ScanStep { return ScanStep{Epoch: epoch, Now: now, Failed: p.failed, Succeeded: p.succeeded} } -// gapOf says which part of a range expired before a run read it. olderAt is -// the range's old end. coveredFrom is how far down a read has got, or the -// range's new end when it got nowhere. Only the part below the retention floor -// is gone. A part above it that holds no attempt was quiet. -func gapOf(olderAt, floor, coveredFrom time.Time) *RedeliveryGap { +func gapOf(bound, floor, coveredFrom time.Time) *RedeliveryGap { to := coveredFrom if floor.Before(to) { to = floor } - if !olderAt.Before(to) { + if !bound.Before(to) { return nil } - return &RedeliveryGap{From: olderAt, To: to} + return &RedeliveryGap{From: bound, To: to} } func (r *redeliveryRun) apply(ctx context.Context, step ScanStep, cause string) (ScanResult, error) { @@ -415,7 +412,7 @@ func (r *redeliveryRun) scan(ctx context.Context, scan RedeliveryScan) (bool, er covered = head.oldest } if gap == nil { - gap = gapOf(olderAt, floor, covered) + gap = gapOf(bound, floor, covered) } } else { step.Open = &RedeliveryRange{NewerAt: head.oldest, OlderAt: olderAt, Cursor: head.next} @@ -443,16 +440,17 @@ func (r *redeliveryRun) readRanges(ctx context.Context, ranges []RedeliveryRange live = append(live, rg) continue } - gap := RedeliveryGap{From: rg.OlderAt, To: laterOf(rg.NewerAt, rg.OlderAt)} + gap := RedeliveryGap{From: rg.OlderAt.Add(-RedeliveryReadOverlap), To: laterOf(rg.NewerAt, rg.OlderAt)} if _, err := r.apply(ctx, ScanStep{Epoch: r.epoch, Now: r.svc.clock.Now(), CloseID: rg.ID, Gap: &gap}, gapExpiredUnread); err != nil { return err } r.res.Ranges-- } for _, rg := range live { + bound := rg.OlderAt.Add(-RedeliveryReadOverlap) for r.res.Pages < r.state.maxPages && ctx.Err() == nil && !r.budget.pastScan() { now := r.svc.clock.Now() - p, err := r.read(ctx, rg.Cursor, rg.OlderAt.Add(-RedeliveryReadOverlap)) + p, err := r.read(ctx, rg.Cursor, bound) if errors.Is(err, errScanBudget) { return nil // the range stays open where the last page left it } @@ -473,7 +471,7 @@ func (r *redeliveryRun) readRanges(ctx context.Context, ranges []RedeliveryRange covered = p.oldest } step.CloseID = rg.ID - step.Gap = gapOf(rg.OlderAt, now.Add(-GitHubDeliveryRetention), covered) + step.Gap = gapOf(bound, now.Add(-GitHubDeliveryRetention), covered) if _, err := r.apply(ctx, step, gapExpiredUnread); err != nil { return err } diff --git a/faucet/redeliver_test.go b/faucet/redeliver_test.go index 9fdec19..d5d1bd7 100644 --- a/faucet/redeliver_test.go +++ b/faucet/redeliver_test.go @@ -597,7 +597,7 @@ func TestRedeliverRecordsOnlyTheExpiredPartOfARangeAsAGap(t *testing.T) { require.Len(t, lines, 2) require.Equal(t, "expired unread", lines[1]["cause"]) from, to := logTime(t, lines[1]["from"]), logTime(t, lines[1]["to"]) - require.WithinDuration(t, start, from, time.Minute, "the gap starts where the position of the earlier run was") + require.WithinDuration(t, start.Add(-faucet.RedeliveryReadOverlap), from, time.Minute, "the gap starts where the read of the range stops, the overlap below the position of the earlier run") require.WithinDuration(t, env.now().Add(-faucet.GitHubDeliveryRetention), to, time.Minute, "and ends where GitHub's list begins") } From 1b27c7f91f8518cefe63ed39af3c9e8ac8ee2f78 Mon Sep 17 00:00:00 2001 From: Thomas Kosiewski Date: Mon, 16 Nov 2026 09:12:59 +0000 Subject: [PATCH 3/3] docs(faucet): keep the doc comment of gapOf MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- _Generated with [`xum`](https://github.com/coder/xum) • Model: `anthropic:claude-opus-5-5` • Thinking: `high`_ --- faucet/redeliver.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/faucet/redeliver.go b/faucet/redeliver.go index 8b8f9f1..d84bf15 100644 --- a/faucet/redeliver.go +++ b/faucet/redeliver.go @@ -336,6 +336,10 @@ func (p pageRead) step(epoch int64, now time.Time) ScanStep { return ScanStep{Epoch: epoch, Now: now, Failed: p.failed, Succeeded: p.succeeded} } +// gapOf says which part of a read expired before a run made it. bound is where +// the read stops. coveredFrom is how far down a read has got, or the range's +// new end when it got nowhere. Only the part below the retention floor is gone. +// A part above it that holds no attempt was quiet. func gapOf(bound, floor, coveredFrom time.Time) *RedeliveryGap { to := coveredFrom if floor.Before(to) {