Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion README.md
Original file line number Diff line number Diff line change
Expand Up @@ -146,7 +146,7 @@ The token, the ID and one of org or repo go together. With none set, `/cron/rede
- `recovered`: faucet stored the event, or a later attempt succeeded.
- `gave_up` with `limit`: every ask was spent. A delivery is asked for 7 times, at 0, 10, 30, 70, 150, 310 and 630 minutes after the run that first read it. The count is raised before the request is sent, so a crash or a refused request costs an ask, and two runs cannot send the same one.
- `gave_up` with `expired`: GitHub keeps a delivery for 72 hours, counted from its first attempt (the earliest attempt that a run read), and the delivery was still not stored. A delivery of a webhook that you replaced ends here too: an attempt ID of one webhook means nothing to another, so a run asks only for the deliveries of the webhook it was configured with. Those deliveries stay pending until faucet stores the event or 72 hours pass.
- `gap`: a time range of attempts that no run read. It names no delivery, because the attempts were never read. A range whose old end left GitHub's retention before a run read it is a gap, and so is the part of GitHub's 72 hours before the first run's lookback of 24 hours, and the open ranges of a webhook that the operator replaced.
- `gap`: a time range of attempts that no run read. It names no delivery, because the attempts were never read. A range whose old end left GitHub's retention before a run read it is a gap, together with the 5 minute overlap below that end, where an attempt that GitHub wrote late is listed. So is the part of GitHub's 72 hours before the first run's lookback of 24 hours, and so are the open ranges of a webhook that the operator replaced.

The reply is JSON: `busy`, `pages`, `attempts`, `tracked`, `asked`, `recovered`, `gave_up` (`limit` and `expired`), `gaps`, `pending` and `ranges`. A run that fails answers 500 with the same JSON and an `error` member, so the outcomes that it recorded before it failed are in a reply as well as in the log. The first reply of a new database says `gaps: 1`, for the time before its lookback. Faucet deletes a final delivery and a recorded gap 72 hours after its last attempt, so a give-up is logged once. A pending delivery and an open range stay until they end, and the scan position is one row that stays.

Expand Down
26 changes: 14 additions & 12 deletions faucet/redeliver.go
Original file line number Diff line number Diff line change
Expand Up @@ -197,7 +197,8 @@ type RedeliverResult struct {
// the ranges, newest first, with what the head read leaves of their page
// limit. A cursor that GitHub gave is kept with the range, so a later run
// continues right after the last attempt read. A range whose old end leaves
// GitHub's retention before it is read becomes a gap.
// GitHub's retention before it is read becomes a gap, together with the
// RedeliveryReadOverlap below that end, where a late attempt is listed.
//
// A delivery is asked for at most MaxAsksPerDelivery times, on the schedule
// that AskOffsets gives, and at most MaxAsksPerRun times in a run, the delivery
Expand Down Expand Up @@ -335,19 +336,19 @@ func (p pageRead) step(epoch int64, now time.Time) ScanStep {
return ScanStep{Epoch: epoch, Now: now, Failed: p.failed, Succeeded: p.succeeded}
}

// gapOf says which part of a range expired before a run read it. olderAt is
// the range's old end. coveredFrom is how far down a read has got, or the
// range's new end when it got nowhere. Only the part below the retention floor
// is gone. A part above it that holds no attempt was quiet.
func gapOf(olderAt, floor, coveredFrom time.Time) *RedeliveryGap {
// gapOf says which part of a read expired before a run made it. bound is where
// the read stops. coveredFrom is how far down a read has got, or the range's
// new end when it got nowhere. Only the part below the retention floor is gone.
// A part above it that holds no attempt was quiet.
func gapOf(bound, floor, coveredFrom time.Time) *RedeliveryGap {
to := coveredFrom
if floor.Before(to) {
to = floor
}
if !olderAt.Before(to) {
if !bound.Before(to) {
return nil
}
return &RedeliveryGap{From: olderAt, To: to}
return &RedeliveryGap{From: bound, To: to}
}

func (r *redeliveryRun) apply(ctx context.Context, step ScanStep, cause string) (ScanResult, error) {
Expand Down Expand Up @@ -415,7 +416,7 @@ func (r *redeliveryRun) scan(ctx context.Context, scan RedeliveryScan) (bool, er
covered = head.oldest
}
if gap == nil {
gap = gapOf(olderAt, floor, covered)
gap = gapOf(bound, floor, covered)
}
} else {
step.Open = &RedeliveryRange{NewerAt: head.oldest, OlderAt: olderAt, Cursor: head.next}
Expand Down Expand Up @@ -443,16 +444,17 @@ func (r *redeliveryRun) readRanges(ctx context.Context, ranges []RedeliveryRange
live = append(live, rg)
continue
}
gap := RedeliveryGap{From: rg.OlderAt, To: laterOf(rg.NewerAt, rg.OlderAt)}
gap := RedeliveryGap{From: rg.OlderAt.Add(-RedeliveryReadOverlap), To: laterOf(rg.NewerAt, rg.OlderAt)}
if _, err := r.apply(ctx, ScanStep{Epoch: r.epoch, Now: r.svc.clock.Now(), CloseID: rg.ID, Gap: &gap}, gapExpiredUnread); err != nil {
return err
}
r.res.Ranges--
}
for _, rg := range live {
bound := rg.OlderAt.Add(-RedeliveryReadOverlap)
for r.res.Pages < r.state.maxPages && ctx.Err() == nil && !r.budget.pastScan() {
now := r.svc.clock.Now()
p, err := r.read(ctx, rg.Cursor, rg.OlderAt.Add(-RedeliveryReadOverlap))
p, err := r.read(ctx, rg.Cursor, bound)
if errors.Is(err, errScanBudget) {
return nil // the range stays open where the last page left it
}
Expand All @@ -473,7 +475,7 @@ func (r *redeliveryRun) readRanges(ctx context.Context, ranges []RedeliveryRange
covered = p.oldest
}
step.CloseID = rg.ID
step.Gap = gapOf(rg.OlderAt, now.Add(-GitHubDeliveryRetention), covered)
step.Gap = gapOf(bound, now.Add(-GitHubDeliveryRetention), covered)
if _, err := r.apply(ctx, step, gapExpiredUnread); err != nil {
return err
}
Expand Down
50 changes: 49 additions & 1 deletion faucet/redeliver_gaps_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,10 @@ type gapWindow struct{ from, to time.Time }

func (w gapWindow) holds(at time.Time) bool { return !at.Before(w.from) && !at.After(w.to) }

func (w gapWindow) String() string {
return w.from.Format(time.DateTime) + " to " + w.to.Format(time.DateTime)
}

// loggedGaps are the time ranges of the gaps that the runs logged.
func (e *redeliveryEnv) loggedGaps() []gapWindow {
var gaps []gapWindow
Expand Down Expand Up @@ -50,7 +54,7 @@ func requireNoSilentLoss(t *testing.T, env *redeliveryEnv, failures []ghAttempt)
for _, w := range gaps {
covered = covered || w.holds(f.at)
}
require.True(t, covered, "%s failed at %s and has no final state: no run tracked it and no gap covers it", f.guid, f.at.Format(time.TimeOnly))
require.True(t, covered, "%s failed at %s and has no final state: no run tracked it and no gap covers it, gaps %v", f.guid, f.at.Format(time.DateTime), gaps)
}
}

Expand Down Expand Up @@ -281,3 +285,47 @@ func TestALateAttemptIsReadWhileARangeStandsAtThePosition(t *testing.T) {
}
}
}

func TestALateFailureEndsInAGapWhenTheCronIsDownLongerThanGitHubKeepsIt(t *testing.T) {
t.Parallel()
env := newRedeliveryEnv(t)
env.clock.WithLogger(quartz.NoOpLogger)
a := env.gh.record("A", env.gh.now().Add(-time.Second), http200)
env.run()
l := env.gh.record("L", a.at.Add(-3*time.Second), 503)

env.advance(80 * time.Hour)
env.run()

requireNoSilentLoss(t, env, []ghAttempt{l})
}

func TestALateFailureEndsInAGapWhenGitHubDeletesItBeforeARunReadsTheRangeAboveIt(t *testing.T) {
t.Parallel()

for name, listStart := range map[string]time.Duration{
"the range expires unread": 8 * time.Hour,
"the range is read to the end of the list": time.Minute,
} {
t.Run(name, func(t *testing.T) {
t.Parallel()
env := newRedeliveryEnv(t, func(o *envOptions) { o.maxPages = 1 })
env.clock.WithLogger(quartz.NoOpLogger)
t0 := env.gh.now()
a := env.gh.record("A", t0.Add(-time.Second), http200)
env.run()
l := env.gh.record("L", a.at.Add(-3*time.Second), 503)
for i := range 150 {
env.gh.record(fmt.Sprintf("b-%03d", i), t0.Add(cronEvery-time.Duration(i+1)*2*time.Second), http200)
}
env.advance(cronEvery)
require.Equal(t, 1, env.run().Ranges, "the burst leaves a range down to A")

env.advance(t0.Add(listStart + faucet.GitHubDeliveryRetention).Sub(env.gh.now()))
env.useMaxPages(faucet.DefaultMaxPagesPerRun)
env.run()

requireNoSilentLoss(t, env, []ghAttempt{l})
})
}
}
2 changes: 1 addition & 1 deletion faucet/redeliver_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -597,7 +597,7 @@ func TestRedeliverRecordsOnlyTheExpiredPartOfARangeAsAGap(t *testing.T) {
require.Len(t, lines, 2)
require.Equal(t, "expired unread", lines[1]["cause"])
from, to := logTime(t, lines[1]["from"]), logTime(t, lines[1]["to"])
require.WithinDuration(t, start, from, time.Minute, "the gap starts where the position of the earlier run was")
require.WithinDuration(t, start.Add(-faucet.RedeliveryReadOverlap), from, time.Minute, "the gap starts where the read of the range stops, the overlap below the position of the earlier run")
require.WithinDuration(t, env.now().Add(-faucet.GitHubDeliveryRetention), to, time.Minute, "and ends where GitHub's list begins")
}

Expand Down
Loading