From ef091a61e475ef033c44df17b3e709cbed7d2806 Mon Sep 17 00:00:00 2001 From: Thomas Kosiewski Date: Mon, 16 Nov 2026 10:57:04 +0000 Subject: [PATCH 01/18] test(faucet): give the fake GitHub a Date, a slow page and a stale cursor The redesign of the redelivery clocks (#26) takes its gap floors from GitHub's Date header. Its tests need a GitHub whose Date leads, lags or is missing, whose pages take time to arrive, and that refuses the cursor of an attempt it has deleted. The fake GitHub had none of these. - `fakeGitHub.date` gives the Date of each list response, by call and cursor. A zero time is a response without a Date. - `redeliveryEnv.slowPages` moves the clock forward inside every list call, before GitHub reads its list. - `fakeGitHub.refusesStaleCursors` answers 422 to a cursor whose attempt left GitHub's retention. - `simSpec.setup` changes the env before the first delivery, and the simulation now follows GitHub's clock, so a test can set a skew. No production code changes, and no existing test changes. One new test pins what the three knobs do, because the tests of #26 lean on them. Refs #26 --- faucet/redeliver_sim_test.go | 14 ++++- faucet/redeliverharness_test.go | 105 ++++++++++++++++++++++++++++++++ 2 files changed, 116 insertions(+), 3 deletions(-) diff --git a/faucet/redeliver_sim_test.go b/faucet/redeliver_sim_test.go index 4fa52ef..fe0c35d 100644 --- a/faucet/redeliver_sim_test.go +++ b/faucet/redeliver_sim_test.go @@ -24,6 +24,9 @@ type simPhase struct { } type simSpec struct { + // setup changes the env before the first delivery: GitHub's clock, the + // Date that it sends, the time that a page takes. + setup func(*redeliveryEnv) phases []simPhase // outage is when faucet answers 503, relative to the first run. outageFrom, outageLength time.Duration @@ -73,7 +76,12 @@ func runSim(t *testing.T, spec simSpec) simOutcome { o.pageSize = spec.pageSize } }) - t0 := env.now() + if spec.setup != nil { + spec.setup(env) + } + // The deliveries, the outage and the cron follow GitHub's clock, which + // stamps the attempts. It is faucet's clock unless setup moved it. + t0 := env.gh.now() down := func(at time.Time) bool { return spec.outageLength > 0 && !at.Before(t0.Add(spec.outageFrom)) && at.Before(t0.Add(spec.outageFrom+spec.outageLength)) } @@ -109,8 +117,8 @@ func runSim(t *testing.T, spec simSpec) simOutcome { var out simOutcome var failedFirst []simDelivery next := 0 - for env.now().Before(t0.Add(spec.horizon)) { - now := env.now() + for env.gh.now().Before(t0.Add(spec.horizon)) { + now := env.gh.now() for next < len(deliveries) && !deliveries[next].at.After(now) { d := deliveries[next] next++ diff --git a/faucet/redeliverharness_test.go b/faucet/redeliverharness_test.go index 6cd7e94..f62cf79 100644 --- a/faucet/redeliverharness_test.go +++ b/faucet/redeliverharness_test.go @@ -66,6 +66,13 @@ type fakeGitHub struct { // listDelay and askDelay make every list call and every redelivery request // take that long, or until its context ends. listDelay, askDelay time.Duration + // date gives the Date header of a list response (DeliveryPage.At), from the + // number of the list call, its cursor and GitHub's clock when it answered. + // A zero time is a response without a Date. Nil sends GitHub's clock. + date func(call int, cursor string, at time.Time) time.Time + // refusesStaleCursors makes GitHub answer a cursor whose attempt it has + // deleted with an error. By default it lists on from where the attempt was. + refusesStaleCursors bool } var _ faucet.HookAdmin = (*fakeGitHub)(nil) @@ -180,11 +187,17 @@ func (g *fakeGitHub) ListDeliveries(ctx context.Context, cursor string) (faucet. if !ok { return faucet.DeliveryPage{}, fmt.Errorf("GitHub rejected the cursor") } + if g.refusesStaleCursors && !key.at.After(g.now().Add(-faucet.GitHubDeliveryRetention)) { + return faucet.DeliveryPage{}, fmt.Errorf("GitHub answered the delivery list with status 422") + } start = sort.Search(len(visible), func(i int) bool { return !olderThan(visible[i], key) }) } start = max(start, 0) end := max(start-g.pageSize, 0) page := faucet.DeliveryPage{At: g.now()} + if g.date != nil { + page.At = g.date(g.listCalls, cursor, g.now()) + } for i := start - 1; i >= end; i-- { a := visible[i] page.Deliveries = append(page.Deliveries, faucet.HookDelivery{ @@ -257,6 +270,86 @@ func (g *fakeGitHub) totalAsks() int { return n } +// The clock tests lean on what the fake GitHub does with a Date, a slow page +// and a stale cursor, so this pins each of them. +func TestFakeGitHubSendsTheDateTakesTheTimeAndRefusesTheCursorThatATestSets(t *testing.T) { + t.Parallel() + ctx := context.Background() + + t.Run("the Date is GitHub's clock unless the test gives one", func(t *testing.T) { + t.Parallel() + env := newRedeliveryEnv(t, func(o *envOptions) { o.pageSize = 1 }) + env.gh.skew = time.Hour + for i := range 2 { + env.gh.record(fmt.Sprintf("g-%d", i), env.gh.now().Add(-time.Duration(i+1)*time.Minute), http200) + } + + first, err := env.gh.ListDeliveries(ctx, "") + require.NoError(t, err) + require.True(t, env.now().Add(time.Hour).Equal(first.At), "the Date is on GitHub's clock, not faucet's") + + env.gh.date = func(call int, cursor string, at time.Time) time.Time { + if cursor != "" { + return time.Time{} + } + return at.Add(time.Duration(call) * time.Minute) + } + head, err := env.gh.ListDeliveries(ctx, "") + require.NoError(t, err) + require.True(t, env.gh.now().Add(2*time.Minute).Equal(head.At), "the second list call got the Date of call 2") + next, err := env.gh.ListDeliveries(ctx, head.Next) + require.NoError(t, err) + require.True(t, next.At.IsZero(), "a zero time is a response without a Date") + + env.gh.date = dateOff(-time.Second) + behind, err := env.gh.ListDeliveries(ctx, "") + require.NoError(t, err) + require.True(t, env.gh.now().Add(-time.Second).Equal(behind.At)) + }) + + t.Run("a slow page is read after GitHub's clock has moved", func(t *testing.T) { + t.Parallel() + env := newRedeliveryEnv(t) + leaving := env.gh.now().Add(-faucet.GitHubDeliveryRetention + time.Second) + env.gh.record("leaving", leaving, 503) + env.gh.record("staying", env.gh.now().Add(-time.Minute), http200) + asked := env.gh.now() + env.slowPages(2 * time.Second) + + page, err := env.gh.ListDeliveries(ctx, "") + require.NoError(t, err) + require.Len(t, page.Deliveries, 1, "GitHub deleted the old attempt while the request was on its way") + require.Equal(t, "staying", page.Deliveries[0].GUID) + require.True(t, asked.Add(2*time.Second).Equal(page.At), "the Date is the time of the answer, not of the request") + }) + + t.Run("a cursor of a deleted attempt lists on, or is refused when the test says so", func(t *testing.T) { + t.Parallel() + env := newRedeliveryEnv(t, func(o *envOptions) { o.pageSize = 1 }) + old := env.gh.now().Add(-faucet.GitHubDeliveryRetention + time.Minute) + env.gh.record("older", old, http200) + env.gh.record("old", old.Add(time.Second), http200) + env.gh.record("new", env.gh.now().Add(-time.Minute), http200) + head, err := env.gh.ListDeliveries(ctx, "") + require.NoError(t, err) + middle, err := env.gh.ListDeliveries(ctx, head.Next) + require.NoError(t, err) + require.Equal(t, "old", middle.Deliveries[0].GUID) + require.NotEmpty(t, middle.Next) + + env.advance(2 * time.Minute) + rest, err := env.gh.ListDeliveries(ctx, middle.Next) + require.NoError(t, err, "by default the list goes on from where the deleted attempt was") + require.Empty(t, rest.Deliveries) + + env.gh.refusesStaleCursors = true + _, err = env.gh.ListDeliveries(ctx, middle.Next) + require.ErrorContains(t, err, "422") + _, err = env.gh.ListDeliveries(ctx, head.Next) + require.NoError(t, err, "a cursor of an attempt that GitHub still lists is not stale") + }) +} + // captureSink keeps the structure of every log entry. type captureSink struct { mu sync.Mutex @@ -355,6 +448,18 @@ func (e *redeliveryEnv) advance(d time.Duration) { e.clock.Advance(d).MustWait(context.Background()) } +// slowPages makes every list request take d to reach GitHub: the clock moves +// forward by d inside the call, before GitHub reads its list. +func (e *redeliveryEnv) slowPages(d time.Duration) { + e.gh.listGate = func() { e.advance(d) } +} + +// dateOff is a Date that is d away from GitHub's clock: ahead of it when d is +// positive, behind it when d is negative. +func dateOff(d time.Duration) func(int, string, time.Time) time.Time { + return func(_ int, _ string, at time.Time) time.Time { return at.Add(d) } +} + func (e *redeliveryEnv) run() faucet.RedeliverResult { e.t.Helper() res, err := e.svc.Redeliver(context.Background()) From 85299f81142879ff6418a2cdb721d8c8782b12db Mon Sep 17 00:00:00 2001 From: Thomas Kosiewski Date: Mon, 16 Nov 2026 11:01:14 +0000 Subject: [PATCH 02/18] feat(faucet): store the Date of the last head read as the redelivery clear A redelivery run kept one time, the top, for two jobs: the next read bound and the old end of the next gap. Those jobs need opposite bounds (#26). The top also held GitHub's Date after an empty list, and a Date can run ahead of GitHub's list. This commit gives the stores a second time, the clear, so that the scan can keep attempt times and Dates apart. The scan starts to use it in the next commit. - `RedeliveryScan.Clear` and `ScanStep.Clear`. A step replaces the stored clear, also with an earlier time: the store keeps the latest Date, not the highest, so one Date that ran ahead does not stay. - Migration `000008_redelivery_clear` adds `clear_at`, moves the old top into it and resets the top. No row says whether its top is an attempt time or a Date, so the next run is a first run that knows when tracking began. The down migration gives an older build its position back. - A changed hook resets the clear with the top, and the gaps of its open ranges start at `OlderAt - RedeliveryReadOverlap`, where the read of the range would have stopped (#36 did this for the scan's own gaps). - `storetest.RunRedelivery` checks all of it, and the fake store and pgstore pass. A pgstore test runs the migration both ways. Refs #26 --- faucet/fakeredelivery_test.go | 11 +++- faucet/pgstore/database/models.go | 1 + faucet/pgstore/database/redelivery.sql.go | 29 ++++++--- .../000008_redelivery_clear.down.sql | 6 ++ .../migrations/000008_redelivery_clear.up.sql | 12 ++++ faucet/pgstore/pgstore_test.go | 59 ++++++++++++++++++- faucet/pgstore/queries/redelivery.sql | 8 ++- faucet/pgstore/redelivery.go | 13 +++- faucet/redeliverystore.go | 16 ++++- faucet/storetest/redelivery.go | 41 +++++++++++-- 10 files changed, 173 insertions(+), 23 deletions(-) create mode 100644 faucet/pgstore/migrations/000008_redelivery_clear.down.sql create mode 100644 faucet/pgstore/migrations/000008_redelivery_clear.up.sql diff --git a/faucet/fakeredelivery_test.go b/faucet/fakeredelivery_test.go index 57ce7e5..45a7fec 100644 --- a/faucet/fakeredelivery_test.go +++ b/faucet/fakeredelivery_test.go @@ -51,6 +51,7 @@ type fakeRedeliveryStore struct { epoch int64 leaseUntil time.Time top time.Time + clearAt time.Time ranges []*fakeRange nextRangeID int64 } @@ -98,14 +99,15 @@ func (s *fakeRedeliveryStore) AcquireRedeliveryLease(_ context.Context, hook str continue } r.gap, r.closedAt = true, now - scan.ClosedGaps = append(scan.ClosedGaps, faucet.RedeliveryGap{From: r.OlderAt, To: laterOf(r.NewerAt, r.OlderAt)}) + from := r.OlderAt.Add(-faucet.RedeliveryReadOverlap) + scan.ClosedGaps = append(scan.ClosedGaps, faucet.RedeliveryGap{From: from, To: laterOf(r.NewerAt, r.OlderAt)}) } - s.top = time.Time{} + s.top, s.clearAt = time.Time{}, time.Time{} } s.hook = hook s.epoch++ s.leaseUntil = now.Add(ttl) - scan.Epoch, scan.Top = s.epoch, s.top + scan.Epoch, scan.Top, scan.Clear = s.epoch, s.top, s.clearAt scan.Ranges = s.openRangesLocked() return scan, true, nil } @@ -189,6 +191,9 @@ func (s *fakeRedeliveryStore) ApplyRedeliveryScan(_ context.Context, step faucet if step.Top.After(s.top) { s.top = step.Top } + if !step.Clear.IsZero() { + s.clearAt = step.Clear + } switch { case step.Open != nil: s.nextRangeID++ diff --git a/faucet/pgstore/database/models.go b/faucet/pgstore/database/models.go index aeff1dd..527f996 100644 --- a/faucet/pgstore/database/models.go +++ b/faucet/pgstore/database/models.go @@ -65,6 +65,7 @@ type FaucetRedeliveryScan struct { Epoch int64 LeaseUntil sql.NullTime TopAt sql.NullTime + ClearAt sql.NullTime } type FaucetSubscription struct { diff --git a/faucet/pgstore/database/redelivery.sql.go b/faucet/pgstore/database/redelivery.sql.go index 7589e4f..0b1d4f0 100644 --- a/faucet/pgstore/database/redelivery.sql.go +++ b/faucet/pgstore/database/redelivery.sql.go @@ -323,6 +323,15 @@ func (q *Queries) ReleaseRedeliveryLease(ctx context.Context, epoch int64) error return err } +const setRedeliveryClear = `-- name: SetRedeliveryClear :exec +UPDATE faucet_redelivery_scan SET clear_at = $1::timestamptz +` + +func (q *Queries) SetRedeliveryClear(ctx context.Context, clearAt time.Time) error { + _, err := q.db.ExecContext(ctx, setRedeliveryClear, clearAt) + return err +} + const settleExpiredRedeliveries = `-- name: SettleExpiredRedeliveries :many UPDATE faucet_redeliveries SET state = 'gave_up', reason = 'expired', settled_at = $1::timestamptz @@ -441,25 +450,27 @@ UPDATE faucet_redelivery_scan SET hook = $1, epoch = epoch + 1, lease_until = $2::timestamptz, - top_at = CASE WHEN $3::boolean THEN NULL ELSE top_at END -RETURNING epoch, top_at + top_at = CASE WHEN $3::boolean THEN NULL ELSE top_at END, + clear_at = CASE WHEN $3::boolean THEN NULL ELSE clear_at END +RETURNING epoch, top_at, clear_at ` type TakeRedeliveryLeaseParams struct { - Hook string - LeaseUntil time.Time - ResetTop bool + Hook string + LeaseUntil time.Time + HookChanged bool } type TakeRedeliveryLeaseRow struct { - Epoch int64 - TopAt sql.NullTime + Epoch int64 + TopAt sql.NullTime + ClearAt sql.NullTime } func (q *Queries) TakeRedeliveryLease(ctx context.Context, arg TakeRedeliveryLeaseParams) (TakeRedeliveryLeaseRow, error) { - row := q.db.QueryRowContext(ctx, takeRedeliveryLease, arg.Hook, arg.LeaseUntil, arg.ResetTop) + row := q.db.QueryRowContext(ctx, takeRedeliveryLease, arg.Hook, arg.LeaseUntil, arg.HookChanged) var i TakeRedeliveryLeaseRow - err := row.Scan(&i.Epoch, &i.TopAt) + err := row.Scan(&i.Epoch, &i.TopAt, &i.ClearAt) return i, err } diff --git a/faucet/pgstore/migrations/000008_redelivery_clear.down.sql b/faucet/pgstore/migrations/000008_redelivery_clear.down.sql new file mode 100644 index 0000000..d367156 --- /dev/null +++ b/faucet/pgstore/migrations/000008_redelivery_clear.down.sql @@ -0,0 +1,6 @@ +-- A build before this migration keeps its position in top_at alone, and after +-- an empty list that position was GitHub's Date. A scan that has read no +-- attempt since the migration gets that position back. +UPDATE faucet_redelivery_scan SET top_at = COALESCE(top_at, clear_at); + +ALTER TABLE faucet_redelivery_scan DROP COLUMN clear_at; diff --git a/faucet/pgstore/migrations/000008_redelivery_clear.up.sql b/faucet/pgstore/migrations/000008_redelivery_clear.up.sql new file mode 100644 index 0000000..a208e19 --- /dev/null +++ b/faucet/pgstore/migrations/000008_redelivery_clear.up.sql @@ -0,0 +1,12 @@ +-- clear_at is the Date header of GitHub's answer to the last head read: when, +-- on GitHub's clock, a run last saw the top of the list. A run takes its gap +-- starts from it and never a read bound, because a Date can run ahead of +-- GitHub's list. top_at keeps attempt times only. +ALTER TABLE faucet_redelivery_scan ADD COLUMN clear_at timestamptz; + +-- Until now top_at was GitHub's Date when a run read an empty list, and no +-- row says which of the two it holds. So the old top moves to clear_at, and +-- the next run starts as a first run that knows when tracking began: it reads +-- the last 24 hours of the list again, and back to the old top less a margin +-- when that is older, and it records what lies below as a gap. +UPDATE faucet_redelivery_scan SET clear_at = top_at, top_at = NULL; diff --git a/faucet/pgstore/pgstore_test.go b/faucet/pgstore/pgstore_test.go index a650724..8b30461 100644 --- a/faucet/pgstore/pgstore_test.go +++ b/faucet/pgstore/pgstore_test.go @@ -119,7 +119,64 @@ func TestMigrateLeavesHostMigrationsTableAlone(t *testing.T) { require.EqualValues(t, 400, hostVersion) var faucetVersion int64 require.NoError(t, db.QueryRowContext(ctx, `SELECT version FROM `+pq.QuoteIdentifier(pgstore.MigrationsTable)).Scan(&faucetVersion)) - require.EqualValues(t, 7, faucetVersion) + require.EqualValues(t, 8, faucetVersion) +} + +// A build before the clear migration kept its position in top_at alone, and +// after an empty list that position was GitHub's Date. The migration cannot +// tell the two apart, so it makes the next run a first run that knows when +// tracking began. The down migration gives such a build its position back. +func TestTheClearMigrationMovesTheOldTopToTheClearAndBack(t *testing.T) { + t.Parallel() + db := newDatabase(t, testDatabaseURL(t)) + store := pgstore.New(db) + ctx := context.Background() + const hook = "/orgs/acme/hooks/7" + migrate := func(direction string) { + t.Helper() + statements, err := os.ReadFile("migrations/000008_redelivery_clear." + direction + ".sql") + require.NoError(t, err) + _, err = db.ExecContext(ctx, string(statements)) + require.NoError(t, err) + } + storedTop := func() (top sql.NullTime) { + t.Helper() + require.NoError(t, db.QueryRowContext(ctx, `SELECT top_at FROM faucet_redelivery_scan`).Scan(&top)) + return top + } + now := time.Date(2026, 11, 13, 12, 0, 0, 0, time.UTC) + oldTop := now.Add(-10 * time.Minute) + + lease, ok, err := store.AcquireRedeliveryLease(ctx, hook, now, time.Minute) + require.NoError(t, err) + require.True(t, ok) + _, err = store.ApplyRedeliveryScan(ctx, faucet.ScanStep{ + Epoch: lease.Epoch, Now: now, Top: oldTop, + Open: &faucet.RedeliveryRange{NewerAt: now.Add(-time.Hour), OlderAt: now.Add(-48 * time.Hour), Cursor: "cursor-1"}, + }) + require.NoError(t, err) + require.NoError(t, store.ReleaseRedeliveryLease(ctx, lease.Epoch)) + migrate("down") + require.True(t, oldTop.Equal(storedTop().Time), "this is the row of a build before the migration") + + migrate("up") + migrated, ok, err := store.AcquireRedeliveryLease(ctx, hook, now.Add(time.Minute), time.Minute) + require.NoError(t, err) + require.True(t, ok) + require.True(t, migrated.Top.IsZero(), "the next run is a first run: %s", migrated.Top) + require.True(t, oldTop.Equal(migrated.Clear), "that knows when the old build last saw the top of the list: %s", migrated.Clear) + require.False(t, migrated.HookChanged) + require.Len(t, migrated.Ranges, 1, "an open range stays open") + + migrate("down") + require.True(t, oldTop.Equal(storedTop().Time), "a scan that read nothing since the migration gets its position back") + + migrate("up") + attempt, date := now.Add(time.Hour), now.Add(time.Hour+time.Second) + _, err = store.ApplyRedeliveryScan(ctx, faucet.ScanStep{Epoch: migrated.Epoch, Now: now.Add(time.Hour), Top: attempt, Clear: date}) + require.NoError(t, err) + migrate("down") + require.True(t, attempt.Equal(storedTop().Time), "a scan that has read an attempt keeps that attempt's time, not the Date") } func testDatabaseURL(t *testing.T) string { diff --git a/faucet/pgstore/queries/redelivery.sql b/faucet/pgstore/queries/redelivery.sql index 5f72871..1925ebe 100644 --- a/faucet/pgstore/queries/redelivery.sql +++ b/faucet/pgstore/queries/redelivery.sql @@ -17,8 +17,9 @@ UPDATE faucet_redelivery_scan SET hook = sqlc.arg(hook), epoch = epoch + 1, lease_until = sqlc.arg(lease_until)::timestamptz, - top_at = CASE WHEN sqlc.arg(reset_top)::boolean THEN NULL ELSE top_at END -RETURNING epoch, top_at; + top_at = CASE WHEN sqlc.arg(hook_changed)::boolean THEN NULL ELSE top_at END, + clear_at = CASE WHEN sqlc.arg(hook_changed)::boolean THEN NULL ELSE clear_at END +RETURNING epoch, top_at, clear_at; -- name: ListOpenRedeliveryRanges :many SELECT id, newer_at, older_at, cursor @@ -64,6 +65,9 @@ RETURNING guid; -- name: RaiseRedeliveryTop :exec UPDATE faucet_redelivery_scan SET top_at = GREATEST(top_at, sqlc.arg(top)::timestamptz); +-- name: SetRedeliveryClear :exec +UPDATE faucet_redelivery_scan SET clear_at = sqlc.arg(clear_at)::timestamptz; + -- name: InsertRedeliveryRange :one INSERT INTO faucet_redelivery_ranges (newer_at, older_at, cursor) VALUES (sqlc.arg(newer_at), sqlc.arg(older_at), sqlc.arg(cursor)) diff --git a/faucet/pgstore/redelivery.go b/faucet/pgstore/redelivery.go index 5780a46..c547810 100644 --- a/faucet/pgstore/redelivery.go +++ b/faucet/pgstore/redelivery.go @@ -47,10 +47,11 @@ func (s *Store) AcquireRedeliveryLease(ctx context.Context, hook string, now tim return faucet.RedeliveryScan{}, false, xerrors.Errorf("close ranges of the old hook: %w", err) } for _, r := range closed { - scan.ClosedGaps = append(scan.ClosedGaps, faucet.RedeliveryGap{From: r.OlderAt.UTC(), To: laterOf(r.NewerAt, r.OlderAt).UTC()}) + from := r.OlderAt.Add(-faucet.RedeliveryReadOverlap) + scan.ClosedGaps = append(scan.ClosedGaps, faucet.RedeliveryGap{From: from.UTC(), To: laterOf(r.NewerAt, r.OlderAt).UTC()}) } } - taken, err := q.TakeRedeliveryLease(ctx, database.TakeRedeliveryLeaseParams{Hook: hook, LeaseUntil: now.Add(ttl), ResetTop: scan.HookChanged}) + taken, err := q.TakeRedeliveryLease(ctx, database.TakeRedeliveryLeaseParams{Hook: hook, LeaseUntil: now.Add(ttl), HookChanged: scan.HookChanged}) if err != nil { return faucet.RedeliveryScan{}, false, xerrors.Errorf("take lease: %w", err) } @@ -58,6 +59,9 @@ func (s *Store) AcquireRedeliveryLease(ctx context.Context, hook string, now tim if taken.TopAt.Valid { scan.Top = taken.TopAt.Time.UTC() } + if taken.ClearAt.Valid { + scan.Clear = taken.ClearAt.Time.UTC() + } ranges, err := q.ListOpenRedeliveryRanges(ctx) if err != nil { return faucet.RedeliveryScan{}, false, xerrors.Errorf("list ranges: %w", err) @@ -137,6 +141,11 @@ func (s *Store) ApplyRedeliveryScan(ctx context.Context, step faucet.ScanStep) ( return faucet.ScanResult{}, xerrors.Errorf("raise top: %w", err) } } + if !step.Clear.IsZero() { + if err := q.SetRedeliveryClear(ctx, step.Clear); err != nil { + return faucet.ScanResult{}, xerrors.Errorf("set clear: %w", err) + } + } if res.OpenedID, err = moveRange(ctx, q, step); err != nil { return faucet.ScanResult{}, err } diff --git a/faucet/redeliverystore.go b/faucet/redeliverystore.go index 5408255..e31a0be 100644 --- a/faucet/redeliverystore.go +++ b/faucet/redeliverystore.go @@ -80,10 +80,17 @@ type RedeliveryScan struct { // GitHub's clock when it read an empty list, or zero before the first // run. It is always a time that GitHub gave, never one of faucet's clock. Top time.Time + // Clear is the Date header that GitHub sent with the first page of the + // last run whose head read worked, or zero before one. It says when a run + // last saw the top of the list. A run takes gap starts from it, and the + // bound of a first run, and never another read bound. + Clear time.Time // Ranges are the open ranges, newest first. Ranges []RedeliveryRange // HookChanged is true when the lease found state of another webhook. The - // store has reset Top and closed that state's open ranges as ClosedGaps. + // store has reset Top and Clear and closed that state's open ranges as + // ClosedGaps. Each gap starts RedeliveryReadOverlap below the range's + // OlderAt, where the read of the range would have stopped. HookChanged bool ClosedGaps []RedeliveryGap } @@ -109,6 +116,10 @@ type ScanStep struct { // Top raises the stored top to this time. A zero time, or one that is not // later than the stored top, leaves it. Top time.Time + // Clear replaces the stored clear with this time, also when it is earlier: + // the store keeps the latest Date, not the highest, so that one Date that + // ran ahead does not stay. A zero time leaves the stored clear. + Clear time.Time // At most one of Open, Advance and CloseID is set. CloseID deletes an open // range that has been read to its end. Open *RedeliveryRange @@ -246,7 +257,8 @@ type RedeliveryStore interface { // AcquireRedeliveryLease starts a run for hook. It returns false when // another run holds a lease that has not run out at now. A lease lasts ttl. // When the stored scan belongs to another hook, the call resets it: Top - // goes back to zero and the open ranges close as gaps (see HookChanged). + // and Clear go back to zero and the open ranges close as gaps (see + // HookChanged). AcquireRedeliveryLease(ctx context.Context, hook string, now time.Time, ttl time.Duration) (RedeliveryScan, bool, error) // ReleaseRedeliveryLease ends the lease of epoch. It does nothing when // another lease has replaced it. diff --git a/faucet/storetest/redelivery.go b/faucet/storetest/redelivery.go index c571ebb..84d9801 100644 --- a/faucet/storetest/redelivery.go +++ b/faucet/storetest/redelivery.go @@ -36,6 +36,7 @@ func RunRedelivery(t *testing.T, newStore func(t *testing.T) RedeliveryStore) { {"ClaimPicksTheDeliveryDueLongest", claimPicksTheDeliveryDueLongest}, {"ConcurrentClaimsNeverReturnOneAskTwice", concurrentClaimsNeverReturnOneAskTwice}, {"ScanPositionSurvivesBetweenRuns", scanPositionSurvivesBetweenRuns}, + {"TheClearIsTheLatestTimeAStepSetNotTheHighest", theClearIsTheLatestTimeAStepSetNotTheHighest}, {"AFailedScanStepRecordsNothing", aFailedScanStepRecordsNothing}, {"ChangingTheHookClosesOpenRangesAsGaps", changingTheHookClosesOpenRangesAsGaps}, {"ClaimSkipsTheDeliveriesOfAnotherHook", claimSkipsTheDeliveriesOfAnotherHook}, @@ -115,6 +116,7 @@ func leaseIsExclusiveAndFencesAStaleHolder(t *testing.T, s RedeliveryStore) { first := acquire(t, s, testHook, t0) require.True(t, first.Top.IsZero()) + require.True(t, first.Clear.IsZero()) require.Empty(t, first.Ranges) require.False(t, first.HookChanged) @@ -347,6 +349,26 @@ func scanPositionSurvivesBetweenRuns(t *testing.T, s RedeliveryStore) { require.EqualValues(t, 2, deleted, "a gap is kept as a final state until it is purged, whether or not a range held it, and a finished range leaves no row") } +func theClearIsTheLatestTimeAStepSetNotTheHighest(t *testing.T, s RedeliveryStore) { + ctx := context.Background() + ahead, later := t0.Add(100*time.Hour), t0.Add(10*time.Minute) + + lease := acquire(t, s, testHook, t0) + apply(t, s, faucet.ScanStep{Epoch: lease.Epoch, Now: t0, Top: t0.Add(-time.Minute), Clear: ahead}) + require.NoError(t, s.ReleaseRedeliveryLease(ctx, lease.Epoch)) + + next := acquire(t, s, testHook, t0.Add(time.Minute)) + require.True(t, ahead.Equal(next.Clear), "a step's clear is stored: %s", next.Clear) + require.True(t, t0.Add(-time.Minute).Equal(next.Top), "and it is not the top") + apply(t, s, faucet.ScanStep{Epoch: next.Epoch, Now: later, Clear: later}) + apply(t, s, faucet.ScanStep{Epoch: next.Epoch, Now: later, Failed: []faucet.FailedAttempt{failedAt("guid-a", 1, t0)}}) + require.NoError(t, s.ReleaseRedeliveryLease(ctx, next.Epoch)) + + last := acquire(t, s, testHook, later.Add(time.Minute)) + require.True(t, later.Equal(last.Clear), "an earlier clear replaces a later one, and a step without a clear leaves it: %s", last.Clear) + require.True(t, t0.Add(-time.Minute).Equal(last.Top), "a clear never moves the top") +} + func normalizeRanges(ranges []faucet.RedeliveryRange) []faucet.RedeliveryRange { out := make([]faucet.RedeliveryRange, len(ranges)) for i, r := range ranges { @@ -359,7 +381,7 @@ func aFailedScanStepRecordsNothing(t *testing.T, s RedeliveryStore) { lease := acquire(t, s, testHook, t0) _, err := s.ApplyRedeliveryScan(context.Background(), faucet.ScanStep{ - Epoch: lease.Epoch, Now: t0, Failed: []faucet.FailedAttempt{failedAt("guid-a", 1, t0)}, Top: t0, + Epoch: lease.Epoch, Now: t0, Failed: []faucet.FailedAttempt{failedAt("guid-a", 1, t0)}, Top: t0, Clear: t0, Advance: &faucet.RangeAdvance{ID: 9999, NewerAt: t0, Cursor: "cursor"}, }) require.Error(t, err, "a range that does not exist cannot move") @@ -367,14 +389,16 @@ func aFailedScanStepRecordsNothing(t *testing.T, s RedeliveryStore) { require.Zero(t, settle(t, s, t0).Pending, "the attempts of a failed step are not recorded") require.NoError(t, s.ReleaseRedeliveryLease(context.Background(), lease.Epoch)) - require.True(t, acquire(t, s, testHook, t0).Top.IsZero(), "and neither is its top") + next := acquire(t, s, testHook, t0) + require.True(t, next.Top.IsZero(), "and neither is its top") + require.True(t, next.Clear.IsZero(), "nor its clear") } func changingTheHookClosesOpenRangesAsGaps(t *testing.T, s RedeliveryStore) { ctx := context.Background() lease := acquire(t, s, testHook, t0) apply(t, s, faucet.ScanStep{ - Epoch: lease.Epoch, Now: t0, Top: t0, + Epoch: lease.Epoch, Now: t0, Top: t0, Clear: t0.Add(time.Second), Open: &faucet.RedeliveryRange{NewerAt: t0.Add(-time.Hour), OlderAt: t0.Add(-48 * time.Hour), Cursor: "cursor-1"}, }) require.NoError(t, s.ReleaseRedeliveryLease(ctx, lease.Epoch)) @@ -382,15 +406,24 @@ func changingTheHookClosesOpenRangesAsGaps(t *testing.T, s RedeliveryStore) { same := acquire(t, s, testHook, t0.Add(time.Minute)) require.False(t, same.HookChanged) require.Len(t, same.Ranges, 1) + require.False(t, same.Clear.IsZero()) require.NoError(t, s.ReleaseRedeliveryLease(ctx, same.Epoch)) other := acquire(t, s, "/orgs/acme/hooks/8", t0.Add(2*time.Minute)) require.True(t, other.HookChanged) require.True(t, other.Top.IsZero(), "a new hook starts from scratch") + require.True(t, other.Clear.IsZero(), "the clear is a time of the old hook's list, and the new hook has not been read: %s", other.Clear) require.Empty(t, other.Ranges) require.Len(t, other.ClosedGaps, 1) - require.True(t, t0.Add(-48*time.Hour).Equal(other.ClosedGaps[0].From)) + require.True(t, t0.Add(-48*time.Hour-faucet.RedeliveryReadOverlap).Equal(other.ClosedGaps[0].From), + "the gap starts where the read of the range would have stopped, the overlap below its old end: %s", other.ClosedGaps[0].From) require.True(t, t0.Add(-time.Hour).Equal(other.ClosedGaps[0].To)) + require.NoError(t, s.ReleaseRedeliveryLease(ctx, other.Epoch)) + + again := acquire(t, s, "/orgs/acme/hooks/8", t0.Add(3*time.Minute)) + require.False(t, again.HookChanged) + require.True(t, again.Clear.IsZero(), "the reset is stored") + require.Empty(t, again.ClosedGaps, "the closed ranges are reported once") } func claimSkipsTheDeliveriesOfAnotherHook(t *testing.T, s RedeliveryStore) { From 3052c1aba1a3ae0a6c7db9e73ca2fdc3e0afb476 Mon Sep 17 00:00:00 2001 From: Thomas Kosiewski Date: Mon, 16 Nov 2026 11:27:42 +0000 Subject: [PATCH 03/18] fix(faucet): scan redeliveries by attempt times and a checked Date, not faucet's clock The redelivery sweep promises that every failed delivery that GitHub lists ends recovered, given up, or inside a counted gap. On main it broke that promise in two ways (#26). A gap ended at faucet's clock less 72 hours, so a clock behind GitHub's left a slice of deleted failures outside the gap, and a slow page widened the slice. The first run's gap started on faucet's clock, so a clock ahead of GitHub's left the oldest listed attempts unread and uncounted. And an empty list moved the position to GitHub's Date, which can run ahead of GitHub's list and put the next failures below the bound. One time served two roles that need opposite bounds. The scan now keeps three kinds of time apart by type (`faucet/redelivertime.go`), so a Date cannot become a read bound: - An attempt time (`delivered_at`, the top, a range's ends) is the only source of a read bound. An empty list leaves the top as it is. - A page's Date gives where GitHub's retention has ended, and only after a check: a missing Date, or one more than `RedeliveryDateSlack` (1h) behind the newest attempt known, fails the read. The scan checks it on the first page and on a page that ends the list, and nowhere else. A gap ends at `Date + 1s - 72h + slack`. - The clear, the stored Date of the last head read, gives gap starts: the head's gap starts at the later of `top - overlap` and `clear - RedeliveryDateMargin (6h) - overlap`. A first run reads 24 hours below the newest attempt and records `[newest - 72h, newest - 24h]` as its gap. After an empty first run or the migration it also reads back to `clear - margin`, and its gap reaches up to what expired since. A range whose new end is below `min(head Date, clear) - 72h` closes unread as a gap, so one Date far ahead closes nothing, and a range whose page fails ends this way. Tests. Each case of #26 and of the three design reviews is a test in `redeliver_gaps_test.go` or `redeliver_sim_test.go` that asserts no silent loss: a clock that lags or leads, a Date that leads, a slow page, one Date far ahead (the clear, and open ranges), an empty first run or the migration followed by an outage of 30 and 80 hours, a migrated top that was a Date ahead, range pages without a Date, a whole-second Date as far behind as the slack, a refused cursor, a changed hook, a quiet hook over 10 days, and the outage after which a quiet hook logs a gap (64h55m). Four tests pinned main's first run and empty list. An empty list now records no gap, the lookback counts from the newest attempt, and a gap ends at the oldest attempt that GitHub still lists. The cron tests' fake hook sends a Date, and the e2e lookback test moves its old delivery below the new bound. Refs #26 --- cmd/faucet/cron_internal_test.go | 12 +- faucet/fakeredelivery_test.go | 18 + faucet/hookadmin.go | 20 +- faucet/redeliver.go | 242 ++++++----- faucet/redeliver_gaps_test.go | 670 +++++++++++++++++++++++++++++++ faucet/redeliver_sim_test.go | 108 ++++- faucet/redeliver_test.go | 51 ++- faucet/redeliverharness_test.go | 13 +- faucet/redelivertime.go | 186 +++++++++ faucet/redeliverystore.go | 7 +- verify/tests/redeliver.e2e.ts | 3 +- 11 files changed, 1205 insertions(+), 125 deletions(-) create mode 100644 faucet/redelivertime.go diff --git a/cmd/faucet/cron_internal_test.go b/cmd/faucet/cron_internal_test.go index c677d3c..4ff1168 100644 --- a/cmd/faucet/cron_internal_test.go +++ b/cmd/faucet/cron_internal_test.go @@ -88,11 +88,17 @@ type cronHooks struct { func (*cronHooks) Hook() string { return "/orgs/octo-org/hooks/9" } +// ListDeliveries lists one attempt of an event that faucet does not store, +// with GitHub's Date, so that a run finds a list to start its tracking at. func (h *cronHooks) ListDeliveries(context.Context, string) (faucet.DeliveryPage, error) { h.mu.Lock() defer h.mu.Unlock() h.lists++ - return faucet.DeliveryPage{}, nil + listed := time.Date(2026, 11, 13, 12, 0, 0, 0, time.UTC) + return faucet.DeliveryPage{ + Deliveries: []faucet.HookDelivery{{ID: 1, GUID: "g-listed", DeliveredAt: listed, StatusCode: http.StatusOK, Event: "ping"}}, + At: listed.Add(time.Second), + }, nil } func (*cronHooks) Redeliver(context.Context, int64) error { return nil } @@ -373,7 +379,7 @@ func TestCronRedeliverReportsEveryOutcome(t *testing.T) { require.Equal(t, http.StatusOK, status, body) require.JSONEq(t, `{ - "busy": false, "pages": 1, "attempts": 0, "tracked": 0, "asked": 0, + "busy": false, "pages": 1, "attempts": 1, "tracked": 0, "asked": 0, "recovered": 2, "gave_up": {"limit": 1, "expired": 1}, "gaps": 1, "pending": 3, "ranges": 0 }`, body, "recoveries, both kinds of give-up and the gap before tracking are counted in the reply") require.Equal(t, 1, strings.Count(srv.logs.String(), "g-3"), "a give-up is logged once") @@ -433,7 +439,7 @@ func TestCronRedeliverFailureStillReportsTheOutcomesItRecorded(t *testing.T) { require.Equal(t, http.StatusInternalServerError, status, "a failed run is still a failure") require.JSONEq(t, `{ - "busy": false, "pages": 1, "attempts": 0, "tracked": 0, "asked": 0, + "busy": false, "pages": 1, "attempts": 1, "tracked": 0, "asked": 0, "recovered": 2, "gave_up": {"limit": 1, "expired": 0}, "gaps": 1, "pending": 3, "ranges": 0, "error": "redelivery failed" }`, body, "the outcomes that the run recorded before it failed are in the reply, and the log has them too") diff --git a/faucet/fakeredelivery_test.go b/faucet/fakeredelivery_test.go index 45a7fec..0851d99 100644 --- a/faucet/fakeredelivery_test.go +++ b/faucet/fakeredelivery_test.go @@ -323,6 +323,24 @@ func TestFakeRedeliveryStoreMatchesContract(t *testing.T) { storetest.RunRedelivery(t, func(*testing.T) storetest.RedeliveryStore { return newFakeRedeliveryStore(nil) }) } +// migrate does what migration 000008 does to the row of a build before it: +// the top moves to the clear, and the top is reset. +func (s *fakeRedeliveryStore) migrate() { + s.rmu.Lock() + defer s.rmu.Unlock() + s.clearAt, s.top = s.top, time.Time{} +} + +// seedMigrated puts a new store where the migration puts a build whose top +// was oldTop. Such a build took its top from the newest attempt, or from +// GitHub's Date when the list was empty. +func (s *fakeRedeliveryStore) seedMigrated(oldTop time.Time) { + s.rmu.Lock() + s.top = oldTop + s.rmu.Unlock() + s.migrate() +} + func (s *fakeRedeliveryStore) rowCount() int { s.rmu.Lock() defer s.rmu.Unlock() diff --git a/faucet/hookadmin.go b/faucet/hookadmin.go index a6704d9..30f47ea 100644 --- a/faucet/hookadmin.go +++ b/faucet/hookadmin.go @@ -47,11 +47,14 @@ type DeliveryPage struct { // Next is the cursor of the next, older page. It is empty on the last page. Next string // At is GitHub's own clock when it answered, which GitHub's Date header - // carries. It is zero when GitHub sent none. When the whole list is empty, - // Redeliver moves its position to At and never to the time of faucet's - // clock: the clocks differ, and a position ahead of GitHub's would make - // the next failed attempt look as if an earlier run had read it. A - // HookAdmin that leaves At zero leaves the position where it was. + // carries. It is zero when GitHub sent none. Redeliver takes from it where + // GitHub's retention has ended and when a run last saw the top of the + // list. It never takes a read position from it: a Date can run ahead of + // GitHub's list, and a position ahead of the list would make the next + // failed attempt look as if an earlier run had read it. Redeliver needs At + // on the first page and on a page that ends the list. There it fails the + // read when At is zero, or more than RedeliveryDateSlack before the newest + // attempt that it knows. At time.Time } @@ -66,7 +69,9 @@ type HookAdmin interface { // ListDeliveries returns one page of delivery attempts, newest first. // cursor is empty for the first page and a DeliveryPage.Next after that. // GitHub's cursor is opaque. A cursor that an earlier call returned must - // still continue the same list after newer attempts have arrived. + // still continue the same list after newer attempts have arrived. The + // call must not fail a page for its Date: it sets DeliveryPage.At, or + // leaves it zero, and Redeliver decides whether it needs it. ListDeliveries(ctx context.Context, cursor string) (DeliveryPage, error) // Redeliver asks GitHub to send the attempt with this ID again. Redeliver(ctx context.Context, attemptID int64) error @@ -190,7 +195,8 @@ func (a GitHubHookAdmin) ListDeliveries(ctx context.Context, cursor string) (Del if err != nil { return DeliveryPage{}, err } - // A missing or malformed Date leaves At zero. + // A missing or malformed Date leaves At zero and does not fail the page: + // most pages of a range need no Date, and Redeliver knows which do. at, _ := http.ParseTime(resp.Header.Get("Date")) page := DeliveryPage{Deliveries: make([]HookDelivery, len(raw)), Next: next, At: at} for i, d := range raw { diff --git a/faucet/redeliver.go b/faucet/redeliver.go index d84bf15..5eb76e9 100644 --- a/faucet/redeliver.go +++ b/faucet/redeliver.go @@ -26,9 +26,10 @@ const ( // late appears below one that ended earlier. The delivery timeout is 10 // seconds, so this is a wide margin. RedeliveryReadOverlap = 5 * time.Minute - // RedeliveryStartLookback is how far back the first run of a webhook reads. - // The part of GitHub's retention before it is recorded as one gap, so the - // first run does not spend its pages on a history that nobody waits for. + // RedeliveryStartLookback is how far below the newest attempt the first + // run of a webhook reads. The part of GitHub's retention before it is + // recorded as one gap, so the first run does not spend its pages on a + // history that nobody waits for. RedeliveryStartLookback = 24 * time.Hour redeliveryLeaseTTL = 2 * time.Minute @@ -200,12 +201,25 @@ type RedeliverResult struct { // GitHub's retention before it is read becomes a gap, together with the // RedeliveryReadOverlap below that end, where a late attempt is listed. // +// The scan reads no clock of faucet's, because that clock and GitHub's +// differ. Where a read stops comes from the time of an attempt, which GitHub's +// clock had passed when it listed the attempt. A first run is the exception: +// after an empty list or the migration it also reads back to the Date of +// that time, less RedeliveryDateMargin, and its gap covers what lies below. +// Where GitHub's retention has ended comes from the Date header of the page +// that saw the end of the list, and a gap ends RedeliveryDateSlack above it. +// After a silence, a gap starts at the Date of the run before, less +// RedeliveryDateMargin. A Date that is missing, or further behind an attempt +// than the slack, fails its read: on the first page the run fails, and on a +// later page the range stays open until it expires. +// // A delivery is asked for at most MaxAsksPerDelivery times, on the schedule // that AskOffsets gives, and at most MaxAsksPerRun times in a run, the delivery // that has waited longest first. The ask count is raised before the request, // so a crash costs an ask and two runs never send the same one. A request that // fails stops the asks of the run, and a failed first page stops them as well, -// because both mean that the token, the hook or GitHub is not working. +// because both mean that the token, the hook or GitHub is not working. A +// first page without a usable Date is a failed first page. // // When ctx has a deadline, the run keeps to it by construction. It reads the // list for half of the time that is left at its start, and a page that is not @@ -282,11 +296,8 @@ type pageRead struct { failed []FailedAttempt succeeded []string attempts int - // newest is the time of the newest attempt on the page. oldest is the - // time of the oldest one at or after the bound, and zero when there is none. - newest, oldest time.Time - // at is the page's DeliveryPage.At. - at time.Time + // oldest is the oldest attempt on the page at or after the bound. + oldest attemptTime reachedBound bool next string } @@ -295,30 +306,35 @@ type pageRead struct { // it: the page reached the range's bound, or GitHub's list ended. func (p pageRead) finished() bool { return p.reachedBound || p.next == "" } -func (r *redeliveryRun) read(ctx context.Context, cursor string, bound time.Time) (pageRead, error) { +// endOfList says that GitHub's list ended above the bound. What the read did +// not see between the two has expired, or was never there. +func (p pageRead) endOfList() bool { return !p.reachedBound && p.next == "" } + +// fetch reads one page of the list inside the time that the scan has. +func (r *redeliveryRun) fetch(ctx context.Context, cursor string) (DeliveryPage, error) { readCtx, cancel := until(ctx, r.budget.scanEnd) defer cancel() page, err := r.state.hooks.ListDeliveries(readCtx, cursor) if err != nil { if ctx.Err() == nil && readCtx.Err() != nil { - return pageRead{}, errScanBudget + return DeliveryPage{}, errScanBudget } - return pageRead{}, xerrors.Errorf("list deliveries: %w", err) + return DeliveryPage{}, xerrors.Errorf("list deliveries: %w", err) } r.res.Pages++ - p := pageRead{next: page.Next, at: page.At} + return page, nil +} + +// classify sorts the attempts of a page that are at or after bound. +func (r *redeliveryRun) classify(page DeliveryPage, bound readBound) pageRead { + p := pageRead{next: page.Next} for _, d := range page.Deliveries { - if d.DeliveredAt.After(p.newest) { - p.newest = d.DeliveredAt - } - if d.DeliveredAt.Before(bound) { + if bound.excludes(deliveredAt(d)) { p.reachedBound = true continue } p.attempts++ - if p.oldest.IsZero() || d.DeliveredAt.Before(p.oldest) { - p.oldest = d.DeliveredAt - } + p.oldest = p.oldest.orEarlier(deliveredAt(d)) if !storesDelivery(d.Event, d.Action) { continue } @@ -329,26 +345,23 @@ func (r *redeliveryRun) read(ctx context.Context, cursor string, bound time.Time p.failed = append(p.failed, FailedAttempt{GUID: d.GUID, AttemptID: d.ID, DeliveredAt: d.DeliveredAt}) } r.res.Attempts += p.attempts - return p, nil + return p } -func (p pageRead) step(epoch int64, now time.Time) ScanStep { - return ScanStep{Epoch: epoch, Now: now, Failed: p.failed, Succeeded: p.succeeded} +// step starts the ScanStep of a page. Its time is faucet's clock, which the +// store keeps for the settle, the asks and the purge. No bound and no gap +// comes from it. +func (r *redeliveryRun) step(p pageRead) ScanStep { + return ScanStep{Epoch: r.epoch, Now: r.svc.clock.Now(), Failed: p.failed, Succeeded: p.succeeded} } -// gapOf says which part of a read expired before a run made it. bound is where -// the read stops. coveredFrom is how far down a read has got, or the range's -// new end when it got nowhere. Only the part below the retention floor is gone. -// A part above it that holds no attempt was quiet. -func gapOf(bound, floor, coveredFrom time.Time) *RedeliveryGap { - to := coveredFrom - if floor.Before(to) { - to = floor - } - if !bound.Before(to) { +// gapBetween is the gap from one time to another. There is none without a +// start, and none when the start is not before the end. +func gapBetween(from, to time.Time) *RedeliveryGap { + if from.IsZero() || !from.Before(to) { return nil } - return &RedeliveryGap{From: bound, To: to} + return &RedeliveryGap{From: from, To: to} } func (r *redeliveryRun) apply(ctx context.Context, step ScanStep, cause string) (ScanResult, error) { @@ -386,42 +399,47 @@ func (r *redeliveryRun) countGap(ctx context.Context, gap RedeliveryGap, cause s // scan reads the head of the list and then the open ranges, newest first. It // returns whether the head read worked. +// +// The head read is one page. Its Date must be usable, because it becomes the +// stored clear and decides which ranges have expired. func (r *redeliveryRun) scan(ctx context.Context, scan RedeliveryScan) (bool, error) { - now := r.svc.clock.Now() - floor := now.Add(-GitHubDeliveryRetention) - olderAt, bound := scan.Top, scan.Top.Add(-RedeliveryReadOverlap) - var gap *RedeliveryGap - cause := gapExpiredUnread - if scan.Top.IsZero() { - olderAt = now.Add(-RedeliveryStartLookback) - bound = olderAt - gap, cause = &RedeliveryGap{From: floor, To: olderAt}, gapBeforeTracking - } - - head, err := r.read(ctx, "", bound) + top, anchor := topOf(scan), clearOf(scan) + page, err := r.fetch(ctx, "") if err != nil { return false, err } - step := head.step(r.epoch, now) - // The position is a time of GitHub's: the newest attempt, or GitHub's clock - // when the list is empty. Faucet's clock does not belong here (see - // DeliveryPage.At). - step.Top = head.newest - if step.Top.IsZero() { - step.Top = head.at - } - if head.finished() { - covered := now - if !head.oldest.IsZero() { - covered = head.oldest - } - if gap == nil { - gap = gapOf(bound, floor, covered) + date, err := checkDate(page, top) + if err != nil { + return false, xerrors.Errorf("first page: %w", err) + } + newest := newestOn(page) + + first := top.at.IsZero() + bound, olderAt := top.bound(), top + if first { + bound = anchor.firstRunBound(newest) + olderAt = bound.rangeEnd() + } + head := r.classify(page, bound) + step := r.step(head) + step.Top, step.Clear = newest.at, date.date + cause := gapExpiredUnread + switch { + case first && !newest.at.IsZero(): + // Tracking begins here, or it began at the clear. Below the bound lies + // the time before that. With a clear, a list that ends above the bound + // has also lost what expired since, and one gap holds both. + to := bound.stop + if anchor.set() && head.endOfList() { + to = laterOf(to, date.gapEnd(head.oldest)) } - } else { - step.Open = &RedeliveryRange{NewerAt: head.oldest, OlderAt: olderAt, Cursor: head.next} + step.Gap, cause = gapBetween(anchor.firstRunGapStart(newest), to), gapBeforeTracking + case head.endOfList(): + step.Gap = gapBetween(anchor.headGapStart(top), date.gapEnd(head.oldest)) + } + if !head.finished() { + step.Open = &RedeliveryRange{NewerAt: head.oldest.at, OlderAt: olderAt.at, Cursor: head.next} } - step.Gap = gap res, err := r.apply(ctx, step, cause) if err != nil { return false, err @@ -433,55 +451,87 @@ func (r *redeliveryRun) scan(ctx context.Context, scan RedeliveryScan) (bool, er r.res.Ranges++ ranges = append([]RedeliveryRange{opened}, ranges...) } - return true, r.readRanges(ctx, ranges) + live, err := r.closeExpired(ctx, ranges, date, anchor) + if err != nil { + return true, err + } + return true, r.readRanges(ctx, live, top.orLater(newest)) } -func (r *redeliveryRun) readRanges(ctx context.Context, ranges []RedeliveryRange) error { - floor := r.svc.clock.Now().Add(-GitHubDeliveryRetention) +// closeExpired closes, without a read, each range whose new end GitHub has +// deleted, and with it all of the range. It returns the others. A range whose +// page fails, because GitHub refuses its cursor or sends no usable Date, ends +// here. +func (r *redeliveryRun) closeExpired(ctx context.Context, ranges []RedeliveryRange, date pageDate, anchor clearAnchor) ([]RedeliveryRange, error) { + floor, ok := date.expiredBelow(anchor) + if !ok { + return ranges, nil + } live := ranges[:0:0] for _, rg := range ranges { if !rg.NewerAt.Before(floor) { live = append(live, rg) continue } - gap := RedeliveryGap{From: rg.OlderAt.Add(-RedeliveryReadOverlap), To: laterOf(rg.NewerAt, rg.OlderAt)} + gap := RedeliveryGap{From: rg.olderEnd().bound().stop, To: laterOf(rg.NewerAt, rg.OlderAt)} if _, err := r.apply(ctx, ScanStep{Epoch: r.epoch, Now: r.svc.clock.Now(), CloseID: rg.ID, Gap: &gap}, gapExpiredUnread); err != nil { - return err + return nil, err } r.res.Ranges-- } - for _, rg := range live { - bound := rg.OlderAt.Add(-RedeliveryReadOverlap) - for r.res.Pages < r.state.maxPages && ctx.Err() == nil && !r.budget.pastScan() { - now := r.svc.clock.Now() - p, err := r.read(ctx, rg.Cursor, bound) - if errors.Is(err, errScanBudget) { - return nil // the range stays open where the last page left it - } + return live, nil +} + +// readRanges reads the ranges, newest first, with the pages that the run has +// left. top is the newest attempt that the run knows. +func (r *redeliveryRun) readRanges(ctx context.Context, ranges []RedeliveryRange, top attemptTime) error { + for _, rg := range ranges { + if err := r.readRange(ctx, rg, top); err != nil { + return err + } + } + return nil +} + +// readRange reads one range until it ends or the run has no page left. A page +// that reaches the bound ends it without a gap: GitHub deletes in the order +// of the attempt times, so nothing above an attempt that it still lists has +// expired. A page that ends the list ends the range with the gap that its +// Date gives, and the range stays open when that Date is not usable. +func (r *redeliveryRun) readRange(ctx context.Context, rg RedeliveryRange, top attemptTime) error { + bound := rg.olderEnd().bound() + for r.res.Pages < r.state.maxPages && ctx.Err() == nil && !r.budget.pastScan() { + page, err := r.fetch(ctx, rg.Cursor) + if errors.Is(err, errScanBudget) { + return nil // the range stays open where the last page left it + } + if err != nil { + return err + } + p := r.classify(page, bound) + step := r.step(p) + newerEnd := rg.newerEnd().orEarlier(p.oldest) + switch { + case p.reachedBound: + step.CloseID = rg.ID + case p.endOfList(): + date, err := checkDate(page, top) if err != nil { - return err - } - step := p.step(r.epoch, now) - if !p.finished() { - step.Advance = &RangeAdvance{ID: rg.ID, NewerAt: earlierOf(rg.NewerAt, p.oldest), Cursor: p.next} - if _, err := r.apply(ctx, step, gapExpiredUnread); err != nil { - return err - } - rg.Cursor = p.next - continue - } - covered := rg.NewerAt - if !p.oldest.IsZero() { - covered = p.oldest + return xerrors.Errorf("range page: %w", err) } step.CloseID = rg.ID - step.Gap = gapOf(bound, now.Add(-GitHubDeliveryRetention), covered) - if _, err := r.apply(ctx, step, gapExpiredUnread); err != nil { - return err - } + step.Gap = gapBetween(bound.stop, date.gapEnd(newerEnd)) + default: + step.Advance = &RangeAdvance{ID: rg.ID, NewerAt: newerEnd.at, Cursor: p.next} + } + if _, err := r.apply(ctx, step, gapExpiredUnread); err != nil { + return err + } + if step.Advance == nil { r.res.Ranges-- - break + return nil } + rg.NewerAt, rg.Cursor = newerEnd.at, p.next } return nil } diff --git a/faucet/redeliver_gaps_test.go b/faucet/redeliver_gaps_test.go index b6deb3f..9d5c2f5 100644 --- a/faucet/redeliver_gaps_test.go +++ b/faucet/redeliver_gaps_test.go @@ -329,3 +329,673 @@ func TestALateFailureEndsInAGapWhenGitHubDeletesItBeforeARunReadsTheRangeAboveIt }) } } + +// The tests below give GitHub a clock of its own, a Date that leads or lags, +// pages that take time and cursors that it refuses. In each, a failure that +// GitHub listed must end tracked or inside a gap that a run logged. + +// requireListed fails unless GitHub's list holds the attempt now. A test calls +// it before the attempt expires, to show that a run could have read it. +func requireListed(t *testing.T, env *redeliveryEnv, a ghAttempt) { + t.Helper() + require.True(t, env.gh.lists(a.guid), "GitHub must list %s, or the scenario proves nothing", a.guid) +} + +func TestAnExpiredFailureIsInsideTheGapWhenFaucetsClockLagsGitHubs(t *testing.T) { + t.Parallel() + + for _, lag := range []time.Duration{10 * time.Minute, time.Hour} { + t.Run(lag.String(), func(t *testing.T) { + t.Parallel() + env := newRedeliveryEnv(t) + env.clock.WithLogger(quartz.NoOpLogger) + env.gh.skew = lag + t0 := env.gh.now() + env.gh.record("A", t0.Add(-time.Minute), http200) + env.run() + + // F leaves GitHub's list half a lag before the run after the outage. + // Faucet's clock says that GitHub still keeps it. + f := env.gh.record("F", t0.Add(8*time.Hour-lag/2), 503) + env.advance(8 * time.Hour) + requireListed(t, env, f) + env.advance(faucet.GitHubDeliveryRetention) + env.run() + + requireNoSilentLoss(t, env, []ghAttempt{f}) + }) + } +} + +func TestTheFirstRunsGapStartsBelowGitHubsListWhenFaucetsClockRunsAhead(t *testing.T) { + t.Parallel() + + for _, ahead := range []time.Duration{10 * time.Minute, time.Hour} { + t.Run(ahead.String(), func(t *testing.T) { + t.Parallel() + env := newRedeliveryEnv(t) + env.clock.WithLogger(quartz.NoOpLogger) + env.gh.skew = -ahead + t0 := env.gh.now() + // F is the oldest attempt that GitHub lists. On faucet's clock its + // 72 hours are over. + f := env.gh.record("F", t0.Add(-faucet.GitHubDeliveryRetention+ahead/2), 503) + env.gh.record("A", t0.Add(-time.Minute), http200) + requireListed(t, env, f) + + env.run() + + requireNoSilentLoss(t, env, []ghAttempt{f}) + }) + } +} + +func TestADateAheadOfGitHubsListDoesNotMoveThePositionPastTheNextFailure(t *testing.T) { + t.Parallel() + + for _, lead := range []time.Duration{10 * time.Minute, time.Hour} { + t.Run(lead.String(), func(t *testing.T) { + t.Parallel() + env := newRedeliveryEnv(t) + env.clock.WithLogger(quartz.NoOpLogger) + env.gh.date = dateOff(lead) + env.run() + env.advance(time.Minute) + f := env.gh.record("F", env.gh.now(), 503) + env.advance(cronEvery) + + res := env.run() + + require.Equal(t, 1, res.Tracked, "the list was empty before F, so F is above everything that a run has read") + require.True(t, env.stored(f.guid)) + }) + } +} + +func TestAPageThatIsSlowToArriveLosesNoFailureThatGitHubDeletesMeanwhile(t *testing.T) { + t.Parallel() + + for _, latency := range []time.Duration{600 * time.Millisecond, 5 * time.Second} { + t.Run(latency.String(), func(t *testing.T) { + t.Parallel() + env := newRedeliveryEnv(t, func(o *envOptions) { o.maxPages = 1 }) + env.clock.WithLogger(quartz.NoOpLogger) + t0 := env.gh.now() + env.gh.record("A", t0.Add(-time.Second), http200) + env.run() + + // F is the oldest attempt of a burst. The run after it has one page, + // which holds the newest 100, so F stays in the range below them. + f := env.gh.record("F", t0.Add(time.Minute), 503) + for i := range 150 { + env.gh.record(fmt.Sprintf("b-%03d", i), t0.Add(2*time.Minute+time.Duration(i)*4*time.Second), http200) + } + env.advance(15 * time.Minute) + require.Equal(t, 1, env.run().Ranges, "the burst leaves a range down to A") + requireListed(t, env, f) + + // GitHub lists F while the first page is on its way, and has deleted it + // when the page of the range arrives. + env.advance(f.at.Add(faucet.GitHubDeliveryRetention - latency*3/2).Sub(env.gh.now())) + env.useMaxPages(faucet.DefaultMaxPagesPerRun) + env.slowPages(latency) + require.Zero(t, env.run().Ranges, "the range is read to the end of the list") + + requireNoSilentLoss(t, env, []ghAttempt{f}) + }) + } +} + +func TestOneDateFarAheadDoesNotStayAsTheStartOfLaterGaps(t *testing.T) { + t.Parallel() + env := newRedeliveryEnv(t) + env.clock.WithLogger(quartz.NoOpLogger) + t0 := env.gh.now() + env.gh.record("A", t0.Add(-time.Minute), http200) + env.gh.date = dateOff(100 * time.Hour) + env.run() + env.gh.date = nil + env.advance(cronEvery) + env.run() + + f := env.gh.record("F", t0.Add(time.Hour), 503) + env.advance(2 * time.Hour) + requireListed(t, env, f) + env.advance(78 * time.Hour) + env.run() + + requireNoSilentLoss(t, env, []ghAttempt{f}) +} + +func TestOneDateFarAheadClosesNoRangeThatGitHubStillLists(t *testing.T) { + t.Parallel() + + for _, lead := range []time.Duration{80 * time.Hour, 30 * 24 * time.Hour} { + t.Run(lead.String(), func(t *testing.T) { + t.Parallel() + env := newRedeliveryEnv(t, func(o *envOptions) { o.maxPages = 1 }) + env.clock.WithLogger(quartz.NoOpLogger) + t0 := env.gh.now() + env.gh.record("A", t0.Add(-time.Second), http200) + env.run() + var failures []ghAttempt + for i := range 150 { + status := http200 + if i < 50 && i%10 == 0 { + status = 503 + } + a := env.gh.record(fmt.Sprintf("b-%03d", i), t0.Add(time.Minute+time.Duration(i)*2*time.Second), status) + if status != http200 { + failures = append(failures, a) + } + } + env.advance(cronEvery) + require.Equal(t, 1, env.run().Ranges, "one page holds the newest 100, and the failures are in the range below") + + env.advance(cronEvery) + env.useMaxPages(faucet.DefaultMaxPagesPerRun) + env.gh.date = func(_ int, cursor string, at time.Time) time.Time { + if cursor == "" { + return at.Add(lead) + } + return at + } + res := env.run() + + require.Zero(t, res.Gaps, "GitHub lists the whole range, so nothing expired") + require.Zero(t, res.Ranges) + for _, f := range failures { + require.True(t, env.tracked(f.guid), "%s is in a range that GitHub still lists, so the run reads it", f.guid) + } + }) + } +} + +func TestTheFirstRunClosesNoRangeOnTheWordOfOneDate(t *testing.T) { + t.Parallel() + env := newRedeliveryEnv(t, func(o *envOptions) { o.maxPages = 1 }) + env.clock.WithLogger(quartz.NoOpLogger) + t0 := env.gh.now() + var failures []ghAttempt + for i := range 150 { + status := http200 + if i < 50 && i%10 == 0 { + status = 503 + } + a := env.gh.record(fmt.Sprintf("b-%03d", i), t0.Add(-time.Hour+time.Duration(i)*2*time.Second), status) + if status != http200 { + failures = append(failures, a) + } + } + + // No clear is stored yet, so this Date is the only one that the run has. + env.gh.date = dateOff(80 * time.Hour) + first := env.run() + require.Equal(t, 1, first.Ranges, "the range that the first run opens stays open") + require.Equal(t, 1, first.Gaps, "and the only gap is the time before tracking") + + env.gh.date = nil + env.advance(cronEvery) + env.useMaxPages(faucet.DefaultMaxPagesPerRun) + require.Zero(t, env.run().Ranges) + for _, f := range failures { + require.True(t, env.tracked(f.guid), "%s is in a range that GitHub still lists, so the next run reads it", f.guid) + } +} + +func TestTheFirstRunOfAHookWithoutAClearRecordsOnlyTheTimeBeforeItsLookback(t *testing.T) { + t.Parallel() + env := newRedeliveryEnv(t) + env.clock.WithLogger(quartz.NoOpLogger) + t0 := env.gh.now() + // The list is two days stale and ends above the bound of the first run. + // Nothing was tracked before this run, so nothing that expired counts. + newest := env.gh.record("newer", t0.Add(-50*time.Hour), 503) + env.gh.record("older", t0.Add(-60*time.Hour), 503) + + res := env.run() + + require.Equal(t, 2, res.Tracked) + gaps := env.loggedGaps() + require.Len(t, gaps, 1) + require.True(t, newest.at.Add(-faucet.GitHubDeliveryRetention).Equal(gaps[0].from), "%s", gaps[0]) + require.True(t, newest.at.Add(-faucet.RedeliveryStartLookback).Equal(gaps[0].to), "the gap ends at the bound of the run, below every attempt that it read: %s", gaps[0]) +} + +func TestTrackingThatBeganWithAnEmptyListOrTheMigrationHoldsThroughAnOutage(t *testing.T) { + t.Parallel() + + for _, began := range []string{"an empty first run", "the migration"} { + for _, tc := range []struct { + outage time.Duration + listEmpty bool + }{{30 * time.Hour, false}, {80 * time.Hour, false}, {80 * time.Hour, true}} { + t.Run(fmt.Sprintf("%s,outage=%s,listEmpty=%v", began, tc.outage, tc.listEmpty), func(t *testing.T) { + t.Parallel() + env := newRedeliveryEnv(t) + env.clock.WithLogger(quartz.NoOpLogger) + t0 := env.gh.now() + if began == "the migration" { + env.store.seedMigrated(t0) + } else { + env.run() + } + f := env.gh.record("F", t0.Add(time.Hour), 503) + env.advance(2 * time.Hour) + requireListed(t, env, f) + env.advance(tc.outage - 2*time.Hour) + if !tc.listEmpty { + env.gh.record("G", env.gh.now().Add(-time.Second), http200) + } + for range 3 { + env.run() + env.advance(cronEvery) + } + + if tc.outage < faucet.GitHubDeliveryRetention { + require.True(t, env.stored(f.guid), "GitHub still lists F, and tracking began before F, so the run reads it") + return + } + requireNoSilentLoss(t, env, []ghAttempt{f}) + }) + } + } +} + +func TestAfterTheMigrationATopThatWasADateAheadDropsNoFailure(t *testing.T) { + t.Parallel() + + // A build before the migration took its top from GitHub's Date when the + // list was empty. A Date that ran ahead then put the next failures below + // the read bound. + for _, lead := range []time.Duration{time.Hour, 7 * time.Hour} { + t.Run(fmt.Sprintf("the cron keeps running,lead=%s", lead), func(t *testing.T) { + t.Parallel() + env := newRedeliveryEnv(t) + env.clock.WithLogger(quartz.NoOpLogger) + env.store.seedMigrated(env.gh.now().Add(lead)) + + var failures []ghAttempt + for i := range 12 { + failures = append(failures, env.gh.record(fmt.Sprintf("f-%02d", i), env.gh.now().Add(-time.Second), 503)) + env.advance(cronEvery) + env.run() + } + + for _, f := range failures { + require.True(t, env.stored(f.guid), "%s was listed above everything that a run had read", f.guid) + } + }) + } + // With an outage after the migration, the failures expire unread. The gap + // covers them while the Date was ahead by no more than the margin and the + // overlap. + for _, lead := range []time.Duration{time.Hour, 6 * time.Hour} { + t.Run(fmt.Sprintf("the cron is down for 80 hours,lead=%s", lead), func(t *testing.T) { + t.Parallel() + env := newRedeliveryEnv(t) + env.clock.WithLogger(quartz.NoOpLogger) + env.store.seedMigrated(env.gh.now().Add(lead)) + + var failures []ghAttempt + for i := range 30 { + env.advance(cronEvery) + failures = append(failures, env.gh.record(fmt.Sprintf("f-%02d", i), env.gh.now().Add(-time.Second), 503)) + } + requireListed(t, env, failures[0]) + env.advance(80 * time.Hour) + env.gh.record("G", env.gh.now().Add(-time.Second), http200) + for range 3 { + env.run() + env.advance(cronEvery) + } + + requireNoSilentLoss(t, env, failures) + }) + } +} + +func TestALateWriteBelowTheTopIsInsideTheGapWhileTheDateLeadsByLessThanTheMarginAndTheOverlap(t *testing.T) { + t.Parallel() + + for _, lead := range []time.Duration{0, time.Hour, faucet.RedeliveryDateMargin + faucet.RedeliveryReadOverlap - 10*time.Second} { + t.Run(lead.String(), func(t *testing.T) { + t.Parallel() + env := newRedeliveryEnv(t) + env.clock.WithLogger(quartz.NoOpLogger) + env.gh.date = dateOff(lead) + a := env.gh.record("A", env.gh.now().Add(-time.Second), http200) + env.run() + l := env.gh.record("L", a.at.Add(-3*time.Second), 503) + requireListed(t, env, l) + + env.advance(80 * time.Hour) + env.run() + + requireNoSilentLoss(t, env, []ghAttempt{l}) + }) + } +} + +func TestARangeIsReadThroughPagesThatCarryNoDate(t *testing.T) { + t.Parallel() + env := newRedeliveryEnv(t, func(o *envOptions) { o.maxPages = 1 }) + env.clock.WithLogger(quartz.NoOpLogger) + t0 := env.gh.now() + env.gh.record("Z", t0.Add(-10*time.Minute), http200) + env.gh.record("A", t0.Add(-time.Second), http200) + env.run() + var failures []ghAttempt + for i := range 250 { + status := http200 + if i < 150 && i%10 == 0 { + status = 503 + } + a := env.gh.record(fmt.Sprintf("b-%03d", i), t0.Add(time.Minute+time.Duration(i)*2*time.Second), status) + if status != http200 { + failures = append(failures, a) + } + } + env.advance(cronEvery) + require.Equal(t, 1, env.run().Ranges) + + // Only the first page of a run has a Date. Every page of a range either + // goes on or reaches the range's bound, and neither uses a Date. + env.gh.date = func(_ int, cursor string, at time.Time) time.Time { + if cursor != "" { + return time.Time{} + } + return at + } + env.advance(cronEvery) + env.useMaxPages(faucet.DefaultMaxPagesPerRun) + res := env.run() + + require.Zero(t, res.Ranges, "a page that reaches the bound of its range ends it") + require.Zero(t, res.Gaps) + for _, f := range failures { + require.True(t, env.tracked(f.guid), "%s is on a page without a Date, and the run reads it", f.guid) + } +} + +func TestARangeWhosePageFailsStaysOpenUntilGitHubHasDeletedItAndEndsAsAGap(t *testing.T) { + t.Parallel() + + const refusedCursor = "GitHub refuses the cursor of an attempt that it deleted" + for name, date := range map[string]func(int, string, time.Time) time.Time{ + "the page that ends the list has no Date": func(_ int, cursor string, at time.Time) time.Time { + if cursor != "" { + return time.Time{} + } + return at + }, + // The newest attempt that the run knows is a minute old, and it is on + // the first page of this run. The top that the store gave is 11 minutes + // old, and the attempts on the page of the range are days old. + "the Date of the page that ends the list is more than the slack behind the newest attempt of the run": func(_ int, cursor string, at time.Time) time.Time { + if cursor != "" { + return at.Add(-faucet.RedeliveryDateSlack - 2*time.Minute) + } + return at + }, + refusedCursor: nil, + } { + t.Run(name, func(t *testing.T) { + t.Parallel() + env := newRedeliveryEnv(t, func(o *envOptions) { o.maxPages = 1 }) + env.clock.WithLogger(quartz.NoOpLogger) + t0 := env.gh.now() + var failures []ghAttempt + for i := range 150 { + status := http200 + if i < 50 && i%10 == 0 { + status = 503 + } + a := env.gh.record(fmt.Sprintf("b-%03d", i), t0.Add(-12*time.Minute+time.Duration(i)*4*time.Second), status) + if status != http200 { + failures = append(failures, a) + } + } + require.Equal(t, 1, env.run().Ranges, "the one page holds the newest 100, and the failures are in the range below") + requireListed(t, env, failures[0]) + + // A run reads one page, so the range stays unread while the hook + // goes on. Then the fault begins, and the runs get their pages back. + // A Date fault begins while GitHub still lists the failures. A cursor + // is stale only after GitHub has deleted its attempt, which is newer + // than they are. + trickle := func() { + env.advance(cronEvery) + env.gh.record(fmt.Sprintf("t-%s", env.gh.now().Sub(t0)), env.gh.now().Add(-time.Minute), http200) + } + lastRunBeforeTheFault := t0.Add(faucet.GitHubDeliveryRetention - 30*time.Minute) + if name == refusedCursor { + lastRunBeforeTheFault = t0.Add(faucet.GitHubDeliveryRetention - 10*time.Minute) + } + for env.gh.now().Before(lastRunBeforeTheFault) { + trickle() + require.Equal(t, 1, env.run().Ranges) + } + trickle() + env.gh.date, env.gh.refusesStaleCursors = date, name == refusedCursor + env.useMaxPages(faucet.DefaultMaxPagesPerRun) + + res, err := env.svc.Redeliver(context.Background()) + require.Error(t, err, "the page of the range fails") + require.Equal(t, 1, res.Ranges, "and the range stays open") + open := 1 + for range 6 { + trickle() + res, _ = env.svc.Redeliver(context.Background()) + open = res.Ranges + } + + require.Zero(t, open, "the range ends without a read once GitHub has deleted all of it") + requireNoSilentLoss(t, env, failures) + }) + } +} + +func TestAWholeSecondDateAsFarBehindAsTheSlackStillEndsTheGapAboveWhatGitHubDeleted(t *testing.T) { + t.Parallel() + + for _, lag := range []time.Duration{0, faucet.RedeliveryDateSlack - time.Second, faucet.RedeliveryDateSlack} { + t.Run(lag.String(), func(t *testing.T) { + t.Parallel() + env := newRedeliveryEnv(t) + env.clock.WithLogger(quartz.NoOpLogger) + env.gh.date = func(_ int, _ string, at time.Time) time.Time { return at.Add(-lag).Truncate(time.Second) } + t0 := env.gh.now() + env.advance(time.Second) + env.gh.record("A", t0, http200) + env.run() + env.advance(time.Hour) + f := env.gh.record("F", t0.Add(time.Hour+400*time.Millisecond), 503) + requireListed(t, env, f) + + // The run starts 300 ms after GitHub deleted F. The Date drops the + // fraction of its second, which puts it before the deletion. + env.advance(f.at.Add(faucet.GitHubDeliveryRetention + 300*time.Millisecond).Sub(env.gh.now())) + require.False(t, env.gh.lists(f.guid)) + env.run() + + requireNoSilentLoss(t, env, []ghAttempt{f}) + }) + } +} + +func TestAFirstPageWithoutAUsableDateFailsTheRunWhichAsksForNothing(t *testing.T) { + t.Parallel() + + for name, date := range map[string]func(int, string, time.Time) time.Time{ + "no Date": func(int, string, time.Time) time.Time { return time.Time{} }, + "a Date more than the slack behind the newest attempt": dateOff(-faucet.RedeliveryDateSlack - 2*time.Minute), + } { + t.Run(name, func(t *testing.T) { + t.Parallel() + env := newRedeliveryEnv(t) + env.clock.WithLogger(quartz.NoOpLogger) + env.gh.answer = failing(400, "due") + env.gh.record("due", env.gh.now().Add(-time.Minute), 400) + require.Equal(t, 1, env.run().Asked) + env.advance(cronEvery) + env.gh.record("new", env.gh.now().Add(-time.Minute), 503) + + env.gh.date = date + res, err := env.svc.Redeliver(context.Background()) + + require.ErrorContains(t, err, "Date") + require.Zero(t, res.Tracked, "a page whose Date is not usable is not used") + require.Zero(t, res.Asked, "the second ask of due was due, and it did not go out") + require.Equal(t, 1, res.Pending, "the settle needs no GitHub, so it still runs") + + env.gh.date = nil + res = env.run() + require.Equal(t, 1, res.Tracked, "the failed run moved no position, so the next run reads what it left") + require.Equal(t, 2, res.Asked) + }) + } +} + +func TestRunsWithoutADateFailLoudlyAndTheNextRunWithOneLosesNothing(t *testing.T) { + t.Parallel() + + for _, listed := range []string{"an empty hook", "a hook whose failure expired during an outage"} { + t.Run(listed, func(t *testing.T) { + t.Parallel() + env := newRedeliveryEnv(t) + env.clock.WithLogger(quartz.NoOpLogger) + var failures []ghAttempt + if listed != "an empty hook" { + t0 := env.gh.now() + env.gh.record("A", t0.Add(-time.Minute), http200) + env.run() + failures = append(failures, env.gh.record("F", t0.Add(time.Hour), 503)) + env.advance(2 * time.Hour) + requireListed(t, env, failures[0]) + env.advance(78 * time.Hour) + } + + env.gh.date = func(int, string, time.Time) time.Time { return time.Time{} } + for range 12 { + res, err := env.svc.Redeliver(context.Background()) + require.ErrorContains(t, err, "Date", "a list without a Date says nothing about what GitHub has deleted") + require.Zero(t, res.Gaps) + env.advance(cronEvery) + } + env.gh.date = nil + res := env.run() + + require.Equal(t, len(failures), res.Gaps, "the run with a Date records what expired, and an empty hook lost nothing") + if len(failures) > 0 { + requireNoSilentLoss(t, env, failures) + } + }) + } +} + +func TestAChangedHookStartsWithoutTheOldHooksClear(t *testing.T) { + t.Parallel() + env := newRedeliveryEnv(t) + env.clock.WithLogger(quartz.NoOpLogger) + env.run() + env.advance(80 * time.Hour) + + // The host replaces the webhook after an outage. The old hook's clear is + // 80 hours old, which on that hook is a gap. The new hook has lost nothing. + env.gh.name = "/orgs/acme/hooks/8" + res := env.run() + + require.Zero(t, res.Gaps, "the new hook has no attempt yet, so nothing of it expired") + require.Empty(t, env.logged("redelivery gap")) +} + +func TestAChangedHooksOpenRangeEndsAsAGapThatHoldsTheOverlapBelowIt(t *testing.T) { + t.Parallel() + env := newRedeliveryEnv(t, func(o *envOptions) { o.maxPages = 1 }) + env.clock.WithLogger(quartz.NoOpLogger) + t0 := env.gh.now() + a := env.gh.record("A", t0.Add(-time.Second), http200) + env.run() + l := env.gh.record("L", a.at.Add(-3*time.Second), 503) + for i := range 150 { + env.gh.record(fmt.Sprintf("b-%03d", i), t0.Add(time.Minute+time.Duration(i)*2*time.Second), http200) + } + env.advance(cronEvery) + require.Equal(t, 1, env.run().Ranges, "the burst leaves a range down to A") + requireListed(t, env, l) + + env.gh.name = "/orgs/acme/hooks/8" + env.gh.attempts = nil + env.advance(cronEvery) + env.run() + + requireNoSilentLoss(t, env, []ghAttempt{l}) +} + +func TestAQuietHookLogsNoGapInTenDays(t *testing.T) { + t.Parallel() + + for name, date := range map[string]func(int, string, time.Time) time.Time{ + "its Date is right": nil, + "its Date is 4 minutes behind": dateOff(-4 * time.Minute), + "its Date is an hour ahead": dateOff(time.Hour), + "its Date is a whole second": func(_ int, _ string, at time.Time) time.Time { return at.Truncate(time.Second) }, + "its Date is an hour behind it": dateOff(-faucet.RedeliveryDateSlack), + } { + for _, listed := range []string{"an empty hook", "a hook with one attempt every 80 hours"} { + t.Run(listed+","+name, func(t *testing.T) { + t.Parallel() + env := newRedeliveryEnv(t) + env.clock.WithLogger(quartz.NoOpLogger) + env.gh.date = date + want := 0 + if listed != "an empty hook" { + for i := range 3 { + env.gh.record(fmt.Sprintf("a-%d", i), env.gh.now().Add(time.Hour+time.Duration(i)*80*time.Hour+500*time.Millisecond), http200) + } + want = 1 + } + + var gaps int + for range 10 * 24 * int(time.Hour/cronEvery) { + gaps += env.run().Gaps + env.advance(cronEvery) + } + + require.Equal(t, want, gaps, "a silence is not a gap, and only the time before the first attempt is one") + for _, line := range env.logged("redelivery gap") { + require.Equal(t, "before tracking began", line["cause"]) + } + }) + } + } +} + +func TestAQuietHookLogsAGapOnlyAfterAnOutageThatTheDateMarginsDoNotCover(t *testing.T) { + t.Parallel() + + // After a silence a gap is [clear - margin - overlap, Date - retention + + // slack]. It is empty until the runs are this far apart. + threshold := faucet.GitHubDeliveryRetention - faucet.RedeliveryDateSlack - faucet.RedeliveryDateMargin - faucet.RedeliveryReadOverlap + require.Equal(t, 64*time.Hour+55*time.Minute, threshold) + for _, listed := range []string{"an empty hook", "a hook whose last attempt is 10 hours old"} { + for _, tc := range []struct { + apart time.Duration + gaps int + }{{threshold - time.Minute, 0}, {threshold + time.Minute, 1}} { + t.Run(fmt.Sprintf("%s,apart=%s", listed, tc.apart), func(t *testing.T) { + t.Parallel() + env := newRedeliveryEnv(t) + env.clock.WithLogger(quartz.NoOpLogger) + if listed != "an empty hook" { + env.gh.record("A", env.gh.now().Add(-10*time.Hour), http200) + } + env.run() + env.advance(tc.apart) + + require.Equal(t, tc.gaps, env.run().Gaps) + }) + } + } +} diff --git a/faucet/redeliver_sim_test.go b/faucet/redeliver_sim_test.go index fe0c35d..de05080 100644 --- a/faucet/redeliver_sim_test.go +++ b/faucet/redeliver_sim_test.go @@ -26,8 +26,10 @@ type simPhase struct { type simSpec struct { // setup changes the env before the first delivery: GitHub's clock, the // Date that it sends, the time that a page takes. - setup func(*redeliveryEnv) - phases []simPhase + setup func(*redeliveryEnv) + // beforeRun runs before each run, with the time since the first run. + beforeRun func(env *redeliveryEnv, since time.Duration) + phases []simPhase // outage is when faucet answers 503, relative to the first run. outageFrom, outageLength time.Duration // cronDuringOutage is false when the platform that runs the cron is faucet @@ -128,6 +130,9 @@ func runSim(t *testing.T, spec simSpec) simOutcome { } } if spec.cronDuringOutage || !down(now) { + if spec.beforeRun != nil { + spec.beforeRun(env, now.Sub(t0)) + } res := env.run() out.runs = append(out.runs, res) out.sumRecovered += res.Recovered @@ -340,3 +345,102 @@ func rangesOf(runs []faucet.RedeliverResult) []int { } return counts } + +func TestSimADateThatLagsLessThanTheSlackFailsNoRunAndLosesNothing(t *testing.T) { + t.Parallel() + + // An HTTP Date is a whole second, and the attempt times have fractions, so + // even a Date that is right lies before the newest attempt on its page. + for _, lag := range []time.Duration{0, 2 * time.Second, 30 * time.Minute, 59 * time.Minute} { + t.Run(lag.String(), func(t *testing.T) { + t.Parallel() + out := runSim(t, simSpec{ + setup: func(env *redeliveryEnv) { + env.gh.date = func(_ int, _ string, at time.Time) time.Time { return at.Add(-lag).Truncate(time.Second) } + }, + phases: steady(1200, 4*time.Hour), + blip: func(_, i int) bool { return i%100 == 37 }, + horizon: 4 * time.Hour, + }) + t.Logf("RESULT 1200 an hour, 1%% blips, a whole-second Date %s behind: %s", lag, out) + require.Greater(t, out.failed, 10) + require.Zero(t, out.silentLost) + require.Equal(t, out.failed, out.recovered, "every run read its list, so every blip is recovered") + }) + } +} + +func TestSimAnOutageLongerThanGitHubKeepsADeliveryLosesNothingSilentlyWhateverTheClocks(t *testing.T) { + t.Parallel() + + // skew is GitHub's clock minus faucet's. Faucet and its cron are down for + // the outage, and GitHub deletes the first failures before a run reads them. + for _, skew := range []time.Duration{-25 * time.Hour, -time.Hour, 0, time.Hour, 25 * time.Hour} { + for _, outage := range []time.Duration{71 * time.Hour, 80 * time.Hour} { + t.Run(fmt.Sprintf("skew=%s,outage=%s", skew, outage), func(t *testing.T) { + t.Parallel() + out := runSim(t, simSpec{ + setup: func(env *redeliveryEnv) { env.gh.skew = skew }, + phases: steady(6, 96*time.Hour), + outageFrom: time.Hour, + outageLength: outage, + blip: func(_, i int) bool { return i%2 == 0 }, + horizon: 96 * time.Hour, + }) + t.Logf("RESULT 6 an hour, faucet and cron down for %s, GitHub's clock %s ahead of faucet's: %s", outage, skew, out) + require.Greater(t, out.failed, 400) + require.Zero(t, out.silentLost) + if outage > faucet.GitHubDeliveryRetention { + require.Positive(t, out.inGap, "what GitHub deleted during the outage is inside a gap") + } + }) + } + } +} + +func TestSimTheMigrationReadsTheLastDayAgainAndLosesNothingSilently(t *testing.T) { + t.Parallel() + // The law of the capacity at a hundredth of the scale: pages of 1 attempt, + // 20 a run, so 42 and 70 an hour stand for 4,200 and 7,000. The store is + // migrated 6 hours in, while the hook delivers. The failures that count + // are the ones inside the lookback of the first run, and all later ones. + const pages, pageSize = 20, 1 + + for _, tc := range []struct { + name string + perHour int + catchesUp bool + }{ + {"below the capacity", 42, true}, + {"near the capacity", 70, false}, + } { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + migrated := false + out := runSim(t, simSpec{ + phases: []simPhase{{from: -27 * time.Hour, to: 90 * time.Hour, perHour: tc.perHour}}, + blip: func(_, i int) bool { return i%25 == 7 }, + beforeRun: func(env *redeliveryEnv, since time.Duration) { + if since == 6*time.Hour { + env.store.migrate() + migrated = true + } + }, + countFrom: -23 * time.Hour, + horizon: 90 * time.Hour, + pages: pages, + pageSize: pageSize, + }) + t.Logf("RESULT %d an hour with pages of %d, migrated at 6h: %s, ranges after the last run %d", tc.perHour, pageSize, out, out.runs[len(out.runs)-1].Ranges) + require.True(t, migrated) + require.Greater(t, out.failed, 100) + require.Zero(t, out.silentLost) + if tc.catchesUp { + require.Equal(t, out.failed, out.recovered, "the sweep reads the last day again, and the migration costs no recovery") + require.Zero(t, out.runs[len(out.runs)-1].Ranges) + return + } + require.Positive(t, out.inGap, "what the sweep cannot read again before GitHub deletes it is inside a gap") + }) + } +} diff --git a/faucet/redeliver_test.go b/faucet/redeliver_test.go index d5d1bd7..fdc97ff 100644 --- a/faucet/redeliver_test.go +++ b/faucet/redeliver_test.go @@ -5,6 +5,10 @@ import ( "encoding/json" "fmt" "net/http" + "os" + "path/filepath" + "regexp" + "strings" "sync" "sync/atomic" "testing" @@ -467,8 +471,8 @@ func TestTheFirstRunReadsTheLastDayAndRecordsTheRestOfRetentionAsOneGap(t *testi t.Parallel() env := newRedeliveryEnv(t) start := env.now() - env.gh.record("inside", start.Add(-20*time.Hour), 503) - env.gh.record("before", start.Add(-30*time.Hour), 503) + newest := env.gh.record("inside", start.Add(-20*time.Hour), 503) + env.gh.record("before", newest.at.Add(-faucet.RedeliveryStartLookback-time.Hour), 503) first := env.run() require.Equal(t, 1, first.Tracked, "only the delivery inside the lookback is tracked") @@ -480,12 +484,12 @@ func TestTheFirstRunReadsTheLastDayAndRecordsTheRestOfRetentionAsOneGap(t *testi require.Equal(t, "before tracking began", lines[0]["cause"]) require.Equal(t, "gap", lines[0]["reason"]) from, to := logTime(t, lines[0]["from"]), logTime(t, lines[0]["to"]) - require.True(t, start.Add(-faucet.GitHubDeliveryRetention).Equal(from)) - require.True(t, start.Add(-faucet.RedeliveryStartLookback).Equal(to)) + require.True(t, newest.at.Add(-faucet.GitHubDeliveryRetention).Equal(from), "the gap is counted from the newest attempt, which is a time of GitHub's: %s", from) + require.True(t, newest.at.Add(-faucet.RedeliveryStartLookback).Equal(to), "and so is the lookback: %s", to) require.False(t, env.stored("before"), "the delivery before the lookback is inside the recorded gap, so no ask goes out for it") } -func TestAnEmptyListIsCoveredSoTheFirstGapIsNotRecordedAgain(t *testing.T) { +func TestAnEmptyListRecordsNoGapAndTheFirstAttemptRecordsTheFirstGapOnce(t *testing.T) { t.Parallel() env := newRedeliveryEnv(t) @@ -494,11 +498,18 @@ func TestAnEmptyListIsCoveredSoTheFirstGapIsNotRecordedAgain(t *testing.T) { gaps += env.run().Gaps env.advance(cronEvery) } + require.Zero(t, gaps, "nothing is listed, so nothing lies before tracking and nothing expired") - require.Equal(t, 1, gaps) + env.gh.record("first", env.gh.now().Add(-time.Minute), http200) + for range 5 { + gaps += env.run().Gaps + env.advance(cronEvery) + } + require.Equal(t, 1, gaps, "the first attempt starts the tracking, and the time before it is one gap") + require.Equal(t, "before tracking began", env.logged("redelivery gap")[0]["cause"]) } -func TestAnEmptyListMovesThePositionToGitHubsClockNotFaucets(t *testing.T) { +func TestAnEmptyListLeavesThePositionSoTheNextFailureIsReadWhateverTheClocks(t *testing.T) { t.Parallel() // skew is GitHub's clock minus faucet's. A position on faucet's clock puts @@ -510,7 +521,7 @@ func TestAnEmptyListMovesThePositionToGitHubsClockNotFaucets(t *testing.T) { env := newRedeliveryEnv(t) env.gh.skew = skew - require.Equal(t, 1, env.run().Gaps, "an empty list is the first run") + require.Zero(t, env.run().Gaps, "an empty list records nothing") env.advance(time.Minute) env.gh.record("g1", env.gh.now(), 503) @@ -597,8 +608,9 @@ func TestRedeliverRecordsOnlyTheExpiredPartOfARangeAsAGap(t *testing.T) { require.Len(t, lines, 2) require.Equal(t, "expired unread", lines[1]["cause"]) from, to := logTime(t, lines[1]["from"]), logTime(t, lines[1]["to"]) - require.WithinDuration(t, start.Add(-faucet.RedeliveryReadOverlap), from, time.Minute, "the gap starts where the read of the range stops, the overlap below the position of the earlier run") - require.WithinDuration(t, env.now().Add(-faucet.GitHubDeliveryRetention), to, time.Minute, "and ends where GitHub's list begins") + require.WithinDuration(t, start.Add(-faucet.RedeliveryDateMargin-faucet.RedeliveryReadOverlap), from, time.Minute, + "the gap starts where the read of the range stops. The first run found an empty list at start, so the range ends at that Date less the margin, and its read stops the overlap below") + require.WithinDuration(t, env.now().Add(-faucet.GitHubDeliveryRetention), to, 2*time.Minute, "and ends at the oldest attempt that GitHub still lists, one attempt above where its list begins") } func TestTwoRunsAtOnceSendOneRequestPerDelivery(t *testing.T) { @@ -713,3 +725,22 @@ func TestEveryRedeliveryLogLineHoldsNoCursorOrURL(t *testing.T) { require.NotContains(t, text, "v1_", "a cursor is GitHub's opaque value and stays out of logs") require.NotContains(t, text, "http") } + +// The scan keeps the time of an attempt, GitHub's Date and the stored clear +// apart by type, so that a Date cannot become a read bound. That holds while +// redelivertime.go alone makes a value of those types. +func TestOnlyRedeliverTimeMakesTheTimesThatTheScanKeepsApart(t *testing.T) { + t.Parallel() + made := regexp.MustCompile(`\b(attemptTime|readBound|pageDate|clearAnchor)\{`) + files, err := filepath.Glob("*.go") + require.NoError(t, err) + require.Contains(t, files, "redelivertime.go") + for _, file := range files { + if file == "redelivertime.go" || strings.HasSuffix(file, "_test.go") { + continue + } + source, err := os.ReadFile(file) + require.NoError(t, err) + require.Empty(t, made.FindAllString(string(source), -1), "%s makes a time of one role from a bare time", file) + } +} diff --git a/faucet/redeliverharness_test.go b/faucet/redeliverharness_test.go index f62cf79..3039ebb 100644 --- a/faucet/redeliverharness_test.go +++ b/faucet/redeliverharness_test.go @@ -214,10 +214,17 @@ func (g *fakeGitHub) ListDeliveries(ctx context.Context, cursor string) (faucet. return page, nil } +// lists says whether GitHub's list holds an attempt at guid now. +func (g *fakeGitHub) lists(guid string) bool { + g.mu.Lock() + defer g.mu.Unlock() + return slices.ContainsFunc(g.visibleLocked(), func(a ghAttempt) bool { return a.guid == guid }) +} + func (g *fakeGitHub) attemptLocked(id int64) (ghAttempt, bool) { - for _, a := range g.attempts { - if a.id == id { - return a, true + for i := range g.attempts { + if g.attempts[i].id == id { + return g.attempts[i], true } } return ghAttempt{}, false diff --git a/faucet/redelivertime.go b/faucet/redelivertime.go new file mode 100644 index 0000000..5242b5b --- /dev/null +++ b/faucet/redelivertime.go @@ -0,0 +1,186 @@ +package faucet + +import ( + "time" + + "golang.org/x/xerrors" +) + +const ( + // RedeliveryDateSlack is how far GitHub's Date header may lie behind the + // clock that deletes attempts. A gap ends this much above the end of + // GitHub's retention that the Date gives, and a Date that is further + // behind an attempt that GitHub has listed fails the read. + RedeliveryDateSlack = time.Hour + // RedeliveryDateMargin is how far GitHub's Date header may run ahead of + // GitHub's list. A gap that starts at the Date of the last head read + // starts this much below it. + RedeliveryDateMargin = 6 * time.Hour + + // An HTTP Date drops the fraction of its second. + redeliveryDateResolution = time.Second +) + +// A scan works with three kinds of time. An attempt time is never ahead of +// GitHub's clock. A Date estimates that clock, and it can run ahead of GitHub's +// list. The clear is the Date of an earlier run. Each kind has a type here, +// every value of one is made in this file from the field that it comes from, +// and only firstRunBound leads from a Date to a readBound. So the compiler +// keeps a Date out of every other read bound. + +// attemptTime is a time that GitHub stamped on a delivery attempt: a +// delivered_at, the stored top, or an end of a range. GitHub's clock had +// passed it when the attempt was listed. An attempt that GitHub lists later +// is therefore not older, unless it ended late, so a read bound comes from +// this type. The zero value means that there is no attempt. +type attemptTime struct{ at time.Time } + +func topOf(scan RedeliveryScan) attemptTime { return attemptTime{scan.Top} } + +func deliveredAt(d HookDelivery) attemptTime { return attemptTime{d.DeliveredAt} } + +func (r RedeliveryRange) olderEnd() attemptTime { return attemptTime{r.OlderAt} } + +func (r RedeliveryRange) newerEnd() attemptTime { return attemptTime{r.NewerAt} } + +// newestOn is the newest attempt on a page. +func newestOn(page DeliveryPage) attemptTime { + var newest attemptTime + for _, d := range page.Deliveries { + newest = newest.orLater(deliveredAt(d)) + } + return newest +} + +func (a attemptTime) orLater(b attemptTime) attemptTime { + if b.at.After(a.at) { + return b + } + return a +} + +// orEarlier is the earlier of two attempts, where no attempt is later than +// any. +func (a attemptTime) orEarlier(b attemptTime) attemptTime { + if a.at.IsZero() || (!b.at.IsZero() && b.at.Before(a.at)) { + return b + } + return a +} + +// readBound is where a read stops: the read takes the attempts at or after +// it. One comes from an attempt time, or from firstRunBound. +type readBound struct{ stop time.Time } + +// bound is the read bound below a position. The overlap is for an attempt +// that GitHub lists below the position because it ended late. +func (a attemptTime) bound() readBound { return readBound{a.at.Add(-RedeliveryReadOverlap)} } + +func (b readBound) excludes(a attemptTime) bool { return a.at.Before(b.stop) } + +// rangeEnd is the old end of a range that a first run leaves unread. A first +// run has no position below it, so the range ends at the bound, and its read +// goes the overlap further, into the gap of the first run. +func (b readBound) rangeEnd() attemptTime { return attemptTime{b.stop} } + +// pageDate is the Date header of a page that passed checkDate. It estimates +// GitHub's clock when GitHub answered. It can run ahead of GitHub's list, so +// it gives the end of a gap, the floor of a range that closes unread and the +// next clear, and never a read bound. +type pageDate struct{ date time.Time } + +var errUnusableDate = xerrors.New("GitHub's Date header is unusable") + +// checkDate makes the pageDate of a page, and it is the only way to one. top +// is the newest attempt that the scan knows. A Date further behind an attempt +// than RedeliveryDateSlack would end a gap below attempts that GitHub has +// deleted, so it fails the read, and a page without a Date does too. +func checkDate(page DeliveryPage, top attemptTime) (pageDate, error) { + known := top.orLater(newestOn(page)) + switch { + case page.At.IsZero(): + return pageDate{}, xerrors.Errorf("%w: the page has none", errUnusableDate) + case page.At.Before(known.at.Add(-RedeliveryDateSlack)): + return pageDate{}, xerrors.Errorf("%w: it is %s before the newest attempt, and at most %s is allowed", + errUnusableDate, known.at.Sub(page.At), RedeliveryDateSlack) + } + return pageDate{page.At}, nil +} + +// gapEnd is where the gap of a read that reached the end of the list ends. +// The Date is at most the slack behind the clock that deletes attempts, and +// it dropped the fraction of its second. So nothing later than the Date plus +// both, less the retention, has expired. covered is the oldest attempt that +// the read saw, and the read missed nothing above it. +func (d pageDate) gapEnd(covered attemptTime) time.Time { + end := d.date.Add(redeliveryDateResolution - GitHubDeliveryRetention + RedeliveryDateSlack) + if !covered.at.IsZero() && covered.at.Before(end) { + end = covered.at + } + return end +} + +// expiredBelow is the time below which GitHub has deleted every attempt, by +// this Date and by the stored clear. A range whose new end is below it closes +// unread. The lower of the two decides, so one Date that runs far ahead +// closes nothing, and without a stored clear nothing closes. +func (d pageDate) expiredBelow(anchor clearAnchor) (time.Time, bool) { + if !anchor.set() { + return time.Time{}, false + } + return earlierOf(d.date, anchor.headDate).Add(-GitHubDeliveryRetention), true +} + +// clearAnchor is the stored clear: the Date of the last head read, or zero +// before one. A run saw the top of the list then, so an attempt that no run +// has seen is not older than this time, less the margin by which a Date runs +// ahead and less the overlap. It gives gap starts and the bound of a first +// run. +type clearAnchor struct{ headDate time.Time } + +func clearOf(scan RedeliveryScan) clearAnchor { return clearAnchor{scan.Clear} } + +func (c clearAnchor) set() bool { return !c.headDate.IsZero() } + +func (c clearAnchor) unseenFrom() time.Time { + return c.headDate.Add(-RedeliveryDateMargin - RedeliveryReadOverlap) +} + +// headGapStart is where the gap of a head read that reached the end of the +// list starts: at the bound of the read, or where the last head read stopped +// seeing, whichever is later. It is the zero time when no run has read the +// hook, and then there is no gap. +func (c clearAnchor) headGapStart(top attemptTime) time.Time { + var start time.Time + if !top.at.IsZero() { + start = top.bound().stop + } + if c.set() && c.unseenFrom().After(start) { + start = c.unseenFrom() + } + return start +} + +// firstRunBound is the bound of a run that has no top: the lookback below the +// newest attempt, and lower when an earlier run or the migration left a +// clear, because tracking began then. It is the one bound that a Date takes +// part in. That is safe only because the gap of the first run covers +// everything below it. +func (c clearAnchor) firstRunBound(newest attemptTime) readBound { + stop := newest.at.Add(-RedeliveryStartLookback) + if c.set() { + stop = earlierOf(stop, c.headDate.Add(-RedeliveryDateMargin)) + } + return readBound{stop} +} + +// firstRunGapStart is where the gap below firstRunBound starts: where +// GitHub's list can start at the earliest, and lower when tracking began +// before that. +func (c clearAnchor) firstRunGapStart(newest attemptTime) time.Time { + start := newest.at.Add(-GitHubDeliveryRetention) + if c.set() { + start = earlierOf(start, c.unseenFrom()) + } + return start +} diff --git a/faucet/redeliverystore.go b/faucet/redeliverystore.go index e31a0be..47abe72 100644 --- a/faucet/redeliverystore.go +++ b/faucet/redeliverystore.go @@ -76,9 +76,10 @@ type RedeliveryGap struct { type RedeliveryScan struct { // Epoch fences the run's writes. A later lease has a higher epoch. Epoch int64 - // Top is the time of the newest attempt that an earlier run read, or - // GitHub's clock when it read an empty list, or zero before the first - // run. It is always a time that GitHub gave, never one of faucet's clock. + // Top is the time of the newest attempt that an earlier run read, or zero + // when no run has read one. It is the only stored time that a read bound + // comes from, so it is always an attempt time: a run that reads an empty + // list leaves it as it is. Top time.Time // Clear is the Date header that GitHub sent with the first page of the // last run whose head read worked, or zero before one. It says when a run diff --git a/verify/tests/redeliver.e2e.ts b/verify/tests/redeliver.e2e.ts index d9ba331..7c84bb8 100644 --- a/verify/tests/redeliver.e2e.ts +++ b/verify/tests/redeliver.e2e.ts @@ -150,7 +150,8 @@ describe('redelivery', () => { await withHook(cron, async (started, output) => { const recent = newRepoId(); const before = newRepoId(); - await hooks.arm(started, [failed(recent, newDeliveryId(), 20 * HOUR), failed(before, newDeliveryId(), 30 * HOUR)]); + // The lookback is 24 hours below the newest attempt, so the second delivery is 30 hours older than the first. + await hooks.arm(started, [failed(recent, newDeliveryId(), 20 * HOUR), failed(before, newDeliveryId(), 50 * HOUR)]); const first = await sweep(started); const second = await sweep(started); From 52f022b6e12a7afd3f9918aaa500f14217944770 Mon Sep 17 00:00:00 2001 From: Thomas Kosiewski Date: Mon, 16 Nov 2026 11:32:45 +0000 Subject: [PATCH 04/18] docs(faucet): describe the redelivery clocks, the two residual risks and the migration The README still described main: a position that is GitHub's Date after an empty list, a first reply of `gaps: 1` for a new database, and two limits that came from faucet's clock and from a Date that leads. - README, "Redeliver failed webhooks". A new paragraph says what the position is and what a run takes from the Date. The `gap` entry says where a gap ends and when a quiet hook logs one (64h55m). The limits are now the two risks that remain outside the window in which redelivery trusts a Date (more than 6h5m ahead before an outage of 72 hours, more than 1h behind on a quiet hook), each with its condition first. The burst limit stays and is tracked separately. - README, "Upgrading". The migration costs one re-read of the last 24 hours and one more `before tracking began` gap, and a `RedeliveryStore` of your own must keep the clear. The rollback paragraph names migration `000008_redelivery_clear`. - `faucet/readme_test.go` ties the new numbers to the constants. - AGENTS.md: the "known exception on main" is gone. The line now says what holds: no faucet clock in the scan, read bounds from attempt times by type, and M and S as the trust window. - The verify skill's redelivery page says the same about the position and the first reply. Refs #26 --- .../verify-faucet/features/redeliver.md | 7 +++-- AGENTS.md | 2 +- README.md | 23 ++++++++------- faucet/readme_test.go | 29 +++++++++++++++++++ faucet/redeliver_gaps_test.go | 1 - 5 files changed, 47 insertions(+), 15 deletions(-) diff --git a/.agents/skills/verify-faucet/features/redeliver.md b/.agents/skills/verify-faucet/features/redeliver.md index beef3f3..1eb140b 100644 --- a/.agents/skills/verify-faucet/features/redeliver.md +++ b/.agents/skills/verify-faucet/features/redeliver.md @@ -7,12 +7,12 @@ - The reply is 200 JSON `{"busy", "pages", "attempts", "tracked", "asked", "recovered", "gave_up": {"limit", "expired"}, "gaps", "pending", "ranges"}`: pages and attempts read, deliveries that became pending, requests GitHub accepted, deliveries recovered, deliveries given up by reason, gaps recorded, deliveries still pending and ranges of the list still unread. - A delivery qualifies when an attempt failed (any answer outside 200 to 299, or none), its event and action have a rule in `Normalize`, and faucet has no event with its GUID. A later attempt that succeeded, or a stored event, recovers it with no further request. - A delivery is asked for at most 7 times, at 0, 10, 30, 70, 150, 310 and 630 minutes after the run that first read it, and at most 50 asks go out per run, the delivery that has waited longest first. The ask count is raised in the statement that claims the delivery, before the request. After the last ask and 5 minutes the delivery is given up with `reason=limit`. GitHub keeps a delivery for 72 hours, counted from its first attempt, and a delivery still pending then is given up with `reason=expired`. -- A run reads the newest attempts first, down to the newest attempt of the run before it less 5 minutes (the overlap that catches an attempt GitHub wrote late), and at most 20 pages. A stretch it cannot read is a range: its cursor and its two times are rows of `faucet_redelivery_ranges`, and later runs read the ranges, newest first, with the pages that the head read leaves. A range whose old end leaves GitHub's retention unread becomes a gap, and so does the time before the first run's lookback of 24 hours (`cause` `expired unread` and `before tracking began`). +- A run reads the newest attempts first, down to the newest attempt of the run before it less 5 minutes (the overlap that catches an attempt GitHub wrote late), and at most 20 pages. A stretch it cannot read is a range: its cursor and its two times are rows of `faucet_redelivery_ranges`, and later runs read the ranges, newest first, with the pages that the head read leaves. A range whose old end leaves GitHub's retention unread becomes a gap, and so does the time before the first run's lookback of 24 hours below the newest attempt (`cause` `expired unread` and `before tracking began`). - One run holds a lease (`faucet_redelivery_scan`) at a time. A second run that starts meanwhile answers `busy: true` and does nothing. - A page that fails to list stops the asks of the run, and so does a request that GitHub refuses (500 with the counts of the run and `"error": "redelivery failed"`, so the outcomes recorded before the failure still reach a reply. No response or log holds the hook token or GitHub's body). A refused ask still counts, and a rate limit is told from a denial in the log and handled the same way. The settle step needs no GitHub, so an expiry is still given up. - A run keeps to the 25 second deadline of the route: half for reading the list (a page not back by then is dropped, and the rest is an open range), then the settle, asks until 80 percent, and the purge. A slow list costs pages, never the asks. - A pending delivery belongs to the webhook that held the lease when it was tracked. After the webhook is replaced, a run does not ask the new one for the old one's deliveries: they end `recovered` or `expired`. -- The position of a run is GitHub's time: the newest attempt, or the `Date` header when the list is empty, never faucet's clock. +- The position of a run is the time of the newest attempt that it read. An empty list leaves it. No read stops at faucet's clock or at GitHub's `Date` header. The `Date` says where a gap ends (72 hours before it, plus 1 hour of slack), and the `Date` of the last run says where a gap after a silence starts (6 hours and 5 minutes before it). A first page whose `Date` is missing, or more than 1 hour behind the newest attempt, fails the run: 500, and no asks. - A final delivery and a recorded gap are deleted 72 hours after their last attempt, so each give-up is logged once. Pending deliveries, open ranges and the scan row stay. - Without the hook settings the route answers 503 with `redelivery is not configured`, after the secret check. - Only the hook token goes to GitHub, and only to two endpoints: `GET .../hooks/{id}/deliveries` and `POST .../deliveries/{attempt}/attempts`. @@ -35,7 +35,8 @@ No charter. The stub's control calls are the only way to put deliveries in place ## Gotchas -- The first run records the part of GitHub's 72 hours before its 24 hour lookback as one gap, so a first reply says `gaps: 1`. That is the design, not a fault. +- The first run that reads an attempt records the part of GitHub's 72 hours before its 24 hour lookback as one gap, so its reply says `gaps: 1`. Both are counted from the newest attempt, not from the time of the run. A run on an empty list says `gaps: 0`. That is the design, not a fault. +- The stub sends Node's `Date` header, which is the stub's clock. A test of a `Date` that leads, lags or is missing belongs in `faucet/redeliver_gaps_test.go`, where the fake GitHub sets it. - A delivery that is given up stays lost until someone redelivers it by hand. - A stored cursor is GitHub's opaque value. If GitHub keeps refusing it, the range is retried by every run until it expires, and then it is a gap. - GitHub lists an attempt when it ends, so an attempt that ended more than 5 minutes late and sits below the position of the last run is not read. diff --git a/AGENTS.md b/AGENTS.md index c23435a..587894a 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -7,5 +7,5 @@ Faucet is a Go library with a thin server (`cmd/faucet`). Read `README.md` first - The end-to-end checks live in `verify/` and are described by the `verify-faucet` skill in `.agents/skills/`. - Redelivery selection keeps persisted state; no stateless sliding-window sweep (see #10). - Redelivery never derives a read position or bound from faucet's clock: the two clocks differ, and a position ahead of GitHub's drops the next failure unseen. It uses times that GitHub gave (attempt times, the `Date` header). A run also keeps to its context deadline by construction: the scan has its own time budget, so the asks and the purge always run. -- Known exception on main: the retention floor and first-run lookback (see README, redelivery limits); a redesign needs a design review first. +- The scan reads no clock of faucet's (see #26). A read bound comes from an attempt time, never from a `Date`; the types in `faucet/redelivertime.go` enforce it. A gap ends at a page `Date` that passed the check (not more than S = 1h behind the newest attempt known), and a gap after a silence starts at the stored `Date` less M = 6h. M and S are the trust window (see README, redelivery limits); a change to them needs a design review first. - No selection shortcut that skips, suppresses or merges an overlap read or an open range (see #22). diff --git a/README.md b/README.md index 931b451..decc6e2 100644 --- a/README.md +++ b/README.md @@ -139,28 +139,30 @@ GitHub does not send a webhook again when faucet was down or answered with an er The token, the ID and one of org or repo go together. With none set, `/cron/redeliver` answers 503. With some set, the server refuses to start and names the missing variables, never a value. -**What a run does.** A run reads GitHub's deliveries, newest first, down to where the run before it stopped, less a 5 minute overlap for attempts that GitHub wrote late. A stretch that the run cannot read inside its page limit is kept as a range with GitHub's cursor, and later runs read the ranges, newest first. A delivery qualifies when an attempt failed (any answer outside 200 to 299, or none), its event and action are ones that faucet stores, and faucet has no event with its GUID. A run asks for it, and a later run sees the event stored, or a later successful attempt, and counts it recovered. A stored delivery, or one that succeeded later, is never asked for. Two runs never overlap: one holds a lease, and a second answers `busy: true`. The position of a run is a time that GitHub gave: the newest attempt it read, or, when the list is empty, the `Date` header of GitHub's answer. It is never a time of faucet's clock, so a clock that runs ahead of GitHub's cannot put a new failure below the position. +**What a run does.** A run reads GitHub's deliveries, newest first, down to where the run before it stopped, less a 5 minute overlap for attempts that GitHub wrote late. A stretch that the run cannot read inside its page limit is kept as a range with GitHub's cursor, and later runs read the ranges, newest first. A delivery qualifies when an attempt failed (any answer outside 200 to 299, or none), its event and action are ones that faucet stores, and faucet has no event with its GUID. A run asks for it, and a later run sees the event stored, or a later successful attempt, and counts it recovered. A stored delivery, or one that succeeded later, is never asked for. Two runs never overlap: one holds a lease, and a second answers `busy: true`. + +**Position and `Date`.** The position of a run is the time of the newest attempt that it read, and an empty list leaves the position where it was. A read never stops at a time of faucet's clock, and never at GitHub's `Date` header. Faucet's clock and GitHub's differ, and a `Date` can run ahead of GitHub's list, so either one could put a new failure below the position. A run uses the `Date` for two other things. The `Date` of a page that ends the list says where GitHub's 72 hours have ended, and a gap ends there, plus a slack of 1 hour for a `Date` that is behind. The `Date` of the last run says when a run last saw the top of the list, and after a silence a gap starts there, less a margin of 6 hours for a `Date` that is ahead and less the 5 minute overlap. A run checks a `Date` where it uses it: on its first page, and on a page that ends the list. A `Date` that is missing, or more than the slack behind the newest attempt that the run knows, fails that read. On the first page the run then fails and asks for nothing. On a later page the range stays open, and it ends as a gap when GitHub has deleted it. The tests in `faucet/redeliver_gaps_test.go` and `faucet/redeliver_sim_test.go` run these cases with a GitHub whose clock is up to 25 hours away from faucet's, whose `Date` leads, lags or is missing, and whose pages are slow. **Final states.** Every failed delivery ends once, in one of these, and the run that ends it counts it in the reply and logs it. A run that fails still answers with its counts (see the reply below), so each outcome that it recorded reaches you: - `recovered`: faucet stored the event, or a later attempt succeeded. - `gave_up` with `limit`: every ask was spent. A delivery is asked for 7 times, at 0, 10, 30, 70, 150, 310 and 630 minutes after the run that first read it. The count is raised before the request is sent, so a crash or a refused request costs an ask, and two runs cannot send the same one. - `gave_up` with `expired`: GitHub keeps a delivery for 72 hours, counted from its first attempt (the earliest attempt that a run read), and the delivery was still not stored. A delivery of a webhook that you replaced ends here too: an attempt ID of one webhook means nothing to another, so a run asks only for the deliveries of the webhook it was configured with. Those deliveries stay pending until faucet stores the event or 72 hours pass. -- `gap`: a time range of attempts that no run read. It names no delivery, because the attempts were never read. A range whose old end left GitHub's retention before a run read it is a gap, together with the 5 minute overlap below that end, where an attempt that GitHub wrote late is listed. So is the part of GitHub's 72 hours before the first run's lookback of 24 hours, and so are the open ranges of a webhook that the operator replaced. +- `gap`: a time range of attempts that no run read. It names no delivery, because the attempts were never read. A range whose old end left GitHub's retention before a run read it is a gap, together with the 5 minute overlap below that end, where an attempt that GitHub wrote late is listed. So is the part of GitHub's 72 hours before the first run's lookback of 24 hours, both counted from the newest attempt that the first run reads, and so are the open ranges of a webhook that the operator replaced. A gap never reaches above the oldest attempt that a run read above it. Below that attempt it can reach up to the slack past the end of GitHub's retention, so a gap can hold up to an hour in which GitHub deleted nothing. After a silence a gap starts 6 hours and 5 minutes before the `Date` of the run before it, so a hook that delivers nothing logs a gap only when two runs are about 64 hours and 55 minutes apart, or more. -The reply is JSON: `busy`, `pages`, `attempts`, `tracked`, `asked`, `recovered`, `gave_up` (`limit` and `expired`), `gaps`, `pending` and `ranges`. A run that fails answers 500 with the same JSON and an `error` member, so the outcomes that it recorded before it failed are in a reply as well as in the log. The first reply of a new database says `gaps: 1`, for the time before its lookback. Faucet deletes a final delivery and a recorded gap 72 hours after its last attempt, so a give-up is logged once. A pending delivery and an open range stay until they end, and the scan position is one row that stays. +The reply is JSON: `busy`, `pages`, `attempts`, `tracked`, `asked`, `recovered`, `gave_up` (`limit` and `expired`), `gaps`, `pending` and `ranges`. A run that fails answers 500 with the same JSON and an `error` member, so the outcomes that it recorded before it failed are in a reply as well as in the log. A run that finds the list empty records no gap. The first run that reads an attempt says `gaps: 1`, for the time before its lookback. Faucet deletes a final delivery and a recorded gap 72 hours after its last attempt, so a give-up is logged once. A pending delivery and an open range stay until they end, and the scan position is one row that stays. -**Capacity.** A run reads at most 20 pages of 100 attempts, which is 2,000 attempts, and sends at most 50 requests. A run every 10 minutes reads 15 minutes of the list: 10 minutes of new attempts and the 5 minute overlap. It also needs one attempt beyond them, to learn that it has reached the end of that window, and it starts from the newest attempt of the run before it, so a run holds at most 1,998 attempts of its window. That is 7,992 attempts an hour at best, and the sweep keeps up with 7,900 attempts an hour, rounded down for a run that starts late, and sends at most 300 requests an hour. These numbers hold only when GitHub answers each page of the list within 0.6 seconds, because a run reads the list for half of its 25 seconds (see Time). A slower API reads fewer pages: at 1.3 seconds a page a run reads 9 pages, and the sweep keeps up with about 3,500 attempts an hour. Faucet has no measurement of GitHub's page time, so size a hook with that margin. The list counts every attempt of the hook: attempts of events that faucet does not store, and the redeliveries that faucet asked for. A hook that records more than that does not lose anything silently: the part of the list that a run could not read stays a range. Later runs read the ranges, newest first, with the pages that the newest attempts leave them, and a range that no run read before its old end left GitHub's retention is counted and logged as a gap, except for the clock-lag slice in Limits. A failure of a request, GitHub's rate limit or a slow API makes a run read fewer pages. The simulations in `faucet/redeliver_sim_test.go` run 4,200 attempts an hour with one failure in 100, and outages of up to an hour at 150, 300 and 600 attempts an hour, and no failure is lost. They also check the law above at a tenth of the scale: the sweep holds at its stated capacity, and ranges grow when the load is a tenth above it. An outage of faucet that ends within 630 minutes of the first run that reads a failure is recovered by one of its asks. +**Capacity.** A run reads at most 20 pages of 100 attempts, which is 2,000 attempts, and sends at most 50 requests. A run every 10 minutes reads 15 minutes of the list: 10 minutes of new attempts and the 5 minute overlap. It also needs one attempt beyond them, to learn that it has reached the end of that window, and it starts from the newest attempt of the run before it, so a run holds at most 1,998 attempts of its window. That is 7,992 attempts an hour at best, and the sweep keeps up with 7,900 attempts an hour, rounded down for a run that starts late, and sends at most 300 requests an hour. These numbers hold only when GitHub answers each page of the list within 0.6 seconds, because a run reads the list for half of its 25 seconds (see Time). A slower API reads fewer pages: at 1.3 seconds a page a run reads 9 pages, and the sweep keeps up with about 3,500 attempts an hour. Faucet has no measurement of GitHub's page time, so size a hook with that margin. The list counts every attempt of the hook: attempts of events that faucet does not store, and the redeliveries that faucet asked for. A hook that records more than that does not lose anything silently: the part of the list that a run could not read stays a range. Later runs read the ranges, newest first, with the pages that the newest attempts leave them, and a range that no run read before its old end left GitHub's retention is counted and logged as a gap. A failure of a request, GitHub's rate limit or a slow API makes a run read fewer pages. The simulations in `faucet/redeliver_sim_test.go` run 4,200 attempts an hour with one failure in 100, and outages of up to an hour at 150, 300 and 600 attempts an hour, and no failure is lost. They also check the law above at a tenth of the scale: the sweep holds at its stated capacity, and ranges grow when the load is a tenth above it. An outage of faucet that ends within 630 minutes of the first run that reads a failure is recovered by one of its asks. **Time.** The cron route stops a run after 25 seconds. A run that has such a deadline spends it so that the asks and the purge always get their turn: it reads the list for half of the time, drops a page that is not back by then and leaves the rest as an open range (as at the page limit), settles, asks until 80 percent of the time has passed, and purges. A slow GitHub costs pages and never the asks. A first page that is not back in half of the time ends the run with an error, and it asks for nothing. -**Limits.** Three limits of redelivery are known. Each one can end a failure without a recovery, and each one is counted or bounded as said. Tracking issue: https://github.com/coder/faucet/issues/26. +**Limits.** GitHub gives no upper bound on the clock that deletes its attempts, other than the `Date` header. So redelivery trusts a `Date` inside a window: up to 6 hours ahead of GitHub's list (the margin), and up to 1 hour behind the clock that deletes (the slack). Outside that window two risks remain, and each can end a failure with no final state. A third limit ends failures inside counted gaps. Tracking issue: https://github.com/coder/faucet/issues/26. -- The retention floor and the first run's lookback come from faucet's clock. When faucet's clock lags GitHub's by D, and part of the list stays unread until GitHub deletes it, the failures in a slice as wide as D end with no final state, next to the gap that the run logs. This happens after a cron outage of 72 hours or more, and also when a range stays unread that long while cron runs (for example GitHub failing the range's pages, or an overload that lasts until the range's old end leaves GitHub's retention). GitHub has already deleted those deliveries. When faucet's clock runs ahead by D, a slice of that width at the old end of the first run's list is neither read nor counted. -- A `Date` header that runs ahead of GitHub's own list moves the position of a run that reads an empty list past attempts that GitHub has not listed yet. A failure that arrives in the next stretch is not read, and the stretch is as long as the `Date` is ahead, less the 5 minute overlap. A `HookAdmin` that sends no `Date` records a gap on every empty run. `GitHubHookAdmin` always gets a `Date` from GitHub. -- A dense burst followed by silence leaves ranges unread. When a burst holds more attempts in the 5 minute overlap than a run has pages, every run reads the same newest pages again and opens one more range, and the older ranges get no page. Their failures are not recovered. A range that no run read before its old end left GitHub's retention is counted and logged as a gap, so each failure in it ends inside a gap, except for the clock-lag slice of the first limit. +- When a `Date` runs more than 6 hours and 5 minutes ahead of GitHub's list (the margin and the overlap), and the cron then stops for 72 hours or more, the gap after the outage starts later than the last run before the outage. The stretch between the two is as long as the `Date` was ahead, less 6 hours and 5 minutes. The failures that GitHub listed in that stretch and then deleted are neither read nor counted. A position that a build before migration `000008_redelivery_clear` took from such a `Date` carries the same risk through the migration. +- When a `Date` is more than 1 hour behind the clock that deletes GitHub's attempts, and the hook is so quiet that no listed attempt shows the lag, a gap after a read to the end of the list ends too early. The failures that GitHub deleted in the slice above the gap are not counted. On a hook with recent attempts the run sees the lag and fails instead. +- A dense burst followed by silence leaves ranges unread. When a burst holds more attempts in the 5 minute overlap than a run has pages, every run reads the same newest pages again and opens one more range, and the older ranges get no page. Their failures are not recovered. A range that no run read before its old end left GitHub's retention is counted and logged as a gap, so each failure in it ends inside a gap. This limit does not come from a clock, and it is tracked separately from the two above. -To embed faucet, set `Deps.Redelivery` to a `RedeliveryDeps`: a `RedeliveryStore` (`pgstore.Store` is one, and `storetest.RunRedelivery` checks yours), and a `HookAdmin`. `NewGitHubHookAdmin` is the one for GitHub's REST API. Call `Service.Redeliver(ctx)` on your own schedule. It has no auth. The host decides who may call it. +To embed faucet, set `Deps.Redelivery` to a `RedeliveryDeps`: a `RedeliveryStore` (`pgstore.Store` is one, and `storetest.RunRedelivery` checks yours), and a `HookAdmin`. `NewGitHubHookAdmin` is the one for GitHub's REST API. A `HookAdmin` of your own must put GitHub's `Date` header in `DeliveryPage.At`, because a run whose first page has no `Date` fails. Call `Service.Redeliver(ctx)` on your own schedule. It has no auth. The host decides who may call it. ## Deploy on Vercel @@ -187,7 +189,7 @@ To embed faucet, set `Deps.Redelivery` to a `RedeliveryDeps`: a `RedeliveryStore The server applies pending migrations when a production instance starts, over `DATABASE_URL_UNPOOLED` (or over `DATABASE_URL` when that is unset), before it serves a request. -**Do not roll back past the newest migration.** After production has migrated to `000007_gap_pending`, a build that does not embed it, which is what Vercel Instant Rollback to an older deployment gives, cannot start: the migration library fails with `no migration found for version 7` and every new instance exits, so the rollback serves nothing. Recover in one of two ways. Redeploy a build that embeds the migration. Or roll the database back with the down migration `faucet/pgstore/migrations/000007_gap_pending.down.sql`, applied through golang-migrate against the `faucet_schema_migrations` table so that the recorded version drops to 6. The down migration drops the `gap_through` and `gap_lease_until` columns, and with them every gap that is owed and not yet sent. The same holds for each later migration. +**Do not roll back past the newest migration.** After production has migrated to `000008_redelivery_clear`, a build that does not embed it, which is what Vercel Instant Rollback to an older deployment gives, cannot start: the migration library fails with `no migration found for version 8` and every new instance exits, so the rollback serves nothing. Recover in one of two ways. Redeploy a build that embeds the migration. Or roll the database back with the down migration `faucet/pgstore/migrations/000008_redelivery_clear.down.sql`, applied through golang-migrate against the `faucet_schema_migrations` table so that the recorded version drops to 7. The down migration drops the `clear_at` column. A redelivery scan that has read no attempt since the migration gets its old position back. The same holds for each later migration, and for the one before: the down migration of `000007_gap_pending` drops the `gap_through` and `gap_lease_until` columns, and with them every gap that is owed and not yet sent. ## Upgrading @@ -196,6 +198,7 @@ The server applies pending migrations when a production instance starts, over `D - **Rows sealed before secret rotation existed.** The first refresh of such a row with the same secret counts as one rotation, because the new sealing of the secret is not the old bytes. For up to 10 minutes the old sealing is the previous secret. It opens to the same secret, so a signature has one `v1,` entry. - **A `CRON_SECRET` that no request can carry.** The server now stops at start, and names the variable, when `CRON_SECRET` has a space, a tab, a newline or another control character, a byte outside printable ASCII, or more than 1024 bytes. Before, it started and answered 401 to every cron call. A value from `openssl rand -hex 32` or `openssl rand -base64 32` passes, and so does any other run of printable ASCII characters. A secret that you read from a file can end in a newline that you must remove. - **A `Store` of your own.** The interface has gained methods, and `storetest` checks each one: `DeleteLiveSubscription`, `ClearExpiredPreviousSecrets`, a `Purge` that also moves the subscriptions that lost nothing past the purge mark, `ClaimGap` and `FinishGap`. `ClaimDelivery` takes a `skip` list of callback origins. `Enqueue` has `GapThrough`, `DeliveryRef` and `GapRef` have `Incarnation`, `ScanTarget` has `Principal` and no callback or secret fields, and `PurgeResult` has two counts. A `Store` without them does not compile. A `Store` that compiles can still fail `storetest`. `ClaimDelivery` and `ClaimGap` must skip a callback by `faucet.CallbackOrigin` and by no other rule of your own, so a stored URL in any spelling (a host in capitals, a scheme in capitals, user info, an IPv6 literal) is held to the cap of its origin. `storetest` checks this since PR 27. A `RedeliveryStore` is a second interface, `faucet.RedeliveryStore`, with its own suite, `storetest.RunRedelivery`. +- **The redelivery position after migration `000008_redelivery_clear`.** Before this migration the position of a run could be GitHub's `Date` header, and no row says whether it is. The migration moves the old position into the new `clear_at` column and resets the position. The first run after the migration is therefore a first run. It knows when tracking began, so it loses no failure silently. The cost comes once. The runs after the migration read the last 24 hours of the list again, and the first of them logs one more `before tracking began` gap for the time before those 24 hours. A delivery that a run tracked before the migration keeps its row and its ask count, so reading its attempts again changes nothing. On a hook near the capacity, the sweep cannot read that day again before GitHub deletes it, and the unread part ends as `expired unread` gaps. A `RedeliveryStore` of your own must keep `Clear` (`RedeliveryScan.Clear` and `ScanStep.Clear`), reset it with the top when the hook changes, and start the gaps of a changed hook the 5 minute overlap below each range's old end. `storetest.RunRedelivery` checks all three. ## Tests diff --git a/faucet/readme_test.go b/faucet/readme_test.go index 6ef3c23..f71d6f8 100644 --- a/faucet/readme_test.go +++ b/faucet/readme_test.go @@ -60,6 +60,35 @@ func TestReadmeStatesTheRedeliveryCapacityThatTheConstantsGive(t *testing.T) { require.Greater(t, keepsUpWith, 4200, "the capacity must cover the busiest hook that the simulations model") } +// The README also states how far redelivery trusts GitHub's Date header, and +// what follows from that for a quiet hook and for the two residual risks. +func TestReadmeStatesTheDateWindowThatTheConstantsGive(t *testing.T) { + t.Parallel() + + slack, margin := faucet.RedeliveryDateSlack, faucet.RedeliveryDateMargin + marginAndOverlap := hoursAndMinutes(margin + faucet.RedeliveryReadOverlap) + quietFor := hoursAndMinutes(faucet.GitHubDeliveryRetention - slack - margin - faucet.RedeliveryReadOverlap) + + readme, err := os.ReadFile("../README.md") + require.NoError(t, err) + text := string(readme) + + for _, want := range []string{ + fmt.Sprintf("a slack of %d hour", int(slack/time.Hour)), + fmt.Sprintf("a margin of %d hour", int(margin/time.Hour)), + fmt.Sprintf("starts %s before the `Date` of the run before it", marginAndOverlap), + fmt.Sprintf("two runs are about %s apart", quietFor), + fmt.Sprintf("more than %s ahead of GitHub's list", marginAndOverlap), + fmt.Sprintf("more than %d hour behind the clock that deletes", int(slack/time.Hour)), + } { + require.Contains(t, text, want) + } +} + +func hoursAndMinutes(d time.Duration) string { + return fmt.Sprintf("%d hours and %d minutes", int(d/time.Hour), int(d%time.Hour/time.Minute)) +} + func thousands(n int) string { return fmt.Sprintf("%d,%03d", n/1000, n%1000) } diff --git a/faucet/redeliver_gaps_test.go b/faucet/redeliver_gaps_test.go index 9d5c2f5..1bc98ee 100644 --- a/faucet/redeliver_gaps_test.go +++ b/faucet/redeliver_gaps_test.go @@ -978,7 +978,6 @@ func TestAQuietHookLogsAGapOnlyAfterAnOutageThatTheDateMarginsDoNotCover(t *test // After a silence a gap is [clear - margin - overlap, Date - retention + // slack]. It is empty until the runs are this far apart. threshold := faucet.GitHubDeliveryRetention - faucet.RedeliveryDateSlack - faucet.RedeliveryDateMargin - faucet.RedeliveryReadOverlap - require.Equal(t, 64*time.Hour+55*time.Minute, threshold) for _, listed := range []string{"an empty hook", "a hook whose last attempt is 10 hours old"} { for _, tc := range []struct { apart time.Duration From 97cdda6b1f06ab20a74fb70801280551b51bf172 Mon Sep 17 00:00:00 2001 From: Thomas Kosiewski Date: Mon, 16 Nov 2026 11:48:12 +0000 Subject: [PATCH 05/18] style(faucet): drop two comments that restate the redelivery scan `classify` and `newestOn` say what they do in their names. Two other comments in `redelivertime.go` say the same in plainer words, and the one on `firstRunBound` now gives both reasons why a clear may take part in that bound. Refs #26 --- faucet/redeliver.go | 1 - faucet/redelivertime.go | 9 ++++----- 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/faucet/redeliver.go b/faucet/redeliver.go index 5eb76e9..010bde4 100644 --- a/faucet/redeliver.go +++ b/faucet/redeliver.go @@ -325,7 +325,6 @@ func (r *redeliveryRun) fetch(ctx context.Context, cursor string) (DeliveryPage, return page, nil } -// classify sorts the attempts of a page that are at or after bound. func (r *redeliveryRun) classify(page DeliveryPage, bound readBound) pageRead { p := pageRead{next: page.Next} for _, d := range page.Deliveries { diff --git a/faucet/redelivertime.go b/faucet/redelivertime.go index 5242b5b..2b8bfa1 100644 --- a/faucet/redelivertime.go +++ b/faucet/redelivertime.go @@ -43,7 +43,6 @@ func (r RedeliveryRange) olderEnd() attemptTime { return attemptTime{r.OlderAt} func (r RedeliveryRange) newerEnd() attemptTime { return attemptTime{r.NewerAt} } -// newestOn is the newest attempt on a page. func newestOn(page DeliveryPage) attemptTime { var newest attemptTime for _, d := range page.Deliveries { @@ -59,8 +58,8 @@ func (a attemptTime) orLater(b attemptTime) attemptTime { return a } -// orEarlier is the earlier of two attempts, where no attempt is later than -// any. +// orEarlier is the earlier of two attempts. No attempt counts as later than +// any attempt. func (a attemptTime) orEarlier(b attemptTime) attemptTime { if a.at.IsZero() || (!b.at.IsZero() && b.at.Before(a.at)) { return b @@ -164,8 +163,8 @@ func (c clearAnchor) headGapStart(top attemptTime) time.Time { // firstRunBound is the bound of a run that has no top: the lookback below the // newest attempt, and lower when an earlier run or the migration left a // clear, because tracking began then. It is the one bound that a Date takes -// part in. That is safe only because the gap of the first run covers -// everything below it. +// part in. That is safe because the clear can only lower the bound, and the +// gap of the first run covers everything below it. func (c clearAnchor) firstRunBound(newest attemptTime) readBound { stop := newest.at.Add(-RedeliveryStartLookback) if c.set() { From 29508db9effba17ed1712d0ca38a34235591ba09 Mon Sep 17 00:00:00 2001 From: Thomas Kosiewski Date: Mon, 16 Nov 2026 11:54:05 +0000 Subject: [PATCH 06/18] style(faucet): cut the redelivery clock comments to what the code cannot say Delete comments that narrate unexported helpers, restate test names or explain test arithmetic. Keep exported API docs, the HTTP Date resolution note and GitHub's deletion order. The migration's reason becomes a pointer to #26. --- cmd/faucet/cron_internal_test.go | 2 - faucet/fakeredelivery_test.go | 5 -- faucet/hookadmin.go | 12 +--- .../000008_redelivery_clear.down.sql | 3 - .../migrations/000008_redelivery_clear.up.sql | 7 +-- faucet/pgstore/pgstore_test.go | 4 -- faucet/readme_test.go | 2 - faucet/redeliver.go | 34 ++---------- faucet/redeliver_gaps_test.go | 39 ------------- faucet/redeliver_sim_test.go | 15 +---- faucet/redeliver_test.go | 3 - faucet/redeliverharness_test.go | 14 +---- faucet/redelivertime.go | 55 ------------------- verify/tests/redeliver.e2e.ts | 1 - 14 files changed, 12 insertions(+), 184 deletions(-) diff --git a/cmd/faucet/cron_internal_test.go b/cmd/faucet/cron_internal_test.go index 4ff1168..f409ee8 100644 --- a/cmd/faucet/cron_internal_test.go +++ b/cmd/faucet/cron_internal_test.go @@ -88,8 +88,6 @@ type cronHooks struct { func (*cronHooks) Hook() string { return "/orgs/octo-org/hooks/9" } -// ListDeliveries lists one attempt of an event that faucet does not store, -// with GitHub's Date, so that a run finds a list to start its tracking at. func (h *cronHooks) ListDeliveries(context.Context, string) (faucet.DeliveryPage, error) { h.mu.Lock() defer h.mu.Unlock() diff --git a/faucet/fakeredelivery_test.go b/faucet/fakeredelivery_test.go index 0851d99..4c4fe24 100644 --- a/faucet/fakeredelivery_test.go +++ b/faucet/fakeredelivery_test.go @@ -323,17 +323,12 @@ func TestFakeRedeliveryStoreMatchesContract(t *testing.T) { storetest.RunRedelivery(t, func(*testing.T) storetest.RedeliveryStore { return newFakeRedeliveryStore(nil) }) } -// migrate does what migration 000008 does to the row of a build before it: -// the top moves to the clear, and the top is reset. func (s *fakeRedeliveryStore) migrate() { s.rmu.Lock() defer s.rmu.Unlock() s.clearAt, s.top = s.top, time.Time{} } -// seedMigrated puts a new store where the migration puts a build whose top -// was oldTop. Such a build took its top from the newest attempt, or from -// GitHub's Date when the list was empty. func (s *fakeRedeliveryStore) seedMigrated(oldTop time.Time) { s.rmu.Lock() s.top = oldTop diff --git a/faucet/hookadmin.go b/faucet/hookadmin.go index 30f47ea..da239ef 100644 --- a/faucet/hookadmin.go +++ b/faucet/hookadmin.go @@ -47,13 +47,9 @@ type DeliveryPage struct { // Next is the cursor of the next, older page. It is empty on the last page. Next string // At is GitHub's own clock when it answered, which GitHub's Date header - // carries. It is zero when GitHub sent none. Redeliver takes from it where - // GitHub's retention has ended and when a run last saw the top of the - // list. It never takes a read position from it: a Date can run ahead of - // GitHub's list, and a position ahead of the list would make the next - // failed attempt look as if an earlier run had read it. Redeliver needs At - // on the first page and on a page that ends the list. There it fails the - // read when At is zero, or more than RedeliveryDateSlack before the newest + // carries. It is zero when GitHub sent none. Redeliver needs At on the + // first page and on a page that ends the list. There it fails the read + // when At is zero, or more than RedeliveryDateSlack before the newest // attempt that it knows. At time.Time } @@ -195,8 +191,6 @@ func (a GitHubHookAdmin) ListDeliveries(ctx context.Context, cursor string) (Del if err != nil { return DeliveryPage{}, err } - // A missing or malformed Date leaves At zero and does not fail the page: - // most pages of a range need no Date, and Redeliver knows which do. at, _ := http.ParseTime(resp.Header.Get("Date")) page := DeliveryPage{Deliveries: make([]HookDelivery, len(raw)), Next: next, At: at} for i, d := range raw { diff --git a/faucet/pgstore/migrations/000008_redelivery_clear.down.sql b/faucet/pgstore/migrations/000008_redelivery_clear.down.sql index d367156..23ee8d0 100644 --- a/faucet/pgstore/migrations/000008_redelivery_clear.down.sql +++ b/faucet/pgstore/migrations/000008_redelivery_clear.down.sql @@ -1,6 +1,3 @@ --- A build before this migration keeps its position in top_at alone, and after --- an empty list that position was GitHub's Date. A scan that has read no --- attempt since the migration gets that position back. UPDATE faucet_redelivery_scan SET top_at = COALESCE(top_at, clear_at); ALTER TABLE faucet_redelivery_scan DROP COLUMN clear_at; diff --git a/faucet/pgstore/migrations/000008_redelivery_clear.up.sql b/faucet/pgstore/migrations/000008_redelivery_clear.up.sql index a208e19..936a1df 100644 --- a/faucet/pgstore/migrations/000008_redelivery_clear.up.sql +++ b/faucet/pgstore/migrations/000008_redelivery_clear.up.sql @@ -4,9 +4,6 @@ -- GitHub's list. top_at keeps attempt times only. ALTER TABLE faucet_redelivery_scan ADD COLUMN clear_at timestamptz; --- Until now top_at was GitHub's Date when a run read an empty list, and no --- row says which of the two it holds. So the old top moves to clear_at, and --- the next run starts as a first run that knows when tracking began: it reads --- the last 24 hours of the list again, and back to the old top less a margin --- when that is older, and it records what lies below as a gap. +-- Until now top_at could hold a Date that ran ahead of GitHub's list, and no +-- row says whether it does (see #26). UPDATE faucet_redelivery_scan SET clear_at = top_at, top_at = NULL; diff --git a/faucet/pgstore/pgstore_test.go b/faucet/pgstore/pgstore_test.go index 8b30461..f00dbe3 100644 --- a/faucet/pgstore/pgstore_test.go +++ b/faucet/pgstore/pgstore_test.go @@ -122,10 +122,6 @@ func TestMigrateLeavesHostMigrationsTableAlone(t *testing.T) { require.EqualValues(t, 8, faucetVersion) } -// A build before the clear migration kept its position in top_at alone, and -// after an empty list that position was GitHub's Date. The migration cannot -// tell the two apart, so it makes the next run a first run that knows when -// tracking began. The down migration gives such a build its position back. func TestTheClearMigrationMovesTheOldTopToTheClearAndBack(t *testing.T) { t.Parallel() db := newDatabase(t, testDatabaseURL(t)) diff --git a/faucet/readme_test.go b/faucet/readme_test.go index f71d6f8..cddb1e5 100644 --- a/faucet/readme_test.go +++ b/faucet/readme_test.go @@ -60,8 +60,6 @@ func TestReadmeStatesTheRedeliveryCapacityThatTheConstantsGive(t *testing.T) { require.Greater(t, keepsUpWith, 4200, "the capacity must cover the busiest hook that the simulations model") } -// The README also states how far redelivery trusts GitHub's Date header, and -// what follows from that for a quiet hook and for the two residual risks. func TestReadmeStatesTheDateWindowThatTheConstantsGive(t *testing.T) { t.Parallel() diff --git a/faucet/redeliver.go b/faucet/redeliver.go index 010bde4..b6c7f33 100644 --- a/faucet/redeliver.go +++ b/faucet/redeliver.go @@ -293,10 +293,9 @@ func (r *redeliveryRun) execute(ctx context.Context, scan RedeliveryScan) (Redel // pageRead is what one page of the delivery list tells a run. type pageRead struct { - failed []FailedAttempt - succeeded []string - attempts int - // oldest is the oldest attempt on the page at or after the bound. + failed []FailedAttempt + succeeded []string + attempts int oldest attemptTime reachedBound bool next string @@ -306,11 +305,8 @@ type pageRead struct { // it: the page reached the range's bound, or GitHub's list ended. func (p pageRead) finished() bool { return p.reachedBound || p.next == "" } -// endOfList says that GitHub's list ended above the bound. What the read did -// not see between the two has expired, or was never there. func (p pageRead) endOfList() bool { return !p.reachedBound && p.next == "" } -// fetch reads one page of the list inside the time that the scan has. func (r *redeliveryRun) fetch(ctx context.Context, cursor string) (DeliveryPage, error) { readCtx, cancel := until(ctx, r.budget.scanEnd) defer cancel() @@ -347,15 +343,10 @@ func (r *redeliveryRun) classify(page DeliveryPage, bound readBound) pageRead { return p } -// step starts the ScanStep of a page. Its time is faucet's clock, which the -// store keeps for the settle, the asks and the purge. No bound and no gap -// comes from it. func (r *redeliveryRun) step(p pageRead) ScanStep { return ScanStep{Epoch: r.epoch, Now: r.svc.clock.Now(), Failed: p.failed, Succeeded: p.succeeded} } -// gapBetween is the gap from one time to another. There is none without a -// start, and none when the start is not before the end. func gapBetween(from, to time.Time) *RedeliveryGap { if from.IsZero() || !from.Before(to) { return nil @@ -398,9 +389,6 @@ func (r *redeliveryRun) countGap(ctx context.Context, gap RedeliveryGap, cause s // scan reads the head of the list and then the open ranges, newest first. It // returns whether the head read worked. -// -// The head read is one page. Its Date must be usable, because it becomes the -// stored clear and decides which ranges have expired. func (r *redeliveryRun) scan(ctx context.Context, scan RedeliveryScan) (bool, error) { top, anchor := topOf(scan), clearOf(scan) page, err := r.fetch(ctx, "") @@ -425,9 +413,6 @@ func (r *redeliveryRun) scan(ctx context.Context, scan RedeliveryScan) (bool, er cause := gapExpiredUnread switch { case first && !newest.at.IsZero(): - // Tracking begins here, or it began at the clear. Below the bound lies - // the time before that. With a clear, a list that ends above the bound - // has also lost what expired since, and one gap holds both. to := bound.stop if anchor.set() && head.endOfList() { to = laterOf(to, date.gapEnd(head.oldest)) @@ -457,10 +442,6 @@ func (r *redeliveryRun) scan(ctx context.Context, scan RedeliveryScan) (bool, er return true, r.readRanges(ctx, live, top.orLater(newest)) } -// closeExpired closes, without a read, each range whose new end GitHub has -// deleted, and with it all of the range. It returns the others. A range whose -// page fails, because GitHub refuses its cursor or sends no usable Date, ends -// here. func (r *redeliveryRun) closeExpired(ctx context.Context, ranges []RedeliveryRange, date pageDate, anchor clearAnchor) ([]RedeliveryRange, error) { floor, ok := date.expiredBelow(anchor) if !ok { @@ -481,8 +462,6 @@ func (r *redeliveryRun) closeExpired(ctx context.Context, ranges []RedeliveryRan return live, nil } -// readRanges reads the ranges, newest first, with the pages that the run has -// left. top is the newest attempt that the run knows. func (r *redeliveryRun) readRanges(ctx context.Context, ranges []RedeliveryRange, top attemptTime) error { for _, rg := range ranges { if err := r.readRange(ctx, rg, top); err != nil { @@ -492,11 +471,8 @@ func (r *redeliveryRun) readRanges(ctx context.Context, ranges []RedeliveryRange return nil } -// readRange reads one range until it ends or the run has no page left. A page -// that reaches the bound ends it without a gap: GitHub deletes in the order -// of the attempt times, so nothing above an attempt that it still lists has -// expired. A page that ends the list ends the range with the gap that its -// Date gives, and the range stays open when that Date is not usable. +// GitHub deletes attempts in the order of their times, so a page that reaches +// the bound ends the range without a gap. func (r *redeliveryRun) readRange(ctx context.Context, rg RedeliveryRange, top attemptTime) error { bound := rg.olderEnd().bound() for r.res.Pages < r.state.maxPages && ctx.Err() == nil && !r.budget.pastScan() { diff --git a/faucet/redeliver_gaps_test.go b/faucet/redeliver_gaps_test.go index 1bc98ee..d89c4ad 100644 --- a/faucet/redeliver_gaps_test.go +++ b/faucet/redeliver_gaps_test.go @@ -330,12 +330,6 @@ func TestALateFailureEndsInAGapWhenGitHubDeletesItBeforeARunReadsTheRangeAboveIt } } -// The tests below give GitHub a clock of its own, a Date that leads or lags, -// pages that take time and cursors that it refuses. In each, a failure that -// GitHub listed must end tracked or inside a gap that a run logged. - -// requireListed fails unless GitHub's list holds the attempt now. A test calls -// it before the attempt expires, to show that a run could have read it. func requireListed(t *testing.T, env *redeliveryEnv, a ghAttempt) { t.Helper() require.True(t, env.gh.lists(a.guid), "GitHub must list %s, or the scenario proves nothing", a.guid) @@ -354,8 +348,6 @@ func TestAnExpiredFailureIsInsideTheGapWhenFaucetsClockLagsGitHubs(t *testing.T) env.gh.record("A", t0.Add(-time.Minute), http200) env.run() - // F leaves GitHub's list half a lag before the run after the outage. - // Faucet's clock says that GitHub still keeps it. f := env.gh.record("F", t0.Add(8*time.Hour-lag/2), 503) env.advance(8 * time.Hour) requireListed(t, env, f) @@ -377,8 +369,6 @@ func TestTheFirstRunsGapStartsBelowGitHubsListWhenFaucetsClockRunsAhead(t *testi env.clock.WithLogger(quartz.NoOpLogger) env.gh.skew = -ahead t0 := env.gh.now() - // F is the oldest attempt that GitHub lists. On faucet's clock its - // 72 hours are over. f := env.gh.record("F", t0.Add(-faucet.GitHubDeliveryRetention+ahead/2), 503) env.gh.record("A", t0.Add(-time.Minute), http200) requireListed(t, env, f) @@ -424,8 +414,6 @@ func TestAPageThatIsSlowToArriveLosesNoFailureThatGitHubDeletesMeanwhile(t *test env.gh.record("A", t0.Add(-time.Second), http200) env.run() - // F is the oldest attempt of a burst. The run after it has one page, - // which holds the newest 100, so F stays in the range below them. f := env.gh.record("F", t0.Add(time.Minute), 503) for i := range 150 { env.gh.record(fmt.Sprintf("b-%03d", i), t0.Add(2*time.Minute+time.Duration(i)*4*time.Second), http200) @@ -434,8 +422,6 @@ func TestAPageThatIsSlowToArriveLosesNoFailureThatGitHubDeletesMeanwhile(t *test require.Equal(t, 1, env.run().Ranges, "the burst leaves a range down to A") requireListed(t, env, f) - // GitHub lists F while the first page is on its way, and has deleted it - // when the page of the range arrives. env.advance(f.at.Add(faucet.GitHubDeliveryRetention - latency*3/2).Sub(env.gh.now())) env.useMaxPages(faucet.DefaultMaxPagesPerRun) env.slowPages(latency) @@ -528,7 +514,6 @@ func TestTheFirstRunClosesNoRangeOnTheWordOfOneDate(t *testing.T) { } } - // No clear is stored yet, so this Date is the only one that the run has. env.gh.date = dateOff(80 * time.Hour) first := env.run() require.Equal(t, 1, first.Ranges, "the range that the first run opens stays open") @@ -548,8 +533,6 @@ func TestTheFirstRunOfAHookWithoutAClearRecordsOnlyTheTimeBeforeItsLookback(t *t env := newRedeliveryEnv(t) env.clock.WithLogger(quartz.NoOpLogger) t0 := env.gh.now() - // The list is two days stale and ends above the bound of the first run. - // Nothing was tracked before this run, so nothing that expired counts. newest := env.gh.record("newer", t0.Add(-50*time.Hour), 503) env.gh.record("older", t0.Add(-60*time.Hour), 503) @@ -605,9 +588,6 @@ func TestTrackingThatBeganWithAnEmptyListOrTheMigrationHoldsThroughAnOutage(t *t func TestAfterTheMigrationATopThatWasADateAheadDropsNoFailure(t *testing.T) { t.Parallel() - // A build before the migration took its top from GitHub's Date when the - // list was empty. A Date that ran ahead then put the next failures below - // the read bound. for _, lead := range []time.Duration{time.Hour, 7 * time.Hour} { t.Run(fmt.Sprintf("the cron keeps running,lead=%s", lead), func(t *testing.T) { t.Parallel() @@ -627,9 +607,6 @@ func TestAfterTheMigrationATopThatWasADateAheadDropsNoFailure(t *testing.T) { } }) } - // With an outage after the migration, the failures expire unread. The gap - // covers them while the Date was ahead by no more than the margin and the - // overlap. for _, lead := range []time.Duration{time.Hour, 6 * time.Hour} { t.Run(fmt.Sprintf("the cron is down for 80 hours,lead=%s", lead), func(t *testing.T) { t.Parallel() @@ -699,8 +676,6 @@ func TestARangeIsReadThroughPagesThatCarryNoDate(t *testing.T) { env.advance(cronEvery) require.Equal(t, 1, env.run().Ranges) - // Only the first page of a run has a Date. Every page of a range either - // goes on or reaches the range's bound, and neither uses a Date. env.gh.date = func(_ int, cursor string, at time.Time) time.Time { if cursor != "" { return time.Time{} @@ -729,9 +704,6 @@ func TestARangeWhosePageFailsStaysOpenUntilGitHubHasDeletedItAndEndsAsAGap(t *te } return at }, - // The newest attempt that the run knows is a minute old, and it is on - // the first page of this run. The top that the store gave is 11 minutes - // old, and the attempts on the page of the range are days old. "the Date of the page that ends the list is more than the slack behind the newest attempt of the run": func(_ int, cursor string, at time.Time) time.Time { if cursor != "" { return at.Add(-faucet.RedeliveryDateSlack - 2*time.Minute) @@ -759,11 +731,6 @@ func TestARangeWhosePageFailsStaysOpenUntilGitHubHasDeletedItAndEndsAsAGap(t *te require.Equal(t, 1, env.run().Ranges, "the one page holds the newest 100, and the failures are in the range below") requireListed(t, env, failures[0]) - // A run reads one page, so the range stays unread while the hook - // goes on. Then the fault begins, and the runs get their pages back. - // A Date fault begins while GitHub still lists the failures. A cursor - // is stale only after GitHub has deleted its attempt, which is newer - // than they are. trickle := func() { env.advance(cronEvery) env.gh.record(fmt.Sprintf("t-%s", env.gh.now().Sub(t0)), env.gh.now().Add(-time.Minute), http200) @@ -813,8 +780,6 @@ func TestAWholeSecondDateAsFarBehindAsTheSlackStillEndsTheGapAboveWhatGitHubDele f := env.gh.record("F", t0.Add(time.Hour+400*time.Millisecond), 503) requireListed(t, env, f) - // The run starts 300 ms after GitHub deleted F. The Date drops the - // fraction of its second, which puts it before the deletion. env.advance(f.at.Add(faucet.GitHubDeliveryRetention + 300*time.Millisecond).Sub(env.gh.now())) require.False(t, env.gh.lists(f.guid)) env.run() @@ -901,8 +866,6 @@ func TestAChangedHookStartsWithoutTheOldHooksClear(t *testing.T) { env.run() env.advance(80 * time.Hour) - // The host replaces the webhook after an outage. The old hook's clear is - // 80 hours old, which on that hook is a gap. The new hook has lost nothing. env.gh.name = "/orgs/acme/hooks/8" res := env.run() @@ -975,8 +938,6 @@ func TestAQuietHookLogsNoGapInTenDays(t *testing.T) { func TestAQuietHookLogsAGapOnlyAfterAnOutageThatTheDateMarginsDoNotCover(t *testing.T) { t.Parallel() - // After a silence a gap is [clear - margin - overlap, Date - retention + - // slack]. It is empty until the runs are this far apart. threshold := faucet.GitHubDeliveryRetention - faucet.RedeliveryDateSlack - faucet.RedeliveryDateMargin - faucet.RedeliveryReadOverlap for _, listed := range []string{"an empty hook", "a hook whose last attempt is 10 hours old"} { for _, tc := range []struct { diff --git a/faucet/redeliver_sim_test.go b/faucet/redeliver_sim_test.go index de05080..b2390fe 100644 --- a/faucet/redeliver_sim_test.go +++ b/faucet/redeliver_sim_test.go @@ -24,10 +24,7 @@ type simPhase struct { } type simSpec struct { - // setup changes the env before the first delivery: GitHub's clock, the - // Date that it sends, the time that a page takes. - setup func(*redeliveryEnv) - // beforeRun runs before each run, with the time since the first run. + setup func(*redeliveryEnv) beforeRun func(env *redeliveryEnv, since time.Duration) phases []simPhase // outage is when faucet answers 503, relative to the first run. @@ -81,8 +78,6 @@ func runSim(t *testing.T, spec simSpec) simOutcome { if spec.setup != nil { spec.setup(env) } - // The deliveries, the outage and the cron follow GitHub's clock, which - // stamps the attempts. It is faucet's clock unless setup moved it. t0 := env.gh.now() down := func(at time.Time) bool { return spec.outageLength > 0 && !at.Before(t0.Add(spec.outageFrom)) && at.Before(t0.Add(spec.outageFrom+spec.outageLength)) @@ -349,8 +344,6 @@ func rangesOf(runs []faucet.RedeliverResult) []int { func TestSimADateThatLagsLessThanTheSlackFailsNoRunAndLosesNothing(t *testing.T) { t.Parallel() - // An HTTP Date is a whole second, and the attempt times have fractions, so - // even a Date that is right lies before the newest attempt on its page. for _, lag := range []time.Duration{0, 2 * time.Second, 30 * time.Minute, 59 * time.Minute} { t.Run(lag.String(), func(t *testing.T) { t.Parallel() @@ -373,8 +366,6 @@ func TestSimADateThatLagsLessThanTheSlackFailsNoRunAndLosesNothing(t *testing.T) func TestSimAnOutageLongerThanGitHubKeepsADeliveryLosesNothingSilentlyWhateverTheClocks(t *testing.T) { t.Parallel() - // skew is GitHub's clock minus faucet's. Faucet and its cron are down for - // the outage, and GitHub deletes the first failures before a run reads them. for _, skew := range []time.Duration{-25 * time.Hour, -time.Hour, 0, time.Hour, 25 * time.Hour} { for _, outage := range []time.Duration{71 * time.Hour, 80 * time.Hour} { t.Run(fmt.Sprintf("skew=%s,outage=%s", skew, outage), func(t *testing.T) { @@ -400,10 +391,6 @@ func TestSimAnOutageLongerThanGitHubKeepsADeliveryLosesNothingSilentlyWhateverTh func TestSimTheMigrationReadsTheLastDayAgainAndLosesNothingSilently(t *testing.T) { t.Parallel() - // The law of the capacity at a hundredth of the scale: pages of 1 attempt, - // 20 a run, so 42 and 70 an hour stand for 4,200 and 7,000. The store is - // migrated 6 hours in, while the hook delivers. The failures that count - // are the ones inside the lookback of the first run, and all later ones. const pages, pageSize = 20, 1 for _, tc := range []struct { diff --git a/faucet/redeliver_test.go b/faucet/redeliver_test.go index fdc97ff..6a01315 100644 --- a/faucet/redeliver_test.go +++ b/faucet/redeliver_test.go @@ -726,9 +726,6 @@ func TestEveryRedeliveryLogLineHoldsNoCursorOrURL(t *testing.T) { require.NotContains(t, text, "http") } -// The scan keeps the time of an attempt, GitHub's Date and the stored clear -// apart by type, so that a Date cannot become a read bound. That holds while -// redelivertime.go alone makes a value of those types. func TestOnlyRedeliverTimeMakesTheTimesThatTheScanKeepsApart(t *testing.T) { t.Parallel() made := regexp.MustCompile(`\b(attemptTime|readBound|pageDate|clearAnchor)\{`) diff --git a/faucet/redeliverharness_test.go b/faucet/redeliverharness_test.go index 3039ebb..2537dea 100644 --- a/faucet/redeliverharness_test.go +++ b/faucet/redeliverharness_test.go @@ -66,12 +66,7 @@ type fakeGitHub struct { // listDelay and askDelay make every list call and every redelivery request // take that long, or until its context ends. listDelay, askDelay time.Duration - // date gives the Date header of a list response (DeliveryPage.At), from the - // number of the list call, its cursor and GitHub's clock when it answered. - // A zero time is a response without a Date. Nil sends GitHub's clock. - date func(call int, cursor string, at time.Time) time.Time - // refusesStaleCursors makes GitHub answer a cursor whose attempt it has - // deleted with an error. By default it lists on from where the attempt was. + date func(call int, cursor string, at time.Time) time.Time refusesStaleCursors bool } @@ -214,7 +209,6 @@ func (g *fakeGitHub) ListDeliveries(ctx context.Context, cursor string) (faucet. return page, nil } -// lists says whether GitHub's list holds an attempt at guid now. func (g *fakeGitHub) lists(guid string) bool { g.mu.Lock() defer g.mu.Unlock() @@ -277,8 +271,6 @@ func (g *fakeGitHub) totalAsks() int { return n } -// The clock tests lean on what the fake GitHub does with a Date, a slow page -// and a stale cursor, so this pins each of them. func TestFakeGitHubSendsTheDateTakesTheTimeAndRefusesTheCursorThatATestSets(t *testing.T) { t.Parallel() ctx := context.Background() @@ -455,14 +447,10 @@ func (e *redeliveryEnv) advance(d time.Duration) { e.clock.Advance(d).MustWait(context.Background()) } -// slowPages makes every list request take d to reach GitHub: the clock moves -// forward by d inside the call, before GitHub reads its list. func (e *redeliveryEnv) slowPages(d time.Duration) { e.gh.listGate = func() { e.advance(d) } } -// dateOff is a Date that is d away from GitHub's clock: ahead of it when d is -// positive, behind it when d is negative. func dateOff(d time.Duration) func(int, string, time.Time) time.Time { return func(_ int, _ string, at time.Time) time.Time { return at.Add(d) } } diff --git a/faucet/redelivertime.go b/faucet/redelivertime.go index 2b8bfa1..16552cc 100644 --- a/faucet/redelivertime.go +++ b/faucet/redelivertime.go @@ -21,18 +21,6 @@ const ( redeliveryDateResolution = time.Second ) -// A scan works with three kinds of time. An attempt time is never ahead of -// GitHub's clock. A Date estimates that clock, and it can run ahead of GitHub's -// list. The clear is the Date of an earlier run. Each kind has a type here, -// every value of one is made in this file from the field that it comes from, -// and only firstRunBound leads from a Date to a readBound. So the compiler -// keeps a Date out of every other read bound. - -// attemptTime is a time that GitHub stamped on a delivery attempt: a -// delivered_at, the stored top, or an end of a range. GitHub's clock had -// passed it when the attempt was listed. An attempt that GitHub lists later -// is therefore not older, unless it ended late, so a read bound comes from -// this type. The zero value means that there is no attempt. type attemptTime struct{ at time.Time } func topOf(scan RedeliveryScan) attemptTime { return attemptTime{scan.Top} } @@ -58,8 +46,6 @@ func (a attemptTime) orLater(b attemptTime) attemptTime { return a } -// orEarlier is the earlier of two attempts. No attempt counts as later than -// any attempt. func (a attemptTime) orEarlier(b attemptTime) attemptTime { if a.at.IsZero() || (!b.at.IsZero() && b.at.Before(a.at)) { return b @@ -67,33 +53,18 @@ func (a attemptTime) orEarlier(b attemptTime) attemptTime { return a } -// readBound is where a read stops: the read takes the attempts at or after -// it. One comes from an attempt time, or from firstRunBound. type readBound struct{ stop time.Time } -// bound is the read bound below a position. The overlap is for an attempt -// that GitHub lists below the position because it ended late. func (a attemptTime) bound() readBound { return readBound{a.at.Add(-RedeliveryReadOverlap)} } func (b readBound) excludes(a attemptTime) bool { return a.at.Before(b.stop) } -// rangeEnd is the old end of a range that a first run leaves unread. A first -// run has no position below it, so the range ends at the bound, and its read -// goes the overlap further, into the gap of the first run. func (b readBound) rangeEnd() attemptTime { return attemptTime{b.stop} } -// pageDate is the Date header of a page that passed checkDate. It estimates -// GitHub's clock when GitHub answered. It can run ahead of GitHub's list, so -// it gives the end of a gap, the floor of a range that closes unread and the -// next clear, and never a read bound. type pageDate struct{ date time.Time } var errUnusableDate = xerrors.New("GitHub's Date header is unusable") -// checkDate makes the pageDate of a page, and it is the only way to one. top -// is the newest attempt that the scan knows. A Date further behind an attempt -// than RedeliveryDateSlack would end a gap below attempts that GitHub has -// deleted, so it fails the read, and a page without a Date does too. func checkDate(page DeliveryPage, top attemptTime) (pageDate, error) { known := top.orLater(newestOn(page)) switch { @@ -106,11 +77,6 @@ func checkDate(page DeliveryPage, top attemptTime) (pageDate, error) { return pageDate{page.At}, nil } -// gapEnd is where the gap of a read that reached the end of the list ends. -// The Date is at most the slack behind the clock that deletes attempts, and -// it dropped the fraction of its second. So nothing later than the Date plus -// both, less the retention, has expired. covered is the oldest attempt that -// the read saw, and the read missed nothing above it. func (d pageDate) gapEnd(covered attemptTime) time.Time { end := d.date.Add(redeliveryDateResolution - GitHubDeliveryRetention + RedeliveryDateSlack) if !covered.at.IsZero() && covered.at.Before(end) { @@ -119,10 +85,6 @@ func (d pageDate) gapEnd(covered attemptTime) time.Time { return end } -// expiredBelow is the time below which GitHub has deleted every attempt, by -// this Date and by the stored clear. A range whose new end is below it closes -// unread. The lower of the two decides, so one Date that runs far ahead -// closes nothing, and without a stored clear nothing closes. func (d pageDate) expiredBelow(anchor clearAnchor) (time.Time, bool) { if !anchor.set() { return time.Time{}, false @@ -130,11 +92,6 @@ func (d pageDate) expiredBelow(anchor clearAnchor) (time.Time, bool) { return earlierOf(d.date, anchor.headDate).Add(-GitHubDeliveryRetention), true } -// clearAnchor is the stored clear: the Date of the last head read, or zero -// before one. A run saw the top of the list then, so an attempt that no run -// has seen is not older than this time, less the margin by which a Date runs -// ahead and less the overlap. It gives gap starts and the bound of a first -// run. type clearAnchor struct{ headDate time.Time } func clearOf(scan RedeliveryScan) clearAnchor { return clearAnchor{scan.Clear} } @@ -145,10 +102,6 @@ func (c clearAnchor) unseenFrom() time.Time { return c.headDate.Add(-RedeliveryDateMargin - RedeliveryReadOverlap) } -// headGapStart is where the gap of a head read that reached the end of the -// list starts: at the bound of the read, or where the last head read stopped -// seeing, whichever is later. It is the zero time when no run has read the -// hook, and then there is no gap. func (c clearAnchor) headGapStart(top attemptTime) time.Time { var start time.Time if !top.at.IsZero() { @@ -160,11 +113,6 @@ func (c clearAnchor) headGapStart(top attemptTime) time.Time { return start } -// firstRunBound is the bound of a run that has no top: the lookback below the -// newest attempt, and lower when an earlier run or the migration left a -// clear, because tracking began then. It is the one bound that a Date takes -// part in. That is safe because the clear can only lower the bound, and the -// gap of the first run covers everything below it. func (c clearAnchor) firstRunBound(newest attemptTime) readBound { stop := newest.at.Add(-RedeliveryStartLookback) if c.set() { @@ -173,9 +121,6 @@ func (c clearAnchor) firstRunBound(newest attemptTime) readBound { return readBound{stop} } -// firstRunGapStart is where the gap below firstRunBound starts: where -// GitHub's list can start at the earliest, and lower when tracking began -// before that. func (c clearAnchor) firstRunGapStart(newest attemptTime) time.Time { start := newest.at.Add(-GitHubDeliveryRetention) if c.set() { diff --git a/verify/tests/redeliver.e2e.ts b/verify/tests/redeliver.e2e.ts index 7c84bb8..f72e0b7 100644 --- a/verify/tests/redeliver.e2e.ts +++ b/verify/tests/redeliver.e2e.ts @@ -150,7 +150,6 @@ describe('redelivery', () => { await withHook(cron, async (started, output) => { const recent = newRepoId(); const before = newRepoId(); - // The lookback is 24 hours below the newest attempt, so the second delivery is 30 hours older than the first. await hooks.arm(started, [failed(recent, newDeliveryId(), 20 * HOUR), failed(before, newDeliveryId(), 50 * HOUR)]); const first = await sweep(started); From 64a8a9b189bfa06a6bcac2a360c8c3496e057a43 Mon Sep 17 00:00:00 2001 From: Thomas Kosiewski Date: Mon, 16 Nov 2026 12:24:26 +0000 Subject: [PATCH 07/18] refactor(faucet): put what the redelivery comments said into names and types The comment cleanup left code that only its comments had explained. This changes no behavior. - A range end is no longer made into an attempt time. `RedeliveryRange.bound` gives the bound of a range, and `readBound.rangeEnd` and `RedeliveryRange.olderEnd` are gone. An `attemptTime` is now always a time that GitHub stamped. - `readBound.loweredTo` is the one way for a bare time into a read bound, and it can only lower the bound. `firstRunBound` uses it for the clear. - `clearAnchor` and its `headDate` are now `storedClear` and `clearAt`, so the clear has one name. - `expiredByBoth(date, clear)` replaces `pageDate.expiredBelow`. Its name says that a range closes unread only below what both Dates call expired. - `storedClear.firstRunGapEnd` holds the rule that a first run with a clear also counts what expired since tracking began. - A missing attempt and a missing gap start are explicit: `attemptTime.set`, an `orEarlier` that treats both sides alike, and a `headGapStart` that says whether there is a start. `gapBetween` no longer reads a zero time as no gap. - `checkDate`, `readRanges` and `readRange` call their parameter `newestKnown`, because it is the stored top raised by the head of this run. Refs #26 --- faucet/redeliver.go | 44 ++++++++++++----------- faucet/redeliver_test.go | 2 +- faucet/redelivertime.go | 77 ++++++++++++++++++++++++---------------- 3 files changed, 70 insertions(+), 53 deletions(-) diff --git a/faucet/redeliver.go b/faucet/redeliver.go index b6c7f33..7605db0 100644 --- a/faucet/redeliver.go +++ b/faucet/redeliver.go @@ -348,7 +348,7 @@ func (r *redeliveryRun) step(p pageRead) ScanStep { } func gapBetween(from, to time.Time) *RedeliveryGap { - if from.IsZero() || !from.Before(to) { + if !from.Before(to) { return nil } return &RedeliveryGap{From: from, To: to} @@ -390,7 +390,7 @@ func (r *redeliveryRun) countGap(ctx context.Context, gap RedeliveryGap, cause s // scan reads the head of the list and then the open ranges, newest first. It // returns whether the head read worked. func (r *redeliveryRun) scan(ctx context.Context, scan RedeliveryScan) (bool, error) { - top, anchor := topOf(scan), clearOf(scan) + top, clearAt := topOf(scan), clearOf(scan) page, err := r.fetch(ctx, "") if err != nil { return false, err @@ -401,28 +401,30 @@ func (r *redeliveryRun) scan(ctx context.Context, scan RedeliveryScan) (bool, er } newest := newestOn(page) - first := top.at.IsZero() - bound, olderAt := top.bound(), top + first := !top.set() + bound, olderAt := top.bound(), top.at if first { - bound = anchor.firstRunBound(newest) - olderAt = bound.rangeEnd() + bound = clearAt.firstRunBound(newest) + olderAt = bound.stop } head := r.classify(page, bound) step := r.step(head) step.Top, step.Clear = newest.at, date.date cause := gapExpiredUnread switch { - case first && !newest.at.IsZero(): + case first && newest.set(): to := bound.stop - if anchor.set() && head.endOfList() { - to = laterOf(to, date.gapEnd(head.oldest)) + if head.endOfList() { + to = clearAt.firstRunGapEnd(bound, date.gapEnd(head.oldest)) } - step.Gap, cause = gapBetween(anchor.firstRunGapStart(newest), to), gapBeforeTracking + step.Gap, cause = gapBetween(clearAt.firstRunGapStart(newest), to), gapBeforeTracking case head.endOfList(): - step.Gap = gapBetween(anchor.headGapStart(top), date.gapEnd(head.oldest)) + if from, ok := clearAt.headGapStart(top); ok { + step.Gap = gapBetween(from, date.gapEnd(head.oldest)) + } } if !head.finished() { - step.Open = &RedeliveryRange{NewerAt: head.oldest.at, OlderAt: olderAt.at, Cursor: head.next} + step.Open = &RedeliveryRange{NewerAt: head.oldest.at, OlderAt: olderAt, Cursor: head.next} } res, err := r.apply(ctx, step, cause) if err != nil { @@ -435,15 +437,15 @@ func (r *redeliveryRun) scan(ctx context.Context, scan RedeliveryScan) (bool, er r.res.Ranges++ ranges = append([]RedeliveryRange{opened}, ranges...) } - live, err := r.closeExpired(ctx, ranges, date, anchor) + live, err := r.closeExpired(ctx, ranges, date, clearAt) if err != nil { return true, err } return true, r.readRanges(ctx, live, top.orLater(newest)) } -func (r *redeliveryRun) closeExpired(ctx context.Context, ranges []RedeliveryRange, date pageDate, anchor clearAnchor) ([]RedeliveryRange, error) { - floor, ok := date.expiredBelow(anchor) +func (r *redeliveryRun) closeExpired(ctx context.Context, ranges []RedeliveryRange, date pageDate, clearAt storedClear) ([]RedeliveryRange, error) { + floor, ok := expiredByBoth(date, clearAt) if !ok { return ranges, nil } @@ -453,7 +455,7 @@ func (r *redeliveryRun) closeExpired(ctx context.Context, ranges []RedeliveryRan live = append(live, rg) continue } - gap := RedeliveryGap{From: rg.olderEnd().bound().stop, To: laterOf(rg.NewerAt, rg.OlderAt)} + gap := RedeliveryGap{From: rg.bound().stop, To: laterOf(rg.NewerAt, rg.OlderAt)} if _, err := r.apply(ctx, ScanStep{Epoch: r.epoch, Now: r.svc.clock.Now(), CloseID: rg.ID, Gap: &gap}, gapExpiredUnread); err != nil { return nil, err } @@ -462,9 +464,9 @@ func (r *redeliveryRun) closeExpired(ctx context.Context, ranges []RedeliveryRan return live, nil } -func (r *redeliveryRun) readRanges(ctx context.Context, ranges []RedeliveryRange, top attemptTime) error { +func (r *redeliveryRun) readRanges(ctx context.Context, ranges []RedeliveryRange, newestKnown attemptTime) error { for _, rg := range ranges { - if err := r.readRange(ctx, rg, top); err != nil { + if err := r.readRange(ctx, rg, newestKnown); err != nil { return err } } @@ -473,8 +475,8 @@ func (r *redeliveryRun) readRanges(ctx context.Context, ranges []RedeliveryRange // GitHub deletes attempts in the order of their times, so a page that reaches // the bound ends the range without a gap. -func (r *redeliveryRun) readRange(ctx context.Context, rg RedeliveryRange, top attemptTime) error { - bound := rg.olderEnd().bound() +func (r *redeliveryRun) readRange(ctx context.Context, rg RedeliveryRange, newestKnown attemptTime) error { + bound := rg.bound() for r.res.Pages < r.state.maxPages && ctx.Err() == nil && !r.budget.pastScan() { page, err := r.fetch(ctx, rg.Cursor) if errors.Is(err, errScanBudget) { @@ -490,7 +492,7 @@ func (r *redeliveryRun) readRange(ctx context.Context, rg RedeliveryRange, top a case p.reachedBound: step.CloseID = rg.ID case p.endOfList(): - date, err := checkDate(page, top) + date, err := checkDate(page, newestKnown) if err != nil { return xerrors.Errorf("range page: %w", err) } diff --git a/faucet/redeliver_test.go b/faucet/redeliver_test.go index 6a01315..c59e67f 100644 --- a/faucet/redeliver_test.go +++ b/faucet/redeliver_test.go @@ -728,7 +728,7 @@ func TestEveryRedeliveryLogLineHoldsNoCursorOrURL(t *testing.T) { func TestOnlyRedeliverTimeMakesTheTimesThatTheScanKeepsApart(t *testing.T) { t.Parallel() - made := regexp.MustCompile(`\b(attemptTime|readBound|pageDate|clearAnchor)\{`) + made := regexp.MustCompile(`\b(attemptTime|readBound|pageDate|storedClear)\{`) files, err := filepath.Glob("*.go") require.NoError(t, err) require.Contains(t, files, "redelivertime.go") diff --git a/faucet/redelivertime.go b/faucet/redelivertime.go index 16552cc..7d81930 100644 --- a/faucet/redelivertime.go +++ b/faucet/redelivertime.go @@ -27,8 +27,6 @@ func topOf(scan RedeliveryScan) attemptTime { return attemptTime{scan.Top} } func deliveredAt(d HookDelivery) attemptTime { return attemptTime{d.DeliveredAt} } -func (r RedeliveryRange) olderEnd() attemptTime { return attemptTime{r.OlderAt} } - func (r RedeliveryRange) newerEnd() attemptTime { return attemptTime{r.NewerAt} } func newestOn(page DeliveryPage) attemptTime { @@ -39,6 +37,8 @@ func newestOn(page DeliveryPage) attemptTime { return newest } +func (a attemptTime) set() bool { return !a.at.IsZero() } + func (a attemptTime) orLater(b attemptTime) attemptTime { if b.at.After(a.at) { return b @@ -47,7 +47,12 @@ func (a attemptTime) orLater(b attemptTime) attemptTime { } func (a attemptTime) orEarlier(b attemptTime) attemptTime { - if a.at.IsZero() || (!b.at.IsZero() && b.at.Before(a.at)) { + switch { + case !a.set(): + return b + case !b.set(): + return a + case b.at.Before(a.at): return b } return a @@ -57,16 +62,18 @@ type readBound struct{ stop time.Time } func (a attemptTime) bound() readBound { return readBound{a.at.Add(-RedeliveryReadOverlap)} } +func (r RedeliveryRange) bound() readBound { return readBound{r.OlderAt.Add(-RedeliveryReadOverlap)} } + func (b readBound) excludes(a attemptTime) bool { return a.at.Before(b.stop) } -func (b readBound) rangeEnd() attemptTime { return attemptTime{b.stop} } +func (b readBound) loweredTo(t time.Time) readBound { return readBound{earlierOf(b.stop, t)} } type pageDate struct{ date time.Time } var errUnusableDate = xerrors.New("GitHub's Date header is unusable") -func checkDate(page DeliveryPage, top attemptTime) (pageDate, error) { - known := top.orLater(newestOn(page)) +func checkDate(page DeliveryPage, newestKnown attemptTime) (pageDate, error) { + known := newestKnown.orLater(newestOn(page)) switch { case page.At.IsZero(): return pageDate{}, xerrors.Errorf("%w: the page has none", errUnusableDate) @@ -79,52 +86,60 @@ func checkDate(page DeliveryPage, top attemptTime) (pageDate, error) { func (d pageDate) gapEnd(covered attemptTime) time.Time { end := d.date.Add(redeliveryDateResolution - GitHubDeliveryRetention + RedeliveryDateSlack) - if !covered.at.IsZero() && covered.at.Before(end) { + if covered.set() && covered.at.Before(end) { end = covered.at } return end } -func (d pageDate) expiredBelow(anchor clearAnchor) (time.Time, bool) { - if !anchor.set() { - return time.Time{}, false - } - return earlierOf(d.date, anchor.headDate).Add(-GitHubDeliveryRetention), true -} +type storedClear struct{ clearAt time.Time } -type clearAnchor struct{ headDate time.Time } +func clearOf(scan RedeliveryScan) storedClear { return storedClear{scan.Clear} } -func clearOf(scan RedeliveryScan) clearAnchor { return clearAnchor{scan.Clear} } +func (c storedClear) set() bool { return !c.clearAt.IsZero() } -func (c clearAnchor) set() bool { return !c.headDate.IsZero() } +func expiredByBoth(d pageDate, c storedClear) (time.Time, bool) { + if !c.set() { + return time.Time{}, false + } + return earlierOf(d.date, c.clearAt).Add(-GitHubDeliveryRetention), true +} -func (c clearAnchor) unseenFrom() time.Time { - return c.headDate.Add(-RedeliveryDateMargin - RedeliveryReadOverlap) +func (c storedClear) unseenFrom() time.Time { + return c.clearAt.Add(-RedeliveryDateMargin - RedeliveryReadOverlap) } -func (c clearAnchor) headGapStart(top attemptTime) time.Time { - var start time.Time - if !top.at.IsZero() { - start = top.bound().stop - } - if c.set() && c.unseenFrom().After(start) { - start = c.unseenFrom() +func (c storedClear) headGapStart(top attemptTime) (time.Time, bool) { + switch { + case top.set() && c.set(): + return laterOf(top.bound().stop, c.unseenFrom()), true + case top.set(): + return top.bound().stop, true + case c.set(): + return c.unseenFrom(), true } - return start + return time.Time{}, false } -func (c clearAnchor) firstRunBound(newest attemptTime) readBound { - stop := newest.at.Add(-RedeliveryStartLookback) +func (c storedClear) firstRunBound(newest attemptTime) readBound { + bound := readBound{newest.at.Add(-RedeliveryStartLookback)} if c.set() { - stop = earlierOf(stop, c.headDate.Add(-RedeliveryDateMargin)) + bound = bound.loweredTo(c.clearAt.Add(-RedeliveryDateMargin)) } - return readBound{stop} + return bound } -func (c clearAnchor) firstRunGapStart(newest attemptTime) time.Time { +func (c storedClear) firstRunGapStart(newest attemptTime) time.Time { start := newest.at.Add(-GitHubDeliveryRetention) if c.set() { start = earlierOf(start, c.unseenFrom()) } return start } + +func (c storedClear) firstRunGapEnd(bound readBound, expiredEnd time.Time) time.Time { + if !c.set() { + return bound.stop + } + return laterOf(bound.stop, expiredEnd) +} From 0bba736350aed65c6d0242f643f3a9c472fa6ca9 Mon Sep 17 00:00:00 2001 From: Thomas Kosiewski Date: Mon, 16 Nov 2026 12:24:37 +0000 Subject: [PATCH 08/18] test(faucet): name the offsets of the redelivery clock tests and check their premises The numbers in these tests had their meaning in comments that are gone. Each one now has a name. Where a scenario depends on what GitHub lists, or on what faucet's clock says, the test asserts that before the run. - The migration simulation takes its two rates from `steadyCapacity`. At half the capacity the sweep reads the last day again and recovers every failure. At 95 percent of it the unread part ends in gaps. Before, 42 and 70 an hour stood for that. - The fake store's `migrate` is now `applyMigration000008`, the cron test names the attempt that starts tracking, and the e2e test writes the age of its second delivery from the lookback. - New test: a range whose last page is empty ends as a gap that stops at the oldest attempt read above it. The README promises that, and no test failed when `orEarlier` dropped the new end of the range. Refs #26 --- cmd/faucet/cron_internal_test.go | 9 ++-- faucet/fakeredelivery_test.go | 6 +-- faucet/redeliver_gaps_test.go | 82 ++++++++++++++++++++++++++------ faucet/redeliver_sim_test.go | 18 ++++--- verify/tests/redeliver.e2e.ts | 5 +- 5 files changed, 92 insertions(+), 28 deletions(-) diff --git a/cmd/faucet/cron_internal_test.go b/cmd/faucet/cron_internal_test.go index f409ee8..1ab6556 100644 --- a/cmd/faucet/cron_internal_test.go +++ b/cmd/faucet/cron_internal_test.go @@ -88,14 +88,17 @@ type cronHooks struct { func (*cronHooks) Hook() string { return "/orgs/octo-org/hooks/9" } +var attemptThatStartsTracking = faucet.HookDelivery{ + ID: 1, GUID: "g-listed", DeliveredAt: time.Date(2026, 11, 13, 12, 0, 0, 0, time.UTC), StatusCode: http.StatusOK, Event: "ping", +} + func (h *cronHooks) ListDeliveries(context.Context, string) (faucet.DeliveryPage, error) { h.mu.Lock() defer h.mu.Unlock() h.lists++ - listed := time.Date(2026, 11, 13, 12, 0, 0, 0, time.UTC) return faucet.DeliveryPage{ - Deliveries: []faucet.HookDelivery{{ID: 1, GUID: "g-listed", DeliveredAt: listed, StatusCode: http.StatusOK, Event: "ping"}}, - At: listed.Add(time.Second), + Deliveries: []faucet.HookDelivery{attemptThatStartsTracking}, + At: attemptThatStartsTracking.DeliveredAt.Add(time.Second), }, nil } diff --git a/faucet/fakeredelivery_test.go b/faucet/fakeredelivery_test.go index 4c4fe24..1a3919e 100644 --- a/faucet/fakeredelivery_test.go +++ b/faucet/fakeredelivery_test.go @@ -323,17 +323,17 @@ func TestFakeRedeliveryStoreMatchesContract(t *testing.T) { storetest.RunRedelivery(t, func(*testing.T) storetest.RedeliveryStore { return newFakeRedeliveryStore(nil) }) } -func (s *fakeRedeliveryStore) migrate() { +func (s *fakeRedeliveryStore) applyMigration000008() { s.rmu.Lock() defer s.rmu.Unlock() s.clearAt, s.top = s.top, time.Time{} } -func (s *fakeRedeliveryStore) seedMigrated(oldTop time.Time) { +func (s *fakeRedeliveryStore) seedMigratedTop(oldTop time.Time) { s.rmu.Lock() s.top = oldTop s.rmu.Unlock() - s.migrate() + s.applyMigration000008() } func (s *fakeRedeliveryStore) rowCount() int { diff --git a/faucet/redeliver_gaps_test.go b/faucet/redeliver_gaps_test.go index d89c4ad..4173c82 100644 --- a/faucet/redeliver_gaps_test.go +++ b/faucet/redeliver_gaps_test.go @@ -330,6 +330,11 @@ func TestALateFailureEndsInAGapWhenGitHubDeletesItBeforeARunReadsTheRangeAboveIt } } +const ( + newestAttemptAge = time.Minute + dateLagPastTheSlack = newestAttemptAge + faucet.RedeliveryDateSlack + time.Minute +) + func requireListed(t *testing.T, env *redeliveryEnv, a ghAttempt) { t.Helper() require.True(t, env.gh.lists(a.guid), "GitHub must list %s, or the scenario proves nothing", a.guid) @@ -348,10 +353,12 @@ func TestAnExpiredFailureIsInsideTheGapWhenFaucetsClockLagsGitHubs(t *testing.T) env.gh.record("A", t0.Add(-time.Minute), http200) env.run() - f := env.gh.record("F", t0.Add(8*time.Hour-lag/2), 503) + expiredFor := lag / 2 + f := env.gh.record("F", t0.Add(8*time.Hour-expiredFor), 503) env.advance(8 * time.Hour) requireListed(t, env, f) env.advance(faucet.GitHubDeliveryRetention) + require.True(t, f.at.After(env.clock.Now().Add(-faucet.GitHubDeliveryRetention)), "faucet's clock says that GitHub still keeps F") env.run() requireNoSilentLoss(t, env, []ghAttempt{f}) @@ -369,9 +376,11 @@ func TestTheFirstRunsGapStartsBelowGitHubsListWhenFaucetsClockRunsAhead(t *testi env.clock.WithLogger(quartz.NoOpLogger) env.gh.skew = -ahead t0 := env.gh.now() - f := env.gh.record("F", t0.Add(-faucet.GitHubDeliveryRetention+ahead/2), 503) + stillListedFor := ahead / 2 + f := env.gh.record("F", t0.Add(-faucet.GitHubDeliveryRetention+stillListedFor), 503) env.gh.record("A", t0.Add(-time.Minute), http200) requireListed(t, env, f) + require.False(t, f.at.After(env.clock.Now().Add(-faucet.GitHubDeliveryRetention)), "faucet's clock says that the 72 hours of F are over") env.run() @@ -422,10 +431,12 @@ func TestAPageThatIsSlowToArriveLosesNoFailureThatGitHubDeletesMeanwhile(t *test require.Equal(t, 1, env.run().Ranges, "the burst leaves a range down to A") requireListed(t, env, f) - env.advance(f.at.Add(faucet.GitHubDeliveryRetention - latency*3/2).Sub(env.gh.now())) + oneAndAHalfPages := latency * 3 / 2 + env.advance(f.at.Add(faucet.GitHubDeliveryRetention - oneAndAHalfPages).Sub(env.gh.now())) env.useMaxPages(faucet.DefaultMaxPagesPerRun) env.slowPages(latency) require.Zero(t, env.run().Ranges, "the range is read to the end of the list") + require.False(t, env.gh.lists(f.guid), "GitHub deleted F between the first page and the page of the range") requireNoSilentLoss(t, env, []ghAttempt{f}) }) @@ -559,7 +570,7 @@ func TestTrackingThatBeganWithAnEmptyListOrTheMigrationHoldsThroughAnOutage(t *t env.clock.WithLogger(quartz.NoOpLogger) t0 := env.gh.now() if began == "the migration" { - env.store.seedMigrated(t0) + env.store.seedMigratedTop(t0) } else { env.run() } @@ -593,7 +604,7 @@ func TestAfterTheMigrationATopThatWasADateAheadDropsNoFailure(t *testing.T) { t.Parallel() env := newRedeliveryEnv(t) env.clock.WithLogger(quartz.NoOpLogger) - env.store.seedMigrated(env.gh.now().Add(lead)) + env.store.seedMigratedTop(env.gh.now().Add(lead)) var failures []ghAttempt for i := range 12 { @@ -612,7 +623,7 @@ func TestAfterTheMigrationATopThatWasADateAheadDropsNoFailure(t *testing.T) { t.Parallel() env := newRedeliveryEnv(t) env.clock.WithLogger(quartz.NoOpLogger) - env.store.seedMigrated(env.gh.now().Add(lead)) + env.store.seedMigratedTop(env.gh.now().Add(lead)) var failures []ghAttempt for i := range 30 { @@ -706,7 +717,7 @@ func TestARangeWhosePageFailsStaysOpenUntilGitHubHasDeletedItAndEndsAsAGap(t *te }, "the Date of the page that ends the list is more than the slack behind the newest attempt of the run": func(_ int, cursor string, at time.Time) time.Time { if cursor != "" { - return at.Add(-faucet.RedeliveryDateSlack - 2*time.Minute) + return at.Add(-dateLagPastTheSlack) } return at }, @@ -718,6 +729,7 @@ func TestARangeWhosePageFailsStaysOpenUntilGitHubHasDeletedItAndEndsAsAGap(t *te env.clock.WithLogger(quartz.NoOpLogger) t0 := env.gh.now() var failures []ghAttempt + var newestOfTheBurst ghAttempt for i := range 150 { status := http200 if i < 50 && i%10 == 0 { @@ -727,23 +739,30 @@ func TestARangeWhosePageFailsStaysOpenUntilGitHubHasDeletedItAndEndsAsAGap(t *te if status != http200 { failures = append(failures, a) } + newestOfTheBurst = a } require.Equal(t, 1, env.run().Ranges, "the one page holds the newest 100, and the failures are in the range below") requireListed(t, env, failures[0]) trickle := func() { env.advance(cronEvery) - env.gh.record(fmt.Sprintf("t-%s", env.gh.now().Sub(t0)), env.gh.now().Add(-time.Minute), http200) + env.gh.record(fmt.Sprintf("t-%s", env.gh.now().Sub(t0)), env.gh.now().Add(-newestAttemptAge), http200) } - lastRunBeforeTheFault := t0.Add(faucet.GitHubDeliveryRetention - 30*time.Minute) + theBurstHasExpired := t0.Add(faucet.GitHubDeliveryRetention) + faultBegins := theBurstHasExpired.Add(-2 * cronEvery) if name == refusedCursor { - lastRunBeforeTheFault = t0.Add(faucet.GitHubDeliveryRetention - 10*time.Minute) + faultBegins = theBurstHasExpired } - for env.gh.now().Before(lastRunBeforeTheFault) { + for env.gh.now().Before(faultBegins.Add(-cronEvery)) { trickle() require.Equal(t, 1, env.run().Ranges) } trickle() + if name == refusedCursor { + require.False(t, env.gh.lists(newestOfTheBurst.guid), "GitHub refuses a cursor only when it has deleted the attempt there") + } else { + requireListed(t, env, failures[0]) + } env.gh.date, env.gh.refusesStaleCursors = date, name == refusedCursor env.useMaxPages(faucet.DefaultMaxPagesPerRun) @@ -763,6 +782,38 @@ func TestARangeWhosePageFailsStaysOpenUntilGitHubHasDeletedItAndEndsAsAGap(t *te } } +func TestARangeWhoseLastPageIsEmptyEndsAsAGapUpToTheOldestAttemptReadAboveIt(t *testing.T) { + t.Parallel() + env := newRedeliveryEnv(t, func(o *envOptions) { o.maxPages = 1 }) + env.clock.WithLogger(quartz.NoOpLogger) + t0 := env.gh.now() + env.gh.record("A", t0.Add(-time.Minute), http200) + env.run() + + l := env.gh.record("L", t0.Add(time.Hour), 503) + var onePage []ghAttempt + for i := range 100 { + onePage = append(onePage, env.gh.record(fmt.Sprintf("b-%03d", i), t0.Add(3*time.Hour+time.Duration(i)*4*time.Second), http200)) + } + env.advance(3*time.Hour + 15*time.Minute) + require.Equal(t, 1, env.run().Ranges, "the one page holds the burst, and L is in the range below it") + requireListed(t, env, l) + + oldestRead := onePage[0] + insideTheSlack := faucet.RedeliveryDateSlack / 2 + env.advance(oldestRead.at.Add(faucet.GitHubDeliveryRetention - insideTheSlack).Sub(env.gh.now())) + requireListed(t, env, oldestRead) + require.False(t, env.gh.lists(l.guid), "GitHub has deleted all of the range") + env.useMaxPages(faucet.DefaultMaxPagesPerRun) + res := env.run() + + require.Zero(t, res.Ranges) + requireNoSilentLoss(t, env, []ghAttempt{l}) + for _, gap := range env.loggedGaps() { + require.False(t, gap.to.After(oldestRead.at), "the gap %s reaches above the oldest attempt that a run read above it", gap) + } +} + func TestAWholeSecondDateAsFarBehindAsTheSlackStillEndsTheGapAboveWhatGitHubDeleted(t *testing.T) { t.Parallel() @@ -780,8 +831,11 @@ func TestAWholeSecondDateAsFarBehindAsTheSlackStillEndsTheGapAboveWhatGitHubDele f := env.gh.record("F", t0.Add(time.Hour+400*time.Millisecond), 503) requireListed(t, env, f) - env.advance(f.at.Add(faucet.GitHubDeliveryRetention + 300*time.Millisecond).Sub(env.gh.now())) + deleted := f.at.Add(faucet.GitHubDeliveryRetention) + laterInTheSameSecond := 300 * time.Millisecond + env.advance(deleted.Add(laterInTheSameSecond).Sub(env.gh.now())) require.False(t, env.gh.lists(f.guid)) + require.True(t, env.gh.now().Truncate(time.Second).Before(deleted), "a Date drops the fraction of its second, which puts it before the deletion of F") env.run() requireNoSilentLoss(t, env, []ghAttempt{f}) @@ -794,7 +848,7 @@ func TestAFirstPageWithoutAUsableDateFailsTheRunWhichAsksForNothing(t *testing.T for name, date := range map[string]func(int, string, time.Time) time.Time{ "no Date": func(int, string, time.Time) time.Time { return time.Time{} }, - "a Date more than the slack behind the newest attempt": dateOff(-faucet.RedeliveryDateSlack - 2*time.Minute), + "a Date more than the slack behind the newest attempt": dateOff(-dateLagPastTheSlack), } { t.Run(name, func(t *testing.T) { t.Parallel() @@ -804,7 +858,7 @@ func TestAFirstPageWithoutAUsableDateFailsTheRunWhichAsksForNothing(t *testing.T env.gh.record("due", env.gh.now().Add(-time.Minute), 400) require.Equal(t, 1, env.run().Asked) env.advance(cronEvery) - env.gh.record("new", env.gh.now().Add(-time.Minute), 503) + env.gh.record("new", env.gh.now().Add(-newestAttemptAge), 503) env.gh.date = date res, err := env.svc.Redeliver(context.Background()) diff --git a/faucet/redeliver_sim_test.go b/faucet/redeliver_sim_test.go index b2390fe..e6d697a 100644 --- a/faucet/redeliver_sim_test.go +++ b/faucet/redeliver_sim_test.go @@ -392,33 +392,37 @@ func TestSimAnOutageLongerThanGitHubKeepsADeliveryLosesNothingSilentlyWhateverTh func TestSimTheMigrationReadsTheLastDayAgainAndLosesNothingSilently(t *testing.T) { t.Parallel() const pages, pageSize = 20, 1 + const migratedAt = 6 * time.Hour + capacity := steadyCapacity(pages*pageSize, cronEvery+faucet.RedeliveryReadOverlap) + beforeTheFirstLookback := -faucet.RedeliveryStartLookback - 3*time.Hour + insideTheFirstLookback := -faucet.RedeliveryStartLookback + time.Hour for _, tc := range []struct { name string perHour int catchesUp bool }{ - {"below the capacity", 42, true}, - {"near the capacity", 70, false}, + {"at half the capacity", capacity / 2, true}, + {"near the capacity", capacity * 95 / 100, false}, } { t.Run(tc.name, func(t *testing.T) { t.Parallel() migrated := false out := runSim(t, simSpec{ - phases: []simPhase{{from: -27 * time.Hour, to: 90 * time.Hour, perHour: tc.perHour}}, + phases: []simPhase{{from: beforeTheFirstLookback, to: 90 * time.Hour, perHour: tc.perHour}}, blip: func(_, i int) bool { return i%25 == 7 }, beforeRun: func(env *redeliveryEnv, since time.Duration) { - if since == 6*time.Hour { - env.store.migrate() + if since == migratedAt { + env.store.applyMigration000008() migrated = true } }, - countFrom: -23 * time.Hour, + countFrom: insideTheFirstLookback, horizon: 90 * time.Hour, pages: pages, pageSize: pageSize, }) - t.Logf("RESULT %d an hour with pages of %d, migrated at 6h: %s, ranges after the last run %d", tc.perHour, pageSize, out, out.runs[len(out.runs)-1].Ranges) + t.Logf("RESULT %d an hour of a capacity of %d, migrated at %s: %s, ranges after the last run %d", tc.perHour, capacity, migratedAt, out, out.runs[len(out.runs)-1].Ranges) require.True(t, migrated) require.Greater(t, out.failed, 100) require.Zero(t, out.silentLost) diff --git a/verify/tests/redeliver.e2e.ts b/verify/tests/redeliver.e2e.ts index f72e0b7..d6f450a 100644 --- a/verify/tests/redeliver.e2e.ts +++ b/verify/tests/redeliver.e2e.ts @@ -5,6 +5,7 @@ const run = samples['workflow_run.completed']; const MINUTE = 60_000; const HOUR = 60 * MINUTE; const MAX_ASKS = 7; +const START_LOOKBACK = 24 * HOUR; const PAGES_PER_RUN = 20; const HOOK = '/orgs/verify/hooks/7'; @@ -150,7 +151,9 @@ describe('redelivery', () => { await withHook(cron, async (started, output) => { const recent = newRepoId(); const before = newRepoId(); - await hooks.arm(started, [failed(recent, newDeliveryId(), 20 * HOUR), failed(before, newDeliveryId(), 50 * HOUR)]); + const newestAge = 20 * HOUR; + const belowTheLookback = newestAge + START_LOOKBACK + 6 * HOUR; + await hooks.arm(started, [failed(recent, newDeliveryId(), newestAge), failed(before, newDeliveryId(), belowTheLookback)]); const first = await sweep(started); const second = await sweep(started); From da6ffd28b4ace02c1376bcee89c9803a4f97b99e Mon Sep 17 00:00:00 2001 From: Thomas Kosiewski Date: Mon, 16 Nov 2026 12:24:37 +0000 Subject: [PATCH 09/18] docs(faucet): say exactly what faucet's clock and a Date do in the scan "The scan reads no clock of faucet's" claimed too much. Each step still gives the store faucet's clock as its time of record (`ScanStep.Now`). What holds is that no read bound and no gap comes from that clock. "A read never stops at a Date" claimed too much as well. The first run that reads an attempt, after an empty list or the migration, reads down to the stored Date less the margin. What holds is that the position is never a Date, and that a Date can lower where a read stops and never raise it. The doc of `Redeliver`, the README, AGENTS.md, the feature note and the doc of `RedeliveryScan.Top` now say this. Refs #26 --- .../verify-faucet/features/redeliver.md | 2 +- AGENTS.md | 2 +- README.md | 2 +- faucet/redeliver.go | 24 ++++++++++--------- faucet/redeliverystore.go | 6 ++--- 5 files changed, 19 insertions(+), 17 deletions(-) diff --git a/.agents/skills/verify-faucet/features/redeliver.md b/.agents/skills/verify-faucet/features/redeliver.md index 1eb140b..f823629 100644 --- a/.agents/skills/verify-faucet/features/redeliver.md +++ b/.agents/skills/verify-faucet/features/redeliver.md @@ -12,7 +12,7 @@ - A page that fails to list stops the asks of the run, and so does a request that GitHub refuses (500 with the counts of the run and `"error": "redelivery failed"`, so the outcomes recorded before the failure still reach a reply. No response or log holds the hook token or GitHub's body). A refused ask still counts, and a rate limit is told from a denial in the log and handled the same way. The settle step needs no GitHub, so an expiry is still given up. - A run keeps to the 25 second deadline of the route: half for reading the list (a page not back by then is dropped, and the rest is an open range), then the settle, asks until 80 percent, and the purge. A slow list costs pages, never the asks. - A pending delivery belongs to the webhook that held the lease when it was tracked. After the webhook is replaced, a run does not ask the new one for the old one's deliveries: they end `recovered` or `expired`. -- The position of a run is the time of the newest attempt that it read. An empty list leaves it. No read stops at faucet's clock or at GitHub's `Date` header. The `Date` says where a gap ends (72 hours before it, plus 1 hour of slack), and the `Date` of the last run says where a gap after a silence starts (6 hours and 5 minutes before it). A first page whose `Date` is missing, or more than 1 hour behind the newest attempt, fails the run: 500, and no asks. +- The position of a run is the time of the newest attempt that it read. An empty list leaves it. The position never comes from faucet's clock or from GitHub's `Date` header, and a `Date` can only lower where a read stops. The `Date` says where a gap ends (72 hours before it, plus 1 hour of slack), and the `Date` of the last run says where a gap after a silence starts (6 hours and 5 minutes before it). A first page whose `Date` is missing, or more than 1 hour behind the newest attempt, fails the run: 500, and no asks. - A final delivery and a recorded gap are deleted 72 hours after their last attempt, so each give-up is logged once. Pending deliveries, open ranges and the scan row stay. - Without the hook settings the route answers 503 with `redelivery is not configured`, after the secret check. - Only the hook token goes to GitHub, and only to two endpoints: `GET .../hooks/{id}/deliveries` and `POST .../deliveries/{attempt}/attempts`. diff --git a/AGENTS.md b/AGENTS.md index 587894a..7c981d7 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -7,5 +7,5 @@ Faucet is a Go library with a thin server (`cmd/faucet`). Read `README.md` first - The end-to-end checks live in `verify/` and are described by the `verify-faucet` skill in `.agents/skills/`. - Redelivery selection keeps persisted state; no stateless sliding-window sweep (see #10). - Redelivery never derives a read position or bound from faucet's clock: the two clocks differ, and a position ahead of GitHub's drops the next failure unseen. It uses times that GitHub gave (attempt times, the `Date` header). A run also keeps to its context deadline by construction: the scan has its own time budget, so the asks and the purge always run. -- The scan reads no clock of faucet's (see #26). A read bound comes from an attempt time, never from a `Date`; the types in `faucet/redelivertime.go` enforce it. A gap ends at a page `Date` that passed the check (not more than S = 1h behind the newest attempt known), and a gap after a silence starts at the stored `Date` less M = 6h. M and S are the trust window (see README, redelivery limits); a change to them needs a design review first. +- The scan takes no read bound and no gap from faucet's clock (see #26). That clock is only the time of record that a step gives the store (`ScanStep.Now`). A read bound comes from an attempt time, and a `Date` can only lower it (the first run after an empty list or the migration); the types in `faucet/redelivertime.go` enforce it. A gap ends at a page `Date` that passed the check (not more than S = 1h behind the newest attempt known), and a gap after a silence starts at the stored `Date` less M = 6h. M and S are the trust window (see README, redelivery limits); a change to them needs a design review first. - No selection shortcut that skips, suppresses or merges an overlap read or an open range (see #22). diff --git a/README.md b/README.md index decc6e2..cbf2629 100644 --- a/README.md +++ b/README.md @@ -141,7 +141,7 @@ The token, the ID and one of org or repo go together. With none set, `/cron/rede **What a run does.** A run reads GitHub's deliveries, newest first, down to where the run before it stopped, less a 5 minute overlap for attempts that GitHub wrote late. A stretch that the run cannot read inside its page limit is kept as a range with GitHub's cursor, and later runs read the ranges, newest first. A delivery qualifies when an attempt failed (any answer outside 200 to 299, or none), its event and action are ones that faucet stores, and faucet has no event with its GUID. A run asks for it, and a later run sees the event stored, or a later successful attempt, and counts it recovered. A stored delivery, or one that succeeded later, is never asked for. Two runs never overlap: one holds a lease, and a second answers `busy: true`. -**Position and `Date`.** The position of a run is the time of the newest attempt that it read, and an empty list leaves the position where it was. A read never stops at a time of faucet's clock, and never at GitHub's `Date` header. Faucet's clock and GitHub's differ, and a `Date` can run ahead of GitHub's list, so either one could put a new failure below the position. A run uses the `Date` for two other things. The `Date` of a page that ends the list says where GitHub's 72 hours have ended, and a gap ends there, plus a slack of 1 hour for a `Date` that is behind. The `Date` of the last run says when a run last saw the top of the list, and after a silence a gap starts there, less a margin of 6 hours for a `Date` that is ahead and less the 5 minute overlap. A run checks a `Date` where it uses it: on its first page, and on a page that ends the list. A `Date` that is missing, or more than the slack behind the newest attempt that the run knows, fails that read. On the first page the run then fails and asks for nothing. On a later page the range stays open, and it ends as a gap when GitHub has deleted it. The tests in `faucet/redeliver_gaps_test.go` and `faucet/redeliver_sim_test.go` run these cases with a GitHub whose clock is up to 25 hours away from faucet's, whose `Date` leads, lags or is missing, and whose pages are slow. +**Position and `Date`.** The position of a run is the time of the newest attempt that it read, and an empty list leaves the position where it was. The position is never a time of faucet's clock, and never GitHub's `Date` header. Faucet's clock and GitHub's differ, and a `Date` can run ahead of GitHub's list, so either one could put a new failure below the position. A run uses the `Date` for two other things. The `Date` of a page that ends the list says where GitHub's 72 hours have ended, and a gap ends there, plus a slack of 1 hour for a `Date` that is behind. The `Date` of the last run says when a run last saw the top of the list, and after a silence a gap starts there, less a margin of 6 hours for a `Date` that is ahead and less the 5 minute overlap. The first run that reads an attempt, after runs that found the list empty or after migration `000008_redelivery_clear`, also reads down to that `Date` less the margin: a `Date` can lower where a read stops, and never raise it. A run checks a `Date` where it uses it: on its first page, and on a page that ends the list. A `Date` that is missing, or more than the slack behind the newest attempt that the run knows, fails that read. On the first page the run then fails and asks for nothing. On a later page the range stays open, and it ends as a gap when GitHub has deleted it. The tests in `faucet/redeliver_gaps_test.go` and `faucet/redeliver_sim_test.go` run these cases with a GitHub whose clock is up to 25 hours away from faucet's, whose `Date` leads, lags or is missing, and whose pages are slow. **Final states.** Every failed delivery ends once, in one of these, and the run that ends it counts it in the reply and logs it. A run that fails still answers with its counts (see the reply below), so each outcome that it recorded reaches you: diff --git a/faucet/redeliver.go b/faucet/redeliver.go index 7605db0..8fca467 100644 --- a/faucet/redeliver.go +++ b/faucet/redeliver.go @@ -201,17 +201,19 @@ type RedeliverResult struct { // GitHub's retention before it is read becomes a gap, together with the // RedeliveryReadOverlap below that end, where a late attempt is listed. // -// The scan reads no clock of faucet's, because that clock and GitHub's -// differ. Where a read stops comes from the time of an attempt, which GitHub's -// clock had passed when it listed the attempt. A first run is the exception: -// after an empty list or the migration it also reads back to the Date of -// that time, less RedeliveryDateMargin, and its gap covers what lies below. -// Where GitHub's retention has ended comes from the Date header of the page -// that saw the end of the list, and a gap ends RedeliveryDateSlack above it. -// After a silence, a gap starts at the Date of the run before, less -// RedeliveryDateMargin. A Date that is missing, or further behind an attempt -// than the slack, fails its read: on the first page the run fails, and on a -// later page the range stays open until it expires. +// The scan takes no read bound and no gap from faucet's clock, because that +// clock and GitHub's differ. Faucet's clock is only the time of record that +// each step gives the store (ScanStep.Now). Where a read stops comes from the +// time of an attempt, which GitHub's clock had passed when it listed the +// attempt. A Date can only lower that bound: the first run after an empty list +// or the migration also reads back to the Date of that time, less +// RedeliveryDateMargin, and its gap covers what lies below. Where GitHub's +// retention has ended comes from the Date header of the page that saw the end +// of the list, and a gap ends RedeliveryDateSlack above it. After a silence, a +// gap starts at the Date of the run before, less RedeliveryDateMargin. A Date +// that is missing, or further behind an attempt than the slack, fails its +// read: on the first page the run fails, and on a later page the range stays +// open until it expires. // // A delivery is asked for at most MaxAsksPerDelivery times, on the schedule // that AskOffsets gives, and at most MaxAsksPerRun times in a run, the delivery diff --git a/faucet/redeliverystore.go b/faucet/redeliverystore.go index 47abe72..20b7eeb 100644 --- a/faucet/redeliverystore.go +++ b/faucet/redeliverystore.go @@ -77,9 +77,9 @@ type RedeliveryScan struct { // Epoch fences the run's writes. A later lease has a higher epoch. Epoch int64 // Top is the time of the newest attempt that an earlier run read, or zero - // when no run has read one. It is the only stored time that a read bound - // comes from, so it is always an attempt time: a run that reads an empty - // list leaves it as it is. + // when no run has read one. The next head read stops below it, so it is + // always an attempt time: a run that reads an empty list leaves it as it + // is. Top time.Time // Clear is the Date header that GitHub sent with the first page of the // last run whose head read worked, or zero before one. It says when a run From 5e1c6322346b4bb62cafe6fc13fb37e86cca30db Mon Sep 17 00:00:00 2001 From: Thomas Kosiewski Date: Mon, 16 Nov 2026 12:39:50 +0000 Subject: [PATCH 10/18] fix(faucet): fail a redelivery run whose store returned a top without a clear `RedeliveryScan.Clear` and `ScanStep.Clear` are new struct fields, so a `RedeliveryStore` that was written before them compiles unchanged and drops the clear. With such a store the scan broke its promise without a word. A fake store that ignores `Clear` fails 7 of 46 redelivery tests, and in two of them a failure ends with no final state: no range closes unread without a stored clear, so the ranges of a burst stay open for good, and an empty first run leaves no start of tracking. A store that keeps `Clear` and still holds a top from before it has the other half of the problem. That top can be a Date, which is what migration `000008_redelivery_clear` exists for. Both cases show as a top without a clear. A store that follows the contract never returns one: a head step writes both, and the migration and a changed hook reset the top. `topAndClearOf` now rejects it, and the run fails before it reads a page. It asks for nothing, and the settle and the purge still run, as after a failed first page. The README tells the owner of such a store what to do. `headGapStart` loses its case of a top without a clear, which can no longer reach it. This check is not in the rules of design v4. It is the last commit of the branch, so it can be dropped on its own. Refs #26 --- README.md | 2 +- faucet/fakeredelivery_test.go | 10 +++++++--- faucet/redeliver.go | 9 +++++++-- faucet/redeliver_gaps_test.go | 20 ++++++++++++++++++++ faucet/redelivertime.go | 22 +++++++++++++--------- faucet/redeliverystore.go | 3 ++- 6 files changed, 50 insertions(+), 16 deletions(-) diff --git a/README.md b/README.md index cbf2629..8742f3f 100644 --- a/README.md +++ b/README.md @@ -198,7 +198,7 @@ The server applies pending migrations when a production instance starts, over `D - **Rows sealed before secret rotation existed.** The first refresh of such a row with the same secret counts as one rotation, because the new sealing of the secret is not the old bytes. For up to 10 minutes the old sealing is the previous secret. It opens to the same secret, so a signature has one `v1,` entry. - **A `CRON_SECRET` that no request can carry.** The server now stops at start, and names the variable, when `CRON_SECRET` has a space, a tab, a newline or another control character, a byte outside printable ASCII, or more than 1024 bytes. Before, it started and answered 401 to every cron call. A value from `openssl rand -hex 32` or `openssl rand -base64 32` passes, and so does any other run of printable ASCII characters. A secret that you read from a file can end in a newline that you must remove. - **A `Store` of your own.** The interface has gained methods, and `storetest` checks each one: `DeleteLiveSubscription`, `ClearExpiredPreviousSecrets`, a `Purge` that also moves the subscriptions that lost nothing past the purge mark, `ClaimGap` and `FinishGap`. `ClaimDelivery` takes a `skip` list of callback origins. `Enqueue` has `GapThrough`, `DeliveryRef` and `GapRef` have `Incarnation`, `ScanTarget` has `Principal` and no callback or secret fields, and `PurgeResult` has two counts. A `Store` without them does not compile. A `Store` that compiles can still fail `storetest`. `ClaimDelivery` and `ClaimGap` must skip a callback by `faucet.CallbackOrigin` and by no other rule of your own, so a stored URL in any spelling (a host in capitals, a scheme in capitals, user info, an IPv6 literal) is held to the cap of its origin. `storetest` checks this since PR 27. A `RedeliveryStore` is a second interface, `faucet.RedeliveryStore`, with its own suite, `storetest.RunRedelivery`. -- **The redelivery position after migration `000008_redelivery_clear`.** Before this migration the position of a run could be GitHub's `Date` header, and no row says whether it is. The migration moves the old position into the new `clear_at` column and resets the position. The first run after the migration is therefore a first run. It knows when tracking began, so it loses no failure silently. The cost comes once. The runs after the migration read the last 24 hours of the list again, and the first of them logs one more `before tracking began` gap for the time before those 24 hours. A delivery that a run tracked before the migration keeps its row and its ask count, so reading its attempts again changes nothing. On a hook near the capacity, the sweep cannot read that day again before GitHub deletes it, and the unread part ends as `expired unread` gaps. A `RedeliveryStore` of your own must keep `Clear` (`RedeliveryScan.Clear` and `ScanStep.Clear`), reset it with the top when the hook changes, and start the gaps of a changed hook the 5 minute overlap below each range's old end. `storetest.RunRedelivery` checks all three. +- **The redelivery position after migration `000008_redelivery_clear`.** Before this migration the position of a run could be GitHub's `Date` header, and no row says whether it is. The migration moves the old position into the new `clear_at` column and resets the position. The first run after the migration is therefore a first run. It knows when tracking began, so it loses no failure silently. The cost comes once. The runs after the migration read the last 24 hours of the list again, and the first of them logs one more `before tracking began` gap for the time before those 24 hours. A delivery that a run tracked before the migration keeps its row and its ask count, so reading its attempts again changes nothing. On a hook near the capacity, the sweep cannot read that day again before GitHub deletes it, and the unread part ends as `expired unread` gaps. A `RedeliveryStore` of your own must keep `Clear` (`RedeliveryScan.Clear` and `ScanStep.Clear`), reset it with the top when the hook changes, and start the gaps of a changed hook the 5 minute overlap below each range's old end. `storetest.RunRedelivery` checks all three. It must also do once what the migration does: move the position that it holds into `Clear`, and reset the position. Until then every run fails with `the store returned a top without a clear` and asks for nothing, because a position from before the migration can be a `Date`. ## Tests diff --git a/faucet/fakeredelivery_test.go b/faucet/fakeredelivery_test.go index 1a3919e..6acb7d6 100644 --- a/faucet/fakeredelivery_test.go +++ b/faucet/fakeredelivery_test.go @@ -329,10 +329,14 @@ func (s *fakeRedeliveryStore) applyMigration000008() { s.clearAt, s.top = s.top, time.Time{} } -func (s *fakeRedeliveryStore) seedMigratedTop(oldTop time.Time) { +func (s *fakeRedeliveryStore) seedTopWithoutClear(top time.Time) { s.rmu.Lock() - s.top = oldTop - s.rmu.Unlock() + defer s.rmu.Unlock() + s.top = top +} + +func (s *fakeRedeliveryStore) seedMigratedTop(oldTop time.Time) { + s.seedTopWithoutClear(oldTop) s.applyMigration000008() } diff --git a/faucet/redeliver.go b/faucet/redeliver.go index 8fca467..59b285b 100644 --- a/faucet/redeliver.go +++ b/faucet/redeliver.go @@ -213,7 +213,9 @@ type RedeliverResult struct { // gap starts at the Date of the run before, less RedeliveryDateMargin. A Date // that is missing, or further behind an attempt than the slack, fails its // read: on the first page the run fails, and on a later page the range stays -// open until it expires. +// open until it expires. A top that the store returns without a clear fails +// the run before it reads: the store did not keep the clear, or it holds a top +// from before there was one, which can be a Date. // // A delivery is asked for at most MaxAsksPerDelivery times, on the schedule // that AskOffsets gives, and at most MaxAsksPerRun times in a run, the delivery @@ -392,7 +394,10 @@ func (r *redeliveryRun) countGap(ctx context.Context, gap RedeliveryGap, cause s // scan reads the head of the list and then the open ranges, newest first. It // returns whether the head read worked. func (r *redeliveryRun) scan(ctx context.Context, scan RedeliveryScan) (bool, error) { - top, clearAt := topOf(scan), clearOf(scan) + top, clearAt, err := topAndClearOf(scan) + if err != nil { + return false, err + } page, err := r.fetch(ctx, "") if err != nil { return false, err diff --git a/faucet/redeliver_gaps_test.go b/faucet/redeliver_gaps_test.go index 4173c82..8ff756d 100644 --- a/faucet/redeliver_gaps_test.go +++ b/faucet/redeliver_gaps_test.go @@ -643,6 +643,26 @@ func TestAfterTheMigrationATopThatWasADateAheadDropsNoFailure(t *testing.T) { } } +func TestATopThatTheStoreReturnsWithoutAClearFailsTheRunUntilItMovesToTheClear(t *testing.T) { + t.Parallel() + env := newRedeliveryEnv(t) + env.clock.WithLogger(quartz.NoOpLogger) + t0 := env.gh.now() + env.store.seedTopWithoutClear(t0.Add(-time.Hour)) + f := env.gh.record("F", t0.Add(-time.Minute), 503) + + res, err := env.svc.Redeliver(context.Background()) + + require.ErrorContains(t, err, "a top without a clear") + require.Zero(t, res.Pages, "the run reads nothing below a top that may be a Date") + require.Zero(t, res.Asked) + require.False(t, env.tracked(f.guid)) + + env.store.applyMigration000008() + require.Equal(t, 1, env.run().Tracked, "the migration moves the top to the clear, and the next run is a first run") + require.True(t, env.tracked(f.guid)) +} + func TestALateWriteBelowTheTopIsInsideTheGapWhileTheDateLeadsByLessThanTheMarginAndTheOverlap(t *testing.T) { t.Parallel() diff --git a/faucet/redelivertime.go b/faucet/redelivertime.go index 7d81930..4243f18 100644 --- a/faucet/redelivertime.go +++ b/faucet/redelivertime.go @@ -23,8 +23,6 @@ const ( type attemptTime struct{ at time.Time } -func topOf(scan RedeliveryScan) attemptTime { return attemptTime{scan.Top} } - func deliveredAt(d HookDelivery) attemptTime { return attemptTime{d.DeliveredAt} } func (r RedeliveryRange) newerEnd() attemptTime { return attemptTime{r.NewerAt} } @@ -94,7 +92,15 @@ func (d pageDate) gapEnd(covered attemptTime) time.Time { type storedClear struct{ clearAt time.Time } -func clearOf(scan RedeliveryScan) storedClear { return storedClear{scan.Clear} } +var errTopWithoutClear = xerrors.New("the store returned a top without a clear: a RedeliveryStore must keep Clear, and must move a top that it stored before into Clear") + +func topAndClearOf(scan RedeliveryScan) (attemptTime, storedClear, error) { + top, clearAt := attemptTime{scan.Top}, storedClear{scan.Clear} + if top.set() && !clearAt.set() { + return attemptTime{}, storedClear{}, errTopWithoutClear + } + return top, clearAt, nil +} func (c storedClear) set() bool { return !c.clearAt.IsZero() } @@ -111,14 +117,12 @@ func (c storedClear) unseenFrom() time.Time { func (c storedClear) headGapStart(top attemptTime) (time.Time, bool) { switch { - case top.set() && c.set(): - return laterOf(top.bound().stop, c.unseenFrom()), true - case top.set(): - return top.bound().stop, true - case c.set(): + case !c.set(): + return time.Time{}, false + case !top.set(): return c.unseenFrom(), true } - return time.Time{}, false + return laterOf(top.bound().stop, c.unseenFrom()), true } func (c storedClear) firstRunBound(newest attemptTime) readBound { diff --git a/faucet/redeliverystore.go b/faucet/redeliverystore.go index 20b7eeb..ce0ab5f 100644 --- a/faucet/redeliverystore.go +++ b/faucet/redeliverystore.go @@ -84,7 +84,8 @@ type RedeliveryScan struct { // Clear is the Date header that GitHub sent with the first page of the // last run whose head read worked, or zero before one. It says when a run // last saw the top of the list. A run takes gap starts from it, and the - // bound of a first run, and never another read bound. + // bound of a first run, and never another read bound. A store keeps it + // with the Top: a run fails on a Top without a Clear. Clear time.Time // Ranges are the open ranges, newest first. Ranges []RedeliveryRange From 586d15465c8beee370a6187a5894d466934ecf51 Mon Sep 17 00:00:00 2001 From: Thomas Kosiewski Date: Mon, 16 Nov 2026 13:36:40 +0000 Subject: [PATCH 11/18] fix(faucet): allow a Date the second that it drops when checking it against an attempt An HTTP Date drops the fraction of its second. checkDate compared it with the newest attempt less the slack, so a Date whose clock was 59m59.5s behind an attempt later in the same second failed the check on every run, and the run asked for nothing. The check now accepts a Date when Date + 1s is not before the newest attempt less the slack, as the gap floor already allows for. --- faucet/redeliver_gaps_test.go | 26 ++++++++++++++++++++++++++ faucet/redelivertime.go | 4 ++-- 2 files changed, 28 insertions(+), 2 deletions(-) diff --git a/faucet/redeliver_gaps_test.go b/faucet/redeliver_gaps_test.go index 8ff756d..afdff3a 100644 --- a/faucet/redeliver_gaps_test.go +++ b/faucet/redeliver_gaps_test.go @@ -863,6 +863,32 @@ func TestAWholeSecondDateAsFarBehindAsTheSlackStillEndsTheGapAboveWhatGitHubDele } } +func TestAWholeSecondDateAsFarBehindAsTheSlackPassesTheCheckAgainstAnAttemptLaterInItsSecond(t *testing.T) { + t.Parallel() + + for _, lag := range []time.Duration{faucet.RedeliveryDateSlack - 500*time.Millisecond, faucet.RedeliveryDateSlack} { + t.Run(lag.String(), func(t *testing.T) { + t.Parallel() + env := newRedeliveryEnv(t) + env.clock.WithLogger(quartz.NoOpLogger) + wholeSecondDate := func(at time.Time) time.Time { return at.Add(-lag).Truncate(time.Second) } + env.gh.date = func(_ int, _ string, at time.Time) time.Time { return wholeSecondDate(at) } + intoItsSecond := 200 * time.Millisecond + env.advance(env.gh.now().Truncate(time.Second).Add(time.Second + intoItsSecond).Sub(env.gh.now())) + + for i := range 6 { + f := env.gh.record(fmt.Sprintf("f-%d", i), env.gh.now().Add(-50*time.Millisecond), 503) + require.True(t, wholeSecondDate(env.gh.now()).Before(f.at.Add(-faucet.RedeliveryDateSlack)), + "the Date drops more of its second than the slack leaves above it, or the scenario proves nothing") + res, err := env.svc.Redeliver(context.Background()) + require.NoError(t, err, "the clock that the Date shows is no more than the slack behind %s", f.guid) + require.Equal(t, 1, res.Tracked) + env.advance(cronEvery) + } + }) + } +} + func TestAFirstPageWithoutAUsableDateFailsTheRunWhichAsksForNothing(t *testing.T) { t.Parallel() diff --git a/faucet/redelivertime.go b/faucet/redelivertime.go index 4243f18..e229fe3 100644 --- a/faucet/redelivertime.go +++ b/faucet/redelivertime.go @@ -75,9 +75,9 @@ func checkDate(page DeliveryPage, newestKnown attemptTime) (pageDate, error) { switch { case page.At.IsZero(): return pageDate{}, xerrors.Errorf("%w: the page has none", errUnusableDate) - case page.At.Before(known.at.Add(-RedeliveryDateSlack)): + case page.At.Add(redeliveryDateResolution).Before(known.at.Add(-RedeliveryDateSlack)): return pageDate{}, xerrors.Errorf("%w: it is %s before the newest attempt, and at most %s is allowed", - errUnusableDate, known.at.Sub(page.At), RedeliveryDateSlack) + errUnusableDate, known.at.Sub(page.At), RedeliveryDateSlack+redeliveryDateResolution) } return pageDate{page.At}, nil } From c8a708391422fcde03ac6b9a0746ad016d5808f1 Mon Sep 17 00:00:00 2001 From: Thomas Kosiewski Date: Mon, 16 Nov 2026 13:41:18 +0000 Subject: [PATCH 12/18] fix(faucet): make migration 000008 fence the redelivery runs of the build before it A run of the build before the migration that held the lease during it, or that an instance not yet replaced by a rolling deploy started after it, still wrote top_at with main's SQL. On a first run that left a top without a clear, and every later run failed; otherwise it put a Date back into the position, and failures below it were lost. The up migration now moves the old position into clear_at, renames top_at to top_attempt_at, raises the epoch and ends the lease. Every redelivery statement of the old build on the scan row reads or writes top_at, so its runs fail with a column error and write no position, and its other routes keep working. The down migration renames the column back. A test runs the old build's statements, as sqlc generated them at e8382ff, against a migrated database, both in flight and started after the migration, and then shows that this build tracks every failure. --- README.md | 4 +- faucet/pgstore/database/models.go | 12 +- faucet/pgstore/database/redelivery.sql.go | 26 +-- .../000008_redelivery_clear.down.sql | 3 +- .../migrations/000008_redelivery_clear.up.sql | 10 +- faucet/pgstore/pgstore_test.go | 5 + faucet/pgstore/queries/redelivery.sql | 8 +- faucet/pgstore/redelivery.go | 4 +- faucet/postgres_migration_test.go | 193 ++++++++++++++++++ 9 files changed, 234 insertions(+), 31 deletions(-) create mode 100644 faucet/postgres_migration_test.go diff --git a/README.md b/README.md index 8742f3f..4908386 100644 --- a/README.md +++ b/README.md @@ -189,7 +189,7 @@ To embed faucet, set `Deps.Redelivery` to a `RedeliveryDeps`: a `RedeliveryStore The server applies pending migrations when a production instance starts, over `DATABASE_URL_UNPOOLED` (or over `DATABASE_URL` when that is unset), before it serves a request. -**Do not roll back past the newest migration.** After production has migrated to `000008_redelivery_clear`, a build that does not embed it, which is what Vercel Instant Rollback to an older deployment gives, cannot start: the migration library fails with `no migration found for version 8` and every new instance exits, so the rollback serves nothing. Recover in one of two ways. Redeploy a build that embeds the migration. Or roll the database back with the down migration `faucet/pgstore/migrations/000008_redelivery_clear.down.sql`, applied through golang-migrate against the `faucet_schema_migrations` table so that the recorded version drops to 7. The down migration drops the `clear_at` column. A redelivery scan that has read no attempt since the migration gets its old position back. The same holds for each later migration, and for the one before: the down migration of `000007_gap_pending` drops the `gap_through` and `gap_lease_until` columns, and with them every gap that is owed and not yet sent. +**Do not roll back past the newest migration.** After production has migrated to `000008_redelivery_clear`, a build that does not embed it, which is what Vercel Instant Rollback to an older deployment gives, cannot start: the migration library fails with `no migration found for version 8` and every new instance exits, so the rollback serves nothing. Recover in one of two ways. Redeploy a build that embeds the migration. Or roll the database back with the down migration `faucet/pgstore/migrations/000008_redelivery_clear.down.sql`, applied through golang-migrate against the `faucet_schema_migrations` table so that the recorded version drops to 7. The down migration gives the position column its old name and drops the `clear_at` column. A redelivery scan that has read no attempt since the migration gets its old position back. The same holds for each later migration, and for the one before: the down migration of `000007_gap_pending` drops the `gap_through` and `gap_lease_until` columns, and with them every gap that is owed and not yet sent. ## Upgrading @@ -198,7 +198,7 @@ The server applies pending migrations when a production instance starts, over `D - **Rows sealed before secret rotation existed.** The first refresh of such a row with the same secret counts as one rotation, because the new sealing of the secret is not the old bytes. For up to 10 minutes the old sealing is the previous secret. It opens to the same secret, so a signature has one `v1,` entry. - **A `CRON_SECRET` that no request can carry.** The server now stops at start, and names the variable, when `CRON_SECRET` has a space, a tab, a newline or another control character, a byte outside printable ASCII, or more than 1024 bytes. Before, it started and answered 401 to every cron call. A value from `openssl rand -hex 32` or `openssl rand -base64 32` passes, and so does any other run of printable ASCII characters. A secret that you read from a file can end in a newline that you must remove. - **A `Store` of your own.** The interface has gained methods, and `storetest` checks each one: `DeleteLiveSubscription`, `ClearExpiredPreviousSecrets`, a `Purge` that also moves the subscriptions that lost nothing past the purge mark, `ClaimGap` and `FinishGap`. `ClaimDelivery` takes a `skip` list of callback origins. `Enqueue` has `GapThrough`, `DeliveryRef` and `GapRef` have `Incarnation`, `ScanTarget` has `Principal` and no callback or secret fields, and `PurgeResult` has two counts. A `Store` without them does not compile. A `Store` that compiles can still fail `storetest`. `ClaimDelivery` and `ClaimGap` must skip a callback by `faucet.CallbackOrigin` and by no other rule of your own, so a stored URL in any spelling (a host in capitals, a scheme in capitals, user info, an IPv6 literal) is held to the cap of its origin. `storetest` checks this since PR 27. A `RedeliveryStore` is a second interface, `faucet.RedeliveryStore`, with its own suite, `storetest.RunRedelivery`. -- **The redelivery position after migration `000008_redelivery_clear`.** Before this migration the position of a run could be GitHub's `Date` header, and no row says whether it is. The migration moves the old position into the new `clear_at` column and resets the position. The first run after the migration is therefore a first run. It knows when tracking began, so it loses no failure silently. The cost comes once. The runs after the migration read the last 24 hours of the list again, and the first of them logs one more `before tracking began` gap for the time before those 24 hours. A delivery that a run tracked before the migration keeps its row and its ask count, so reading its attempts again changes nothing. On a hook near the capacity, the sweep cannot read that day again before GitHub deletes it, and the unread part ends as `expired unread` gaps. A `RedeliveryStore` of your own must keep `Clear` (`RedeliveryScan.Clear` and `ScanStep.Clear`), reset it with the top when the hook changes, and start the gaps of a changed hook the 5 minute overlap below each range's old end. `storetest.RunRedelivery` checks all three. It must also do once what the migration does: move the position that it holds into `Clear`, and reset the position. Until then every run fails with `the store returned a top without a clear` and asks for nothing, because a position from before the migration can be a `Date`. +- **The redelivery position after migration `000008_redelivery_clear`.** Before this migration the position of a run could be GitHub's `Date` header, and no row says whether it is. The migration moves the old position into the new `clear_at` column, resets the position and gives its column a new name. Until a deploy has replaced every instance of the build before the migration, the redelivery runs of those instances fail with `column "top_at" does not exist` and write no position, and so does a run that was in flight during the migration. Their other routes keep working, and the runs of the new build read the list in their place. The first run after the migration is therefore a first run. It knows when tracking began, so it loses no failure silently. The cost comes once. The runs after the migration read the last 24 hours of the list again, and the first of them logs one more `before tracking began` gap for the time before those 24 hours. A delivery that a run tracked before the migration keeps its row and its ask count, so reading its attempts again changes nothing. On a hook near the capacity, the sweep cannot read that day again before GitHub deletes it, and the unread part ends as `expired unread` gaps. A `RedeliveryStore` of your own must keep `Clear` (`RedeliveryScan.Clear` and `ScanStep.Clear`), reset it with the top when the hook changes, and start the gaps of a changed hook the 5 minute overlap below each range's old end. `storetest.RunRedelivery` checks all three. It must also do once what the migration does: move the position that it holds into `Clear`, and reset the position. Until then every run fails with `the store returned a top without a clear` and asks for nothing, because a position from before the migration can be a `Date`. ## Tests diff --git a/faucet/pgstore/database/models.go b/faucet/pgstore/database/models.go index 527f996..5e6a13d 100644 --- a/faucet/pgstore/database/models.go +++ b/faucet/pgstore/database/models.go @@ -60,12 +60,12 @@ type FaucetRedeliveryRange struct { } type FaucetRedeliveryScan struct { - ID bool - Hook string - Epoch int64 - LeaseUntil sql.NullTime - TopAt sql.NullTime - ClearAt sql.NullTime + ID bool + Hook string + Epoch int64 + LeaseUntil sql.NullTime + TopAttemptAt sql.NullTime + ClearAt sql.NullTime } type FaucetSubscription struct { diff --git a/faucet/pgstore/database/redelivery.sql.go b/faucet/pgstore/database/redelivery.sql.go index 0b1d4f0..cbb3572 100644 --- a/faucet/pgstore/database/redelivery.sql.go +++ b/faucet/pgstore/database/redelivery.sql.go @@ -228,14 +228,14 @@ func (q *Queries) ListOpenRedeliveryRanges(ctx context.Context) ([]ListOpenRedel } const lockRedeliveryScan = `-- name: LockRedeliveryScan :one -SELECT hook, epoch, lease_until, top_at FROM faucet_redelivery_scan FOR UPDATE +SELECT hook, epoch, lease_until, top_attempt_at FROM faucet_redelivery_scan FOR UPDATE ` type LockRedeliveryScanRow struct { - Hook string - Epoch int64 - LeaseUntil sql.NullTime - TopAt sql.NullTime + Hook string + Epoch int64 + LeaseUntil sql.NullTime + TopAttemptAt sql.NullTime } func (q *Queries) LockRedeliveryScan(ctx context.Context) (LockRedeliveryScanRow, error) { @@ -245,7 +245,7 @@ func (q *Queries) LockRedeliveryScan(ctx context.Context) (LockRedeliveryScanRow &i.Hook, &i.Epoch, &i.LeaseUntil, - &i.TopAt, + &i.TopAttemptAt, ) return i, err } @@ -271,7 +271,7 @@ func (q *Queries) PurgeRedeliveries(ctx context.Context, before time.Time) (int6 } const raiseRedeliveryTop = `-- name: RaiseRedeliveryTop :exec -UPDATE faucet_redelivery_scan SET top_at = GREATEST(top_at, $1::timestamptz) +UPDATE faucet_redelivery_scan SET top_attempt_at = GREATEST(top_attempt_at, $1::timestamptz) ` func (q *Queries) RaiseRedeliveryTop(ctx context.Context, top time.Time) error { @@ -450,9 +450,9 @@ UPDATE faucet_redelivery_scan SET hook = $1, epoch = epoch + 1, lease_until = $2::timestamptz, - top_at = CASE WHEN $3::boolean THEN NULL ELSE top_at END, + top_attempt_at = CASE WHEN $3::boolean THEN NULL ELSE top_attempt_at END, clear_at = CASE WHEN $3::boolean THEN NULL ELSE clear_at END -RETURNING epoch, top_at, clear_at +RETURNING epoch, top_attempt_at, clear_at ` type TakeRedeliveryLeaseParams struct { @@ -462,15 +462,15 @@ type TakeRedeliveryLeaseParams struct { } type TakeRedeliveryLeaseRow struct { - Epoch int64 - TopAt sql.NullTime - ClearAt sql.NullTime + Epoch int64 + TopAttemptAt sql.NullTime + ClearAt sql.NullTime } func (q *Queries) TakeRedeliveryLease(ctx context.Context, arg TakeRedeliveryLeaseParams) (TakeRedeliveryLeaseRow, error) { row := q.db.QueryRowContext(ctx, takeRedeliveryLease, arg.Hook, arg.LeaseUntil, arg.HookChanged) var i TakeRedeliveryLeaseRow - err := row.Scan(&i.Epoch, &i.TopAt, &i.ClearAt) + err := row.Scan(&i.Epoch, &i.TopAttemptAt, &i.ClearAt) return i, err } diff --git a/faucet/pgstore/migrations/000008_redelivery_clear.down.sql b/faucet/pgstore/migrations/000008_redelivery_clear.down.sql index 23ee8d0..d0b1b8c 100644 --- a/faucet/pgstore/migrations/000008_redelivery_clear.down.sql +++ b/faucet/pgstore/migrations/000008_redelivery_clear.down.sql @@ -1,3 +1,4 @@ -UPDATE faucet_redelivery_scan SET top_at = COALESCE(top_at, clear_at); +ALTER TABLE faucet_redelivery_scan RENAME COLUMN top_attempt_at TO top_at; +UPDATE faucet_redelivery_scan SET top_at = COALESCE(top_at, clear_at), epoch = epoch + 1, lease_until = NULL; ALTER TABLE faucet_redelivery_scan DROP COLUMN clear_at; diff --git a/faucet/pgstore/migrations/000008_redelivery_clear.up.sql b/faucet/pgstore/migrations/000008_redelivery_clear.up.sql index 936a1df..78381e3 100644 --- a/faucet/pgstore/migrations/000008_redelivery_clear.up.sql +++ b/faucet/pgstore/migrations/000008_redelivery_clear.up.sql @@ -1,9 +1,13 @@ -- clear_at is the Date header of GitHub's answer to the last head read: when, -- on GitHub's clock, a run last saw the top of the list. A run takes its gap -- starts from it and never a read bound, because a Date can run ahead of --- GitHub's list. top_at keeps attempt times only. +-- GitHub's list. top_attempt_at keeps attempt times only. ALTER TABLE faucet_redelivery_scan ADD COLUMN clear_at timestamptz; -- Until now top_at could hold a Date that ran ahead of GitHub's list, and no --- row says whether it does (see #26). -UPDATE faucet_redelivery_scan SET clear_at = top_at, top_at = NULL; +-- row says whether it does (see #26). The position gets a new name, so a run +-- of an older build fails when it locks this row and never writes a Date into +-- the position again. The epoch ends the lease of a run that started before +-- the migration. +UPDATE faucet_redelivery_scan SET clear_at = top_at, top_at = NULL, epoch = epoch + 1, lease_until = NULL; +ALTER TABLE faucet_redelivery_scan RENAME COLUMN top_at TO top_attempt_at; diff --git a/faucet/pgstore/pgstore_test.go b/faucet/pgstore/pgstore_test.go index f00dbe3..b9295af 100644 --- a/faucet/pgstore/pgstore_test.go +++ b/faucet/pgstore/pgstore_test.go @@ -170,6 +170,11 @@ func TestTheClearMigrationMovesTheOldTopToTheClearAndBack(t *testing.T) { migrate("up") attempt, date := now.Add(time.Hour), now.Add(time.Hour+time.Second) _, err = store.ApplyRedeliveryScan(ctx, faucet.ScanStep{Epoch: migrated.Epoch, Now: now.Add(time.Hour), Top: attempt, Clear: date}) + require.ErrorIs(t, err, faucet.ErrRedeliveryLeaseLost, "the migration ends a lease that a run took before it") + again, ok, err := store.AcquireRedeliveryLease(ctx, hook, now.Add(time.Hour), time.Minute) + require.NoError(t, err) + require.True(t, ok) + _, err = store.ApplyRedeliveryScan(ctx, faucet.ScanStep{Epoch: again.Epoch, Now: now.Add(time.Hour), Top: attempt, Clear: date}) require.NoError(t, err) migrate("down") require.True(t, attempt.Equal(storedTop().Time), "a scan that has read an attempt keeps that attempt's time, not the Date") diff --git a/faucet/pgstore/queries/redelivery.sql b/faucet/pgstore/queries/redelivery.sql index 1925ebe..332fa1e 100644 --- a/faucet/pgstore/queries/redelivery.sql +++ b/faucet/pgstore/queries/redelivery.sql @@ -4,7 +4,7 @@ VALUES (true, sqlc.arg(hook), 0) ON CONFLICT (id) DO NOTHING; -- name: LockRedeliveryScan :one -SELECT hook, epoch, lease_until, top_at FROM faucet_redelivery_scan FOR UPDATE; +SELECT hook, epoch, lease_until, top_attempt_at FROM faucet_redelivery_scan FOR UPDATE; -- name: CloseOpenRedeliveryRanges :many UPDATE faucet_redelivery_ranges @@ -17,9 +17,9 @@ UPDATE faucet_redelivery_scan SET hook = sqlc.arg(hook), epoch = epoch + 1, lease_until = sqlc.arg(lease_until)::timestamptz, - top_at = CASE WHEN sqlc.arg(hook_changed)::boolean THEN NULL ELSE top_at END, + top_attempt_at = CASE WHEN sqlc.arg(hook_changed)::boolean THEN NULL ELSE top_attempt_at END, clear_at = CASE WHEN sqlc.arg(hook_changed)::boolean THEN NULL ELSE clear_at END -RETURNING epoch, top_at, clear_at; +RETURNING epoch, top_attempt_at, clear_at; -- name: ListOpenRedeliveryRanges :many SELECT id, newer_at, older_at, cursor @@ -63,7 +63,7 @@ WHERE guid = ANY(sqlc.arg(guids)::text[]) AND state = 'pending' RETURNING guid; -- name: RaiseRedeliveryTop :exec -UPDATE faucet_redelivery_scan SET top_at = GREATEST(top_at, sqlc.arg(top)::timestamptz); +UPDATE faucet_redelivery_scan SET top_attempt_at = GREATEST(top_attempt_at, sqlc.arg(top)::timestamptz); -- name: SetRedeliveryClear :exec UPDATE faucet_redelivery_scan SET clear_at = sqlc.arg(clear_at)::timestamptz; diff --git a/faucet/pgstore/redelivery.go b/faucet/pgstore/redelivery.go index c547810..846039a 100644 --- a/faucet/pgstore/redelivery.go +++ b/faucet/pgstore/redelivery.go @@ -56,8 +56,8 @@ func (s *Store) AcquireRedeliveryLease(ctx context.Context, hook string, now tim return faucet.RedeliveryScan{}, false, xerrors.Errorf("take lease: %w", err) } scan.Epoch = taken.Epoch - if taken.TopAt.Valid { - scan.Top = taken.TopAt.Time.UTC() + if taken.TopAttemptAt.Valid { + scan.Top = taken.TopAttemptAt.Time.UTC() } if taken.ClearAt.Valid { scan.Clear = taken.ClearAt.Time.UTC() diff --git a/faucet/postgres_migration_test.go b/faucet/postgres_migration_test.go new file mode 100644 index 0000000..009dd22 --- /dev/null +++ b/faucet/postgres_migration_test.go @@ -0,0 +1,193 @@ +package faucet_test + +import ( + "context" + "database/sql" + "fmt" + "os" + "testing" + "time" + + "github.com/lib/pq" + "github.com/stretchr/testify/require" + + "github.com/coder/faucet/faucet" + "github.com/coder/faucet/faucet/pgstore" + "github.com/coder/quartz" +) + +// The statements on the scan row of the build before migration +// 000008_redelivery_clear, as sqlc generated them at e8382ff. +const ( + mainEnsureRedeliveryScan = `INSERT INTO faucet_redelivery_scan (id, hook, epoch) +VALUES (true, $1, 0) +ON CONFLICT (id) DO NOTHING` + mainLockRedeliveryScan = `SELECT hook, epoch, lease_until, top_at FROM faucet_redelivery_scan FOR UPDATE` + mainTakeRedeliveryLease = `UPDATE faucet_redelivery_scan +SET hook = $1, + epoch = epoch + 1, + lease_until = $2::timestamptz, + top_at = CASE WHEN $3::boolean THEN NULL ELSE top_at END +RETURNING epoch, top_at` + mainRaiseRedeliveryTop = `UPDATE faucet_redelivery_scan SET top_at = GREATEST(top_at, $1::timestamptz)` + mainReleaseRedeliveryLease = `UPDATE faucet_redelivery_scan SET lease_until = NULL WHERE epoch = $1` +) + +// mainBuild takes the lease and applies a step that carries only a top, in +// the order and the transactions of the build before the migration. On an +// empty list that build's top is the Date. +type mainBuild struct { + db *sql.DB + hook string +} + +func (b mainBuild) acquire(ctx context.Context, now time.Time) (epoch int64, err error) { + tx, err := b.db.BeginTx(ctx, &sql.TxOptions{Isolation: sql.LevelReadCommitted}) + if err != nil { + return 0, err + } + defer func() { _ = tx.Rollback() }() + if _, err := tx.ExecContext(ctx, mainEnsureRedeliveryScan, b.hook); err != nil { + return 0, err + } + if err := b.lock(ctx, tx, nil); err != nil { + return 0, err + } + var top sql.NullTime + if err := tx.QueryRowContext(ctx, mainTakeRedeliveryLease, b.hook, now.Add(2*time.Minute), false).Scan(&epoch, &top); err != nil { + return 0, err + } + return epoch, tx.Commit() +} + +func (b mainBuild) raiseTop(ctx context.Context, epoch int64, top time.Time) error { + tx, err := b.db.BeginTx(ctx, &sql.TxOptions{Isolation: sql.LevelReadCommitted}) + if err != nil { + return err + } + defer func() { _ = tx.Rollback() }() + var locked int64 + if err := b.lock(ctx, tx, &locked); err != nil { + return err + } + if locked != epoch { + return faucet.ErrRedeliveryLeaseLost + } + if _, err := tx.ExecContext(ctx, mainRaiseRedeliveryTop, top); err != nil { + return err + } + return tx.Commit() +} + +func (mainBuild) lock(ctx context.Context, tx *sql.Tx, epoch *int64) error { + var ( + hook string + e int64 + lease, top sql.NullTime + ) + if err := tx.QueryRowContext(ctx, mainLockRedeliveryScan).Scan(&hook, &e, &lease, &top); err != nil { + return err + } + if epoch != nil { + *epoch = e + } + return nil +} + +func (b mainBuild) release(ctx context.Context, epoch int64) error { + _, err := b.db.ExecContext(ctx, mainReleaseRedeliveryLease, epoch) + return err +} + +func migrate000008(t *testing.T, db *sql.DB, direction string) { + t.Helper() + statements, err := os.ReadFile("pgstore/migrations/000008_redelivery_clear." + direction + ".sql") + require.NoError(t, err) + _, err = db.ExecContext(context.Background(), string(statements)) + require.NoError(t, err) +} + +func scanRow(t *testing.T, db *sql.DB) string { + t.Helper() + var row string + require.NoError(t, db.QueryRowContext(context.Background(), `SELECT to_jsonb(s)::text FROM faucet_redelivery_scan s`).Scan(&row)) + return row +} + +func TestARunOfTheBuildBeforeTheClearMigrationFailsAfterItAndWritesNoPosition(t *testing.T) { + t.Parallel() + + for _, tc := range []struct { + name string + hadTop, startsAfter bool + }{ + {"it holds the lease during the migration, and the row has a top", true, false}, + {"it holds the lease during the migration, and it is the first run", false, false}, + {"an instance of that build starts it after the migration", true, true}, + } { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + ctx := context.Background() + env := newRedeliveryEnv(t) + env.clock.WithLogger(quartz.NoOpLogger) + db := postgresDB(t) + env.use(pgstore.New(db), env.gh) + old := mainBuild{db: db, hook: env.gh.Hook()} + + migrate000008(t, db, "down") + if tc.hadTop { + epoch, err := old.acquire(ctx, env.now()) + require.NoError(t, err) + require.NoError(t, old.raiseTop(ctx, epoch, env.gh.now().Add(-time.Minute))) + require.NoError(t, old.release(ctx, epoch)) + env.advance(cronEvery) + } + var inFlight int64 + if !tc.startsAfter { + var err error + inFlight, err = old.acquire(ctx, env.now()) + require.NoError(t, err) + } + migrate000008(t, db, "up") + + migrated := scanRow(t, db) + epoch, oldErr := inFlight, error(nil) + if tc.startsAfter { + epoch, oldErr = old.acquire(ctx, env.now()) + } + if oldErr == nil { + oldErr = old.raiseTop(ctx, epoch, env.gh.now().Add(time.Hour)) + } + require.ErrorContains(t, oldErr, `column "top_at" does not exist`) + require.NoError(t, old.release(ctx, epoch)) + for _, statement := range []struct { + sql string + args []any + }{ + {mainLockRedeliveryScan, nil}, + {mainTakeRedeliveryLease, []any{old.hook, env.now(), false}}, + {mainRaiseRedeliveryTop, []any{env.gh.now()}}, + } { + _, err := db.ExecContext(ctx, statement.sql, statement.args...) + require.ErrorContains(t, err, `column "top_at" does not exist`, "no statement of the old build reads or writes the position: %s", statement.sql) + } + require.Equal(t, migrated, scanRow(t, db), "the old build's run changed nothing") + + res := env.run() + require.False(t, res.Busy, "a run of this build need not wait for the lease of a run that the migration fenced") + var failures []ghAttempt + for i := range 12 { + env.advance(cronEvery) + failures = append(failures, env.gh.record(fmt.Sprintf("f-%02d", i), env.gh.now().Add(-time.Second), 503)) + env.run() + } + guids := make([]string, len(failures)) + for i, f := range failures { + guids[i] = f.guid + } + var tracked int + require.NoError(t, db.QueryRowContext(ctx, `SELECT count(*) FROM faucet_redeliveries WHERE guid = ANY($1)`, pq.Array(guids)).Scan(&tracked)) + require.Equal(t, len(failures), tracked, "every failure after the migration is tracked") + }) + } +} From 4483bd79927593b629974983b2d925622467648b Mon Sep 17 00:00:00 2001 From: Thomas Kosiewski Date: Mon, 16 Nov 2026 13:45:04 +0000 Subject: [PATCH 13/18] fix(faucet): keep the gaps of a changed hook as the run logs them On a hook change the stores logged each open range as the gap [OlderAt - O, max(NewerAt, OlderAt)], but kept the row with its own older_at and newer_at: without the overlap, and inverted when NewerAt lay inside the overlap below OlderAt. RedeliveryRange.UnreadGap now gives the gap of a range that closes unread, for the run's expired ranges and for both stores. pgstore deletes the open ranges of the old hook and records each gap with the statement that a scan step uses, and the fake store does the same. storetest gains GapRecords: a store that lists the gaps it keeps is checked to keep each one as its step gave it, or as the store reported it when the hook changed. The pgstore and fake suites implement it. --- README.md | 2 +- faucet/fakeredelivery_test.go | 41 +++++++++++++------ faucet/pgstore/database/redelivery.sql.go | 38 +++++++++--------- faucet/pgstore/pgstore_test.go | 33 ++++++++++++++- faucet/pgstore/queries/redelivery.sql | 6 +-- faucet/pgstore/redelivery.go | 16 +++----- faucet/redeliver.go | 2 +- faucet/redeliverystore.go | 12 +++++- faucet/storetest/redelivery.go | 49 ++++++++++++++++++----- 9 files changed, 138 insertions(+), 61 deletions(-) diff --git a/README.md b/README.md index 4908386..7034d17 100644 --- a/README.md +++ b/README.md @@ -198,7 +198,7 @@ The server applies pending migrations when a production instance starts, over `D - **Rows sealed before secret rotation existed.** The first refresh of such a row with the same secret counts as one rotation, because the new sealing of the secret is not the old bytes. For up to 10 minutes the old sealing is the previous secret. It opens to the same secret, so a signature has one `v1,` entry. - **A `CRON_SECRET` that no request can carry.** The server now stops at start, and names the variable, when `CRON_SECRET` has a space, a tab, a newline or another control character, a byte outside printable ASCII, or more than 1024 bytes. Before, it started and answered 401 to every cron call. A value from `openssl rand -hex 32` or `openssl rand -base64 32` passes, and so does any other run of printable ASCII characters. A secret that you read from a file can end in a newline that you must remove. - **A `Store` of your own.** The interface has gained methods, and `storetest` checks each one: `DeleteLiveSubscription`, `ClearExpiredPreviousSecrets`, a `Purge` that also moves the subscriptions that lost nothing past the purge mark, `ClaimGap` and `FinishGap`. `ClaimDelivery` takes a `skip` list of callback origins. `Enqueue` has `GapThrough`, `DeliveryRef` and `GapRef` have `Incarnation`, `ScanTarget` has `Principal` and no callback or secret fields, and `PurgeResult` has two counts. A `Store` without them does not compile. A `Store` that compiles can still fail `storetest`. `ClaimDelivery` and `ClaimGap` must skip a callback by `faucet.CallbackOrigin` and by no other rule of your own, so a stored URL in any spelling (a host in capitals, a scheme in capitals, user info, an IPv6 literal) is held to the cap of its origin. `storetest` checks this since PR 27. A `RedeliveryStore` is a second interface, `faucet.RedeliveryStore`, with its own suite, `storetest.RunRedelivery`. -- **The redelivery position after migration `000008_redelivery_clear`.** Before this migration the position of a run could be GitHub's `Date` header, and no row says whether it is. The migration moves the old position into the new `clear_at` column, resets the position and gives its column a new name. Until a deploy has replaced every instance of the build before the migration, the redelivery runs of those instances fail with `column "top_at" does not exist` and write no position, and so does a run that was in flight during the migration. Their other routes keep working, and the runs of the new build read the list in their place. The first run after the migration is therefore a first run. It knows when tracking began, so it loses no failure silently. The cost comes once. The runs after the migration read the last 24 hours of the list again, and the first of them logs one more `before tracking began` gap for the time before those 24 hours. A delivery that a run tracked before the migration keeps its row and its ask count, so reading its attempts again changes nothing. On a hook near the capacity, the sweep cannot read that day again before GitHub deletes it, and the unread part ends as `expired unread` gaps. A `RedeliveryStore` of your own must keep `Clear` (`RedeliveryScan.Clear` and `ScanStep.Clear`), reset it with the top when the hook changes, and start the gaps of a changed hook the 5 minute overlap below each range's old end. `storetest.RunRedelivery` checks all three. It must also do once what the migration does: move the position that it holds into `Clear`, and reset the position. Until then every run fails with `the store returned a top without a clear` and asks for nothing, because a position from before the migration can be a `Date`. +- **The redelivery position after migration `000008_redelivery_clear`.** Before this migration the position of a run could be GitHub's `Date` header, and no row says whether it is. The migration moves the old position into the new `clear_at` column, resets the position and gives its column a new name. Until a deploy has replaced every instance of the build before the migration, the redelivery runs of those instances fail with `column "top_at" does not exist` and write no position, and so does a run that was in flight during the migration. Their other routes keep working, and the runs of the new build read the list in their place. The first run after the migration is therefore a first run. It knows when tracking began, so it loses no failure silently. The cost comes once. The runs after the migration read the last 24 hours of the list again, and the first of them logs one more `before tracking began` gap for the time before those 24 hours. A delivery that a run tracked before the migration keeps its row and its ask count, so reading its attempts again changes nothing. On a hook near the capacity, the sweep cannot read that day again before GitHub deletes it, and the unread part ends as `expired unread` gaps. A `RedeliveryStore` of your own must keep `Clear` (`RedeliveryScan.Clear` and `ScanStep.Clear`), reset it with the top when the hook changes, and close each open range of a changed hook as the gap that `RedeliveryRange.UnreadGap` gives, which it reports and keeps. `storetest.RunRedelivery` checks all three, and checks the gaps that the store keeps when it implements `storetest.GapRecords`. It must also do once what the migration does: move the position that it holds into `Clear`, and reset the position. Until then every run fails with `the store returned a top without a clear` and asks for nothing, because a position from before the migration can be a `Date`. ## Tests diff --git a/faucet/fakeredelivery_test.go b/faucet/fakeredelivery_test.go index 6acb7d6..1dbec1f 100644 --- a/faucet/fakeredelivery_test.go +++ b/faucet/fakeredelivery_test.go @@ -94,13 +94,12 @@ func (s *fakeRedeliveryStore) AcquireRedeliveryLease(_ context.Context, hook str scan := faucet.RedeliveryScan{} if s.epoch > 0 && s.hook != hook { scan.HookChanged = true - for _, r := range s.ranges { - if r.gap { - continue - } - r.gap, r.closedAt = true, now - from := r.OlderAt.Add(-faucet.RedeliveryReadOverlap) - scan.ClosedGaps = append(scan.ClosedGaps, faucet.RedeliveryGap{From: from, To: laterOf(r.NewerAt, r.OlderAt)}) + open := s.openRangesLocked() + s.ranges = slices.DeleteFunc(s.ranges, func(r *fakeRange) bool { return !r.gap }) + for _, r := range open { + gap := r.UnreadGap() + s.recordGapLocked(gap, now) + scan.ClosedGaps = append(scan.ClosedGaps, gap) } s.top, s.clearAt = time.Time{}, time.Time{} } @@ -209,15 +208,19 @@ func (s *fakeRedeliveryStore) ApplyRedeliveryScan(_ context.Context, step faucet s.ranges = slices.DeleteFunc(s.ranges, func(x *fakeRange) bool { return x == r }) } if step.Gap != nil { - s.nextRangeID++ - s.ranges = append(s.ranges, &fakeRange{ - RedeliveryRange: faucet.RedeliveryRange{ID: s.nextRangeID, NewerAt: step.Gap.To, OlderAt: step.Gap.From}, - gap: true, closedAt: step.Now, - }) + s.recordGapLocked(*step.Gap, step.Now) } return res, nil } +func (s *fakeRedeliveryStore) recordGapLocked(gap faucet.RedeliveryGap, now time.Time) { + s.nextRangeID++ + s.ranges = append(s.ranges, &fakeRange{ + RedeliveryRange: faucet.RedeliveryRange{ID: s.nextRangeID, NewerAt: gap.To, OlderAt: gap.From}, + gap: true, closedAt: now, + }) +} + func (s *fakeRedeliveryStore) openRangeLocked(id int64) *fakeRange { for _, r := range s.ranges { if r.ID == id && !r.gap { @@ -323,6 +326,20 @@ func TestFakeRedeliveryStoreMatchesContract(t *testing.T) { storetest.RunRedelivery(t, func(*testing.T) storetest.RedeliveryStore { return newFakeRedeliveryStore(nil) }) } +var _ storetest.GapRecords = (*fakeRedeliveryStore)(nil) + +func (s *fakeRedeliveryStore) RecordedRedeliveryGaps(context.Context) ([]faucet.RedeliveryGap, error) { + s.rmu.Lock() + defer s.rmu.Unlock() + var gaps []faucet.RedeliveryGap + for _, r := range s.ranges { + if r.gap { + gaps = append(gaps, faucet.RedeliveryGap{From: r.OlderAt, To: r.NewerAt}) + } + } + return gaps, nil +} + func (s *fakeRedeliveryStore) applyMigration000008() { s.rmu.Lock() defer s.rmu.Unlock() diff --git a/faucet/pgstore/database/redelivery.sql.go b/faucet/pgstore/database/redelivery.sql.go index cbb3572..fb8126d 100644 --- a/faucet/pgstore/database/redelivery.sql.go +++ b/faucet/pgstore/database/redelivery.sql.go @@ -81,27 +81,36 @@ func (q *Queries) ClaimRedelivery(ctx context.Context, arg ClaimRedeliveryParams return i, err } -const closeOpenRedeliveryRanges = `-- name: CloseOpenRedeliveryRanges :many -UPDATE faucet_redelivery_ranges -SET state = 'gap', closed_at = $1::timestamptz -WHERE state = 'open' +const countPendingRedeliveries = `-- name: CountPendingRedeliveries :one +SELECT COUNT(*)::bigint AS pending FROM faucet_redeliveries WHERE state = 'pending' +` + +func (q *Queries) CountPendingRedeliveries(ctx context.Context) (int64, error) { + row := q.db.QueryRowContext(ctx, countPendingRedeliveries) + var pending int64 + err := row.Scan(&pending) + return pending, err +} + +const deleteOpenRedeliveryRanges = `-- name: DeleteOpenRedeliveryRanges :many +DELETE FROM faucet_redelivery_ranges WHERE state = 'open' RETURNING older_at, newer_at ` -type CloseOpenRedeliveryRangesRow struct { +type DeleteOpenRedeliveryRangesRow struct { OlderAt time.Time NewerAt time.Time } -func (q *Queries) CloseOpenRedeliveryRanges(ctx context.Context, now time.Time) ([]CloseOpenRedeliveryRangesRow, error) { - rows, err := q.db.QueryContext(ctx, closeOpenRedeliveryRanges, now) +func (q *Queries) DeleteOpenRedeliveryRanges(ctx context.Context) ([]DeleteOpenRedeliveryRangesRow, error) { + rows, err := q.db.QueryContext(ctx, deleteOpenRedeliveryRanges) if err != nil { return nil, err } defer rows.Close() - var items []CloseOpenRedeliveryRangesRow + var items []DeleteOpenRedeliveryRangesRow for rows.Next() { - var i CloseOpenRedeliveryRangesRow + var i DeleteOpenRedeliveryRangesRow if err := rows.Scan(&i.OlderAt, &i.NewerAt); err != nil { return nil, err } @@ -116,17 +125,6 @@ func (q *Queries) CloseOpenRedeliveryRanges(ctx context.Context, now time.Time) return items, nil } -const countPendingRedeliveries = `-- name: CountPendingRedeliveries :one -SELECT COUNT(*)::bigint AS pending FROM faucet_redeliveries WHERE state = 'pending' -` - -func (q *Queries) CountPendingRedeliveries(ctx context.Context) (int64, error) { - row := q.db.QueryRowContext(ctx, countPendingRedeliveries) - var pending int64 - err := row.Scan(&pending) - return pending, err -} - const ensureRedeliveryScan = `-- name: EnsureRedeliveryScan :exec INSERT INTO faucet_redelivery_scan (id, hook, epoch) VALUES (true, $1, 0) diff --git a/faucet/pgstore/pgstore_test.go b/faucet/pgstore/pgstore_test.go index b9295af..f321053 100644 --- a/faucet/pgstore/pgstore_test.go +++ b/faucet/pgstore/pgstore_test.go @@ -38,10 +38,39 @@ func TestRedeliveryStore(t *testing.T) { adminURL := testDatabaseURL(t) storetest.RunRedelivery(t, func(t *testing.T) storetest.RedeliveryStore { - return pgstore.New(newDatabase(t, adminURL)) + return newGapRecordingStore(newDatabase(t, adminURL)) }) } +// gapRecordingStore lets storetest read the gaps that pgstore keeps. +type gapRecordingStore struct { + *pgstore.Store + db *sql.DB +} + +var _ storetest.GapRecords = gapRecordingStore{} + +func newGapRecordingStore(db *sql.DB) gapRecordingStore { + return gapRecordingStore{Store: pgstore.New(db), db: db} +} + +func (s gapRecordingStore) RecordedRedeliveryGaps(ctx context.Context) ([]faucet.RedeliveryGap, error) { + rows, err := s.db.QueryContext(ctx, `SELECT older_at, newer_at FROM faucet_redelivery_ranges WHERE state = 'gap'`) + if err != nil { + return nil, err + } + defer func() { _ = rows.Close() }() + var gaps []faucet.RedeliveryGap + for rows.Next() { + var gap faucet.RedeliveryGap + if err := rows.Scan(&gap.From, &gap.To); err != nil { + return nil, err + } + gaps = append(gaps, gap) + } + return gaps, rows.Err() +} + // A transaction-mode pooler such as Neon's pooled URL breaks lib/pq's default // parse-then-bind pair of batches, and binary_parameters=yes avoids it by // sending each query in one batch. That mode sends every []byte parameter in @@ -60,7 +89,7 @@ func TestRedeliveryStoreOverBinaryParameters(t *testing.T) { adminURL := withBinaryParameters(t, testDatabaseURL(t)) storetest.RunRedelivery(t, func(t *testing.T) storetest.RedeliveryStore { - return pgstore.New(newDatabase(t, adminURL)) + return newGapRecordingStore(newDatabase(t, adminURL)) }) } diff --git a/faucet/pgstore/queries/redelivery.sql b/faucet/pgstore/queries/redelivery.sql index 332fa1e..af6f136 100644 --- a/faucet/pgstore/queries/redelivery.sql +++ b/faucet/pgstore/queries/redelivery.sql @@ -6,10 +6,8 @@ ON CONFLICT (id) DO NOTHING; -- name: LockRedeliveryScan :one SELECT hook, epoch, lease_until, top_attempt_at FROM faucet_redelivery_scan FOR UPDATE; --- name: CloseOpenRedeliveryRanges :many -UPDATE faucet_redelivery_ranges -SET state = 'gap', closed_at = sqlc.arg(now)::timestamptz -WHERE state = 'open' +-- name: DeleteOpenRedeliveryRanges :many +DELETE FROM faucet_redelivery_ranges WHERE state = 'open' RETURNING older_at, newer_at; -- name: TakeRedeliveryLease :one diff --git a/faucet/pgstore/redelivery.go b/faucet/pgstore/redelivery.go index 846039a..5cafe05 100644 --- a/faucet/pgstore/redelivery.go +++ b/faucet/pgstore/redelivery.go @@ -42,13 +42,16 @@ func (s *Store) AcquireRedeliveryLease(ctx context.Context, hook string, now tim var scan faucet.RedeliveryScan scan.HookChanged = row.Epoch > 0 && row.Hook != hook if scan.HookChanged { - closed, err := q.CloseOpenRedeliveryRanges(ctx, now) + closed, err := q.DeleteOpenRedeliveryRanges(ctx) if err != nil { return faucet.RedeliveryScan{}, false, xerrors.Errorf("close ranges of the old hook: %w", err) } for _, r := range closed { - from := r.OlderAt.Add(-faucet.RedeliveryReadOverlap) - scan.ClosedGaps = append(scan.ClosedGaps, faucet.RedeliveryGap{From: from.UTC(), To: laterOf(r.NewerAt, r.OlderAt).UTC()}) + gap := faucet.RedeliveryRange{NewerAt: r.NewerAt.UTC(), OlderAt: r.OlderAt.UTC()}.UnreadGap() + if err := q.InsertRedeliveryGap(ctx, database.InsertRedeliveryGapParams{FromAt: gap.From, ToAt: gap.To, Now: now}); err != nil { + return faucet.RedeliveryScan{}, false, xerrors.Errorf("record gap of the old hook: %w", err) + } + scan.ClosedGaps = append(scan.ClosedGaps, gap) } } taken, err := q.TakeRedeliveryLease(ctx, database.TakeRedeliveryLeaseParams{Hook: hook, LeaseUntil: now.Add(ttl), HookChanged: scan.HookChanged}) @@ -75,13 +78,6 @@ func (s *Store) AcquireRedeliveryLease(ctx context.Context, hook string, now tim return scan, true, nil } -func laterOf(a, b time.Time) time.Time { - if a.After(b) { - return a - } - return b -} - func (s *Store) ReleaseRedeliveryLease(ctx context.Context, epoch int64) (err error) { defer func() { err = endedBy(ctx, err) }() if err := s.q.ReleaseRedeliveryLease(ctx, epoch); err != nil { diff --git a/faucet/redeliver.go b/faucet/redeliver.go index 59b285b..c8f778b 100644 --- a/faucet/redeliver.go +++ b/faucet/redeliver.go @@ -462,7 +462,7 @@ func (r *redeliveryRun) closeExpired(ctx context.Context, ranges []RedeliveryRan live = append(live, rg) continue } - gap := RedeliveryGap{From: rg.bound().stop, To: laterOf(rg.NewerAt, rg.OlderAt)} + gap := rg.UnreadGap() if _, err := r.apply(ctx, ScanStep{Epoch: r.epoch, Now: r.svc.clock.Now(), CloseID: rg.ID, Gap: &gap}, gapExpiredUnread); err != nil { return nil, err } diff --git a/faucet/redeliverystore.go b/faucet/redeliverystore.go index ce0ab5f..313ab2f 100644 --- a/faucet/redeliverystore.go +++ b/faucet/redeliverystore.go @@ -72,6 +72,14 @@ type RedeliveryGap struct { From, To time.Time } +// UnreadGap is the gap that the range leaves when it closes before a run has +// read it to its end. It starts where a read of the range would have stopped, +// RedeliveryReadOverlap below OlderAt, and ends at NewerAt, or at OlderAt when +// NewerAt lies inside the overlap. +func (r RedeliveryRange) UnreadGap() RedeliveryGap { + return RedeliveryGap{From: r.bound().stop, To: laterOf(r.NewerAt, r.OlderAt)} +} + // RedeliveryScan is what a run knows when it takes the lease. type RedeliveryScan struct { // Epoch fences the run's writes. A later lease has a higher epoch. @@ -91,8 +99,8 @@ type RedeliveryScan struct { Ranges []RedeliveryRange // HookChanged is true when the lease found state of another webhook. The // store has reset Top and Clear and closed that state's open ranges as - // ClosedGaps. Each gap starts RedeliveryReadOverlap below the range's - // OlderAt, where the read of the range would have stopped. + // ClosedGaps: each one is the range's UnreadGap, and the store keeps it + // as it reports it. HookChanged bool ClosedGaps []RedeliveryGap } diff --git a/faucet/storetest/redelivery.go b/faucet/storetest/redelivery.go index 84d9801..c5f2c91 100644 --- a/faucet/storetest/redelivery.go +++ b/faucet/storetest/redelivery.go @@ -18,6 +18,16 @@ type RedeliveryStore interface { faucet.RedeliveryStore } +// GapRecords is what a RedeliveryStore can also implement so that +// RunRedelivery checks the gaps that it keeps as final states: each one as a +// step gave it, or as the store reported it when the hook changed. Without it +// the suite checks only the gaps that the store reports. +type GapRecords interface { + // RecordedRedeliveryGaps returns every gap that the store keeps, in any + // order. + RecordedRedeliveryGaps(ctx context.Context) ([]faucet.RedeliveryGap, error) +} + // RunRedelivery runs the redelivery contract against stores returned by // newStore. Every call must return an empty store that does not share state // with the others, because the subtests run in parallel. @@ -341,6 +351,7 @@ func scanPositionSurvivesBetweenRuns(t *testing.T, s RedeliveryStore) { apply(t, s, faucet.ScanStep{Epoch: third.Epoch, Now: t0.Add(5 * time.Minute), CloseID: opened.OpenedID, Gap: &faucet.RedeliveryGap{From: gapFrom, To: gapTo}}) apply(t, s, faucet.ScanStep{Epoch: third.Epoch, Now: t0.Add(5 * time.Minute), Gap: &faucet.RedeliveryGap{From: gapFrom.Add(-time.Hour), To: gapFrom}}) require.NoError(t, s.ReleaseRedeliveryLease(ctx, third.Epoch)) + requireRecordedGaps(t, s, []faucet.RedeliveryGap{{From: gapFrom, To: gapTo}, {From: gapFrom.Add(-time.Hour), To: gapFrom}}, "a store keeps each gap as its step gave it") last := acquire(t, s, testHook, t0.Add(10*time.Minute)) require.Empty(t, last.Ranges, "a closed range, finished or a gap, is no longer open") @@ -369,6 +380,25 @@ func theClearIsTheLatestTimeAStepSetNotTheHighest(t *testing.T, s RedeliveryStor require.True(t, t0.Add(-time.Minute).Equal(last.Top), "a clear never moves the top") } +func requireRecordedGaps(t *testing.T, s RedeliveryStore, want []faucet.RedeliveryGap, msg string) { + t.Helper() + records, ok := s.(GapRecords) + if !ok { + return + } + got, err := records.RecordedRedeliveryGaps(context.Background()) + require.NoError(t, err) + require.ElementsMatch(t, normalizeGaps(want), normalizeGaps(got), msg) +} + +func normalizeGaps(gaps []faucet.RedeliveryGap) []faucet.RedeliveryGap { + out := make([]faucet.RedeliveryGap, len(gaps)) + for i, g := range gaps { + out[i] = faucet.RedeliveryGap{From: g.From.UTC(), To: g.To.UTC()} + } + return out +} + func normalizeRanges(ranges []faucet.RedeliveryRange) []faucet.RedeliveryRange { out := make([]faucet.RedeliveryRange, len(ranges)) for i, r := range ranges { @@ -396,16 +426,16 @@ func aFailedScanStepRecordsNothing(t *testing.T, s RedeliveryStore) { func changingTheHookClosesOpenRangesAsGaps(t *testing.T, s RedeliveryStore) { ctx := context.Background() + long := faucet.RedeliveryRange{NewerAt: t0.Add(-time.Hour), OlderAt: t0.Add(-48 * time.Hour), Cursor: "cursor-1"} + insideTheOverlap := faucet.RedeliveryRange{NewerAt: t0.Add(-50*time.Hour - 2*time.Minute), OlderAt: t0.Add(-50 * time.Hour), Cursor: "cursor-2"} lease := acquire(t, s, testHook, t0) - apply(t, s, faucet.ScanStep{ - Epoch: lease.Epoch, Now: t0, Top: t0, Clear: t0.Add(time.Second), - Open: &faucet.RedeliveryRange{NewerAt: t0.Add(-time.Hour), OlderAt: t0.Add(-48 * time.Hour), Cursor: "cursor-1"}, - }) + apply(t, s, faucet.ScanStep{Epoch: lease.Epoch, Now: t0, Top: t0, Clear: t0.Add(time.Second), Open: &long}) + apply(t, s, faucet.ScanStep{Epoch: lease.Epoch, Now: t0, Open: &insideTheOverlap}) require.NoError(t, s.ReleaseRedeliveryLease(ctx, lease.Epoch)) same := acquire(t, s, testHook, t0.Add(time.Minute)) require.False(t, same.HookChanged) - require.Len(t, same.Ranges, 1) + require.Len(t, same.Ranges, 2) require.False(t, same.Clear.IsZero()) require.NoError(t, s.ReleaseRedeliveryLease(ctx, same.Epoch)) @@ -414,10 +444,11 @@ func changingTheHookClosesOpenRangesAsGaps(t *testing.T, s RedeliveryStore) { require.True(t, other.Top.IsZero(), "a new hook starts from scratch") require.True(t, other.Clear.IsZero(), "the clear is a time of the old hook's list, and the new hook has not been read: %s", other.Clear) require.Empty(t, other.Ranges) - require.Len(t, other.ClosedGaps, 1) - require.True(t, t0.Add(-48*time.Hour-faucet.RedeliveryReadOverlap).Equal(other.ClosedGaps[0].From), - "the gap starts where the read of the range would have stopped, the overlap below its old end: %s", other.ClosedGaps[0].From) - require.True(t, t0.Add(-time.Hour).Equal(other.ClosedGaps[0].To)) + require.ElementsMatch(t, []faucet.RedeliveryGap{ + {From: long.OlderAt.Add(-faucet.RedeliveryReadOverlap), To: long.NewerAt}, + {From: insideTheOverlap.OlderAt.Add(-faucet.RedeliveryReadOverlap), To: insideTheOverlap.OlderAt}, + }, normalizeGaps(other.ClosedGaps), "each gap starts where the read of its range would have stopped, the overlap below its old end") + requireRecordedGaps(t, s, other.ClosedGaps, "a store keeps each gap of a changed hook as it reports it") require.NoError(t, s.ReleaseRedeliveryLease(ctx, other.Epoch)) again := acquire(t, s, "/orgs/acme/hooks/8", t0.Add(3*time.Minute)) From 551f69bd2f65b0697ebd56927d5f4b71dc5199db Mon Sep 17 00:00:00 2001 From: Thomas Kosiewski Date: Mon, 16 Nov 2026 13:46:42 +0000 Subject: [PATCH 14/18] fix(faucet): end the gap of an unread range at the oldest attempt read above it Since #36 the gap of a range that closes unread starts at OlderAt - O, but it still ended at max(NewerAt, OlderAt). A range whose NewerAt lay inside the overlap below OlderAt then logged a gap up to 3 minutes above the oldest attempt that a run had read. UnreadGap now ends at max(NewerAt, OlderAt - O), rule 8's [OlderAt - O, NewerAt], for the expired ranges and the ranges of a changed hook. The #36 test "the range expires unread" closes its range by reading it, because its one run after the outage still has the Date from before it. A new test closes a range in the run after that, without a page for it, and checks both ends of its gap. --- faucet/redeliver_gaps_test.go | 28 ++++++++++++++++++++++++++++ faucet/redeliverystore.go | 7 ++++--- faucet/storetest/redelivery.go | 4 ++-- 3 files changed, 34 insertions(+), 5 deletions(-) diff --git a/faucet/redeliver_gaps_test.go b/faucet/redeliver_gaps_test.go index afdff3a..6bd0d74 100644 --- a/faucet/redeliver_gaps_test.go +++ b/faucet/redeliver_gaps_test.go @@ -330,6 +330,34 @@ func TestALateFailureEndsInAGapWhenGitHubDeletesItBeforeARunReadsTheRangeAboveIt } } +func TestARangeThatExpiresUnreadEndsAsAGapFromTheOverlapBelowItsOldEndToTheOldestAttemptReadAboveIt(t *testing.T) { + t.Parallel() + env := newRedeliveryEnv(t, func(o *envOptions) { o.maxPages, o.pageSize = 1, 3 }) + env.clock.WithLogger(quartz.NoOpLogger) + t0 := env.gh.now() + top := env.gh.record("A", t0, http200) + env.run() + + unread := env.gh.record("unread", top.at.Add(-4*time.Minute), 503) + oldestRead := env.gh.record("read", top.at.Add(-3*time.Minute), 503) + env.gh.record("new", top.at.Add(time.Second), http200) + env.advance(cronEvery) + require.Equal(t, 1, env.run().Ranges, "the one page ends at an attempt inside the overlap, above a late one") + + env.advance(faucet.GitHubDeliveryRetention + time.Hour) + require.False(t, env.gh.lists(unread.guid)) + require.Equal(t, 1, env.run().Ranges, "the stored Date is from before the outage, so it closes nothing yet") + env.advance(cronEvery) + before := len(env.loggedGaps()) + res := env.run() + + require.Zero(t, res.Ranges) + require.Equal(t, 1, res.Pages, "the head read took the only page, so no read of the range closed it") + require.Equal(t, []gapWindow{{from: top.at.Add(-faucet.RedeliveryReadOverlap), to: oldestRead.at}}, env.loggedGaps()[before:], + "the gap starts where a read of the range would have stopped and ends at the oldest attempt read above it") + requireNoSilentLoss(t, env, []ghAttempt{unread}) +} + const ( newestAttemptAge = time.Minute dateLagPastTheSlack = newestAttemptAge + faucet.RedeliveryDateSlack + time.Minute diff --git a/faucet/redeliverystore.go b/faucet/redeliverystore.go index 313ab2f..d2135f8 100644 --- a/faucet/redeliverystore.go +++ b/faucet/redeliverystore.go @@ -74,10 +74,11 @@ type RedeliveryGap struct { // UnreadGap is the gap that the range leaves when it closes before a run has // read it to its end. It starts where a read of the range would have stopped, -// RedeliveryReadOverlap below OlderAt, and ends at NewerAt, or at OlderAt when -// NewerAt lies inside the overlap. +// RedeliveryReadOverlap below OlderAt, and ends at NewerAt, the oldest attempt +// read above it, or at its start when NewerAt is lower. func (r RedeliveryRange) UnreadGap() RedeliveryGap { - return RedeliveryGap{From: r.bound().stop, To: laterOf(r.NewerAt, r.OlderAt)} + from := r.bound().stop + return RedeliveryGap{From: from, To: laterOf(r.NewerAt, from)} } // RedeliveryScan is what a run knows when it takes the lease. diff --git a/faucet/storetest/redelivery.go b/faucet/storetest/redelivery.go index c5f2c91..e863e1f 100644 --- a/faucet/storetest/redelivery.go +++ b/faucet/storetest/redelivery.go @@ -446,8 +446,8 @@ func changingTheHookClosesOpenRangesAsGaps(t *testing.T, s RedeliveryStore) { require.Empty(t, other.Ranges) require.ElementsMatch(t, []faucet.RedeliveryGap{ {From: long.OlderAt.Add(-faucet.RedeliveryReadOverlap), To: long.NewerAt}, - {From: insideTheOverlap.OlderAt.Add(-faucet.RedeliveryReadOverlap), To: insideTheOverlap.OlderAt}, - }, normalizeGaps(other.ClosedGaps), "each gap starts where the read of its range would have stopped, the overlap below its old end") + {From: insideTheOverlap.OlderAt.Add(-faucet.RedeliveryReadOverlap), To: insideTheOverlap.NewerAt}, + }, normalizeGaps(other.ClosedGaps), "each gap starts where the read of its range would have stopped, the overlap below its old end, and ends at the oldest attempt read above it") requireRecordedGaps(t, s, other.ClosedGaps, "a store keeps each gap of a changed hook as it reports it") require.NoError(t, s.ReleaseRedeliveryLease(ctx, other.Epoch)) From cbf2046fd6c4f8ed5eb9bdf630695a954d4cfd1f Mon Sep 17 00:00:00 2001 From: Thomas Kosiewski Date: Mon, 16 Nov 2026 13:49:51 +0000 Subject: [PATCH 15/18] docs(faucet): state when an empty list, a gap, the migration and its rollback do what the README says Four README claims did not hold: - "A run that finds the list empty records no gap": it records one when the last run that read the list was about 64h55m before it, or more. - "a gap can hold up to an hour in which GitHub deleted nothing": it can reach past the end of GitHub's retention by the slack, a second and the lead of the Date, up to about 7 hours inside the trust window. - the migration "loses no failure silently": the first limit, a Date far ahead and then an outage, applies through it. - the rollback "gets its old position back": only when no run has read the head of the list since the migration. After runs that read only an empty list, the old build gets the Date of the last of them. The README now says each, condition first. A test pins how far a gap reaches past GitHub's retention, the migration test checks the position that the rollback leaves after an empty list, and the README test pins the two new numbers to the constants. Item 3 gets a placeholder for its tracking issue. --- README.md | 10 +++++----- faucet/pgstore/pgstore_test.go | 20 +++++++++++++++----- faucet/readme_test.go | 2 ++ faucet/redeliver_gaps_test.go | 21 +++++++++++++++++++++ 4 files changed, 43 insertions(+), 10 deletions(-) diff --git a/README.md b/README.md index 7034d17..3056ffa 100644 --- a/README.md +++ b/README.md @@ -148,9 +148,9 @@ The token, the ID and one of org or repo go together. With none set, `/cron/rede - `recovered`: faucet stored the event, or a later attempt succeeded. - `gave_up` with `limit`: every ask was spent. A delivery is asked for 7 times, at 0, 10, 30, 70, 150, 310 and 630 minutes after the run that first read it. The count is raised before the request is sent, so a crash or a refused request costs an ask, and two runs cannot send the same one. - `gave_up` with `expired`: GitHub keeps a delivery for 72 hours, counted from its first attempt (the earliest attempt that a run read), and the delivery was still not stored. A delivery of a webhook that you replaced ends here too: an attempt ID of one webhook means nothing to another, so a run asks only for the deliveries of the webhook it was configured with. Those deliveries stay pending until faucet stores the event or 72 hours pass. -- `gap`: a time range of attempts that no run read. It names no delivery, because the attempts were never read. A range whose old end left GitHub's retention before a run read it is a gap, together with the 5 minute overlap below that end, where an attempt that GitHub wrote late is listed. So is the part of GitHub's 72 hours before the first run's lookback of 24 hours, both counted from the newest attempt that the first run reads, and so are the open ranges of a webhook that the operator replaced. A gap never reaches above the oldest attempt that a run read above it. Below that attempt it can reach up to the slack past the end of GitHub's retention, so a gap can hold up to an hour in which GitHub deleted nothing. After a silence a gap starts 6 hours and 5 minutes before the `Date` of the run before it, so a hook that delivers nothing logs a gap only when two runs are about 64 hours and 55 minutes apart, or more. +- `gap`: a time range of attempts that no run read. It names no delivery, because the attempts were never read. A range whose old end left GitHub's retention before a run read it is a gap, together with the 5 minute overlap below that end, where an attempt that GitHub wrote late is listed. So is the part of GitHub's 72 hours before the first run's lookback of 24 hours, both counted from the newest attempt that the first run reads, and so are the open ranges of a webhook that the operator replaced. A gap never reaches above the oldest attempt that a run read above it. Below that attempt it can reach past the end of GitHub's retention by the slack, plus a second, plus as far as the `Date` runs ahead of GitHub's list. When the `Date` is right, a gap can hold just over an hour in which GitHub deleted nothing, and when it runs the margin of 6 hours ahead, up to about 7 hours. After a silence a gap starts 6 hours and 5 minutes before the `Date` of the run before it, so a hook that delivers nothing logs a gap only when two runs are about 64 hours and 55 minutes apart, or more. -The reply is JSON: `busy`, `pages`, `attempts`, `tracked`, `asked`, `recovered`, `gave_up` (`limit` and `expired`), `gaps`, `pending` and `ranges`. A run that fails answers 500 with the same JSON and an `error` member, so the outcomes that it recorded before it failed are in a reply as well as in the log. A run that finds the list empty records no gap. The first run that reads an attempt says `gaps: 1`, for the time before its lookback. Faucet deletes a final delivery and a recorded gap 72 hours after its last attempt, so a give-up is logged once. A pending delivery and an open range stay until they end, and the scan position is one row that stays. +The reply is JSON: `busy`, `pages`, `attempts`, `tracked`, `asked`, `recovered`, `gave_up` (`limit` and `expired`), `gaps`, `pending` and `ranges`. A run that fails answers 500 with the same JSON and an `error` member, so the outcomes that it recorded before it failed are in a reply as well as in the log. A run that finds the list empty records a gap only when the last run that read the list was about 64 hours and 55 minutes before it, or more. The first run that reads an attempt says `gaps: 1`, for the time before its lookback. Faucet deletes a final delivery and a recorded gap 72 hours after its last attempt, so a give-up is logged once. A pending delivery and an open range stay until they end, and the scan position is one row that stays. **Capacity.** A run reads at most 20 pages of 100 attempts, which is 2,000 attempts, and sends at most 50 requests. A run every 10 minutes reads 15 minutes of the list: 10 minutes of new attempts and the 5 minute overlap. It also needs one attempt beyond them, to learn that it has reached the end of that window, and it starts from the newest attempt of the run before it, so a run holds at most 1,998 attempts of its window. That is 7,992 attempts an hour at best, and the sweep keeps up with 7,900 attempts an hour, rounded down for a run that starts late, and sends at most 300 requests an hour. These numbers hold only when GitHub answers each page of the list within 0.6 seconds, because a run reads the list for half of its 25 seconds (see Time). A slower API reads fewer pages: at 1.3 seconds a page a run reads 9 pages, and the sweep keeps up with about 3,500 attempts an hour. Faucet has no measurement of GitHub's page time, so size a hook with that margin. The list counts every attempt of the hook: attempts of events that faucet does not store, and the redeliveries that faucet asked for. A hook that records more than that does not lose anything silently: the part of the list that a run could not read stays a range. Later runs read the ranges, newest first, with the pages that the newest attempts leave them, and a range that no run read before its old end left GitHub's retention is counted and logged as a gap. A failure of a request, GitHub's rate limit or a slow API makes a run read fewer pages. The simulations in `faucet/redeliver_sim_test.go` run 4,200 attempts an hour with one failure in 100, and outages of up to an hour at 150, 300 and 600 attempts an hour, and no failure is lost. They also check the law above at a tenth of the scale: the sweep holds at its stated capacity, and ranges grow when the load is a tenth above it. An outage of faucet that ends within 630 minutes of the first run that reads a failure is recovered by one of its asks. @@ -160,7 +160,7 @@ The reply is JSON: `busy`, `pages`, `attempts`, `tracked`, `asked`, `recovered`, - When a `Date` runs more than 6 hours and 5 minutes ahead of GitHub's list (the margin and the overlap), and the cron then stops for 72 hours or more, the gap after the outage starts later than the last run before the outage. The stretch between the two is as long as the `Date` was ahead, less 6 hours and 5 minutes. The failures that GitHub listed in that stretch and then deleted are neither read nor counted. A position that a build before migration `000008_redelivery_clear` took from such a `Date` carries the same risk through the migration. - When a `Date` is more than 1 hour behind the clock that deletes GitHub's attempts, and the hook is so quiet that no listed attempt shows the lag, a gap after a read to the end of the list ends too early. The failures that GitHub deleted in the slice above the gap are not counted. On a hook with recent attempts the run sees the lag and fails instead. -- A dense burst followed by silence leaves ranges unread. When a burst holds more attempts in the 5 minute overlap than a run has pages, every run reads the same newest pages again and opens one more range, and the older ranges get no page. Their failures are not recovered. A range that no run read before its old end left GitHub's retention is counted and logged as a gap, so each failure in it ends inside a gap. This limit does not come from a clock, and it is tracked separately from the two above. +- A dense burst followed by silence leaves ranges unread. When a burst holds more attempts in the 5 minute overlap than a run has pages, every run reads the same newest pages again and opens one more range, and the older ranges get no page. Their failures are not recovered. A range that no run read before its old end left GitHub's retention is counted and logged as a gap, so each failure in it ends inside a gap. This limit does not come from a clock, and it is tracked separately from the two above. Tracking issue: TODO-ITEM-3-ISSUE. To embed faucet, set `Deps.Redelivery` to a `RedeliveryDeps`: a `RedeliveryStore` (`pgstore.Store` is one, and `storetest.RunRedelivery` checks yours), and a `HookAdmin`. `NewGitHubHookAdmin` is the one for GitHub's REST API. A `HookAdmin` of your own must put GitHub's `Date` header in `DeliveryPage.At`, because a run whose first page has no `Date` fails. Call `Service.Redeliver(ctx)` on your own schedule. It has no auth. The host decides who may call it. @@ -189,7 +189,7 @@ To embed faucet, set `Deps.Redelivery` to a `RedeliveryDeps`: a `RedeliveryStore The server applies pending migrations when a production instance starts, over `DATABASE_URL_UNPOOLED` (or over `DATABASE_URL` when that is unset), before it serves a request. -**Do not roll back past the newest migration.** After production has migrated to `000008_redelivery_clear`, a build that does not embed it, which is what Vercel Instant Rollback to an older deployment gives, cannot start: the migration library fails with `no migration found for version 8` and every new instance exits, so the rollback serves nothing. Recover in one of two ways. Redeploy a build that embeds the migration. Or roll the database back with the down migration `faucet/pgstore/migrations/000008_redelivery_clear.down.sql`, applied through golang-migrate against the `faucet_schema_migrations` table so that the recorded version drops to 7. The down migration gives the position column its old name and drops the `clear_at` column. A redelivery scan that has read no attempt since the migration gets its old position back. The same holds for each later migration, and for the one before: the down migration of `000007_gap_pending` drops the `gap_through` and `gap_lease_until` columns, and with them every gap that is owed and not yet sent. +**Do not roll back past the newest migration.** After production has migrated to `000008_redelivery_clear`, a build that does not embed it, which is what Vercel Instant Rollback to an older deployment gives, cannot start: the migration library fails with `no migration found for version 8` and every new instance exits, so the rollback serves nothing. Recover in one of two ways. Redeploy a build that embeds the migration. Or roll the database back with the down migration `faucet/pgstore/migrations/000008_redelivery_clear.down.sql`, applied through golang-migrate against the `faucet_schema_migrations` table so that the recorded version drops to 7. The down migration gives the position column its old name and drops the `clear_at` column. The position that it leaves depends on the runs since the migration. When no run has read the head of the list, the scan gets its old position back. When runs read attempts, the position is the newest of them. When runs read only an empty list, the position is the `Date` of the last of them: a `Date` can run ahead of GitHub's list, so the old build can then put a new failure below its position, as it could before the migration. The same holds for each later migration, and for the one before: the down migration of `000007_gap_pending` drops the `gap_through` and `gap_lease_until` columns, and with them every gap that is owed and not yet sent. ## Upgrading @@ -198,7 +198,7 @@ The server applies pending migrations when a production instance starts, over `D - **Rows sealed before secret rotation existed.** The first refresh of such a row with the same secret counts as one rotation, because the new sealing of the secret is not the old bytes. For up to 10 minutes the old sealing is the previous secret. It opens to the same secret, so a signature has one `v1,` entry. - **A `CRON_SECRET` that no request can carry.** The server now stops at start, and names the variable, when `CRON_SECRET` has a space, a tab, a newline or another control character, a byte outside printable ASCII, or more than 1024 bytes. Before, it started and answered 401 to every cron call. A value from `openssl rand -hex 32` or `openssl rand -base64 32` passes, and so does any other run of printable ASCII characters. A secret that you read from a file can end in a newline that you must remove. - **A `Store` of your own.** The interface has gained methods, and `storetest` checks each one: `DeleteLiveSubscription`, `ClearExpiredPreviousSecrets`, a `Purge` that also moves the subscriptions that lost nothing past the purge mark, `ClaimGap` and `FinishGap`. `ClaimDelivery` takes a `skip` list of callback origins. `Enqueue` has `GapThrough`, `DeliveryRef` and `GapRef` have `Incarnation`, `ScanTarget` has `Principal` and no callback or secret fields, and `PurgeResult` has two counts. A `Store` without them does not compile. A `Store` that compiles can still fail `storetest`. `ClaimDelivery` and `ClaimGap` must skip a callback by `faucet.CallbackOrigin` and by no other rule of your own, so a stored URL in any spelling (a host in capitals, a scheme in capitals, user info, an IPv6 literal) is held to the cap of its origin. `storetest` checks this since PR 27. A `RedeliveryStore` is a second interface, `faucet.RedeliveryStore`, with its own suite, `storetest.RunRedelivery`. -- **The redelivery position after migration `000008_redelivery_clear`.** Before this migration the position of a run could be GitHub's `Date` header, and no row says whether it is. The migration moves the old position into the new `clear_at` column, resets the position and gives its column a new name. Until a deploy has replaced every instance of the build before the migration, the redelivery runs of those instances fail with `column "top_at" does not exist` and write no position, and so does a run that was in flight during the migration. Their other routes keep working, and the runs of the new build read the list in their place. The first run after the migration is therefore a first run. It knows when tracking began, so it loses no failure silently. The cost comes once. The runs after the migration read the last 24 hours of the list again, and the first of them logs one more `before tracking began` gap for the time before those 24 hours. A delivery that a run tracked before the migration keeps its row and its ask count, so reading its attempts again changes nothing. On a hook near the capacity, the sweep cannot read that day again before GitHub deletes it, and the unread part ends as `expired unread` gaps. A `RedeliveryStore` of your own must keep `Clear` (`RedeliveryScan.Clear` and `ScanStep.Clear`), reset it with the top when the hook changes, and close each open range of a changed hook as the gap that `RedeliveryRange.UnreadGap` gives, which it reports and keeps. `storetest.RunRedelivery` checks all three, and checks the gaps that the store keeps when it implements `storetest.GapRecords`. It must also do once what the migration does: move the position that it holds into `Clear`, and reset the position. Until then every run fails with `the store returned a top without a clear` and asks for nothing, because a position from before the migration can be a `Date`. +- **The redelivery position after migration `000008_redelivery_clear`.** Before this migration the position of a run could be GitHub's `Date` header, and no row says whether it is. The migration moves the old position into the new `clear_at` column, resets the position and gives its column a new name. Until a deploy has replaced every instance of the build before the migration, the redelivery runs of those instances fail with `column "top_at" does not exist` and write no position, and so does a run that was in flight during the migration. Their other routes keep working, and the runs of the new build read the list in their place. The first run after the migration is therefore a first run, and it knows when tracking began. When the old position came from a `Date` more than 6 hours and 5 minutes ahead of GitHub's list and the cron then stops for 72 hours or more, failures can end with no final state, as the first limit in **Redeliver failed webhooks** says. Apart from that limit, the migration loses no failure silently. The cost comes once. The runs after the migration read the last 24 hours of the list again, and the first of them logs one more `before tracking began` gap for the time before those 24 hours. A delivery that a run tracked before the migration keeps its row and its ask count, so reading its attempts again changes nothing. On a hook near the capacity, the sweep cannot read that day again before GitHub deletes it, and the unread part ends as `expired unread` gaps. A `RedeliveryStore` of your own must keep `Clear` (`RedeliveryScan.Clear` and `ScanStep.Clear`), reset it with the top when the hook changes, and close each open range of a changed hook as the gap that `RedeliveryRange.UnreadGap` gives, which it reports and keeps. `storetest.RunRedelivery` checks all three, and checks the gaps that the store keeps when it implements `storetest.GapRecords`. It must also do once what the migration does: move the position that it holds into `Clear`, and reset the position. Until then every run fails with `the store returned a top without a clear` and asks for nothing, because a position from before the migration can be a `Date`. ## Tests diff --git a/faucet/pgstore/pgstore_test.go b/faucet/pgstore/pgstore_test.go index f321053..0511da2 100644 --- a/faucet/pgstore/pgstore_test.go +++ b/faucet/pgstore/pgstore_test.go @@ -194,16 +194,26 @@ func TestTheClearMigrationMovesTheOldTopToTheClearAndBack(t *testing.T) { require.Len(t, migrated.Ranges, 1, "an open range stays open") migrate("down") - require.True(t, oldTop.Equal(storedTop().Time), "a scan that read nothing since the migration gets its position back") + require.True(t, oldTop.Equal(storedTop().Time), "a scan whose head no run has read since the migration gets its position back") migrate("up") - attempt, date := now.Add(time.Hour), now.Add(time.Hour+time.Second) - _, err = store.ApplyRedeliveryScan(ctx, faucet.ScanStep{Epoch: migrated.Epoch, Now: now.Add(time.Hour), Top: attempt, Clear: date}) + emptyListDate := now.Add(30 * time.Minute) + _, err = store.ApplyRedeliveryScan(ctx, faucet.ScanStep{Epoch: migrated.Epoch, Now: now.Add(30 * time.Minute), Clear: emptyListDate}) require.ErrorIs(t, err, faucet.ErrRedeliveryLeaseLost, "the migration ends a lease that a run took before it") - again, ok, err := store.AcquireRedeliveryLease(ctx, hook, now.Add(time.Hour), time.Minute) + emptyList, ok, err := store.AcquireRedeliveryLease(ctx, hook, now.Add(30*time.Minute), time.Minute) + require.NoError(t, err) + require.True(t, ok) + _, err = store.ApplyRedeliveryScan(ctx, faucet.ScanStep{Epoch: emptyList.Epoch, Now: now.Add(30 * time.Minute), Clear: emptyListDate}) + require.NoError(t, err) + migrate("down") + require.True(t, emptyListDate.Equal(storedTop().Time), "a scan whose runs read only an empty list since the migration gets the Date of the last one") + + migrate("up") + read, ok, err := store.AcquireRedeliveryLease(ctx, hook, now.Add(time.Hour), time.Minute) require.NoError(t, err) require.True(t, ok) - _, err = store.ApplyRedeliveryScan(ctx, faucet.ScanStep{Epoch: again.Epoch, Now: now.Add(time.Hour), Top: attempt, Clear: date}) + attempt, date := now.Add(time.Hour), now.Add(time.Hour+time.Second) + _, err = store.ApplyRedeliveryScan(ctx, faucet.ScanStep{Epoch: read.Epoch, Now: now.Add(time.Hour), Top: attempt, Clear: date}) require.NoError(t, err) migrate("down") require.True(t, attempt.Equal(storedTop().Time), "a scan that has read an attempt keeps that attempt's time, not the Date") diff --git a/faucet/readme_test.go b/faucet/readme_test.go index cddb1e5..c5624af 100644 --- a/faucet/readme_test.go +++ b/faucet/readme_test.go @@ -78,6 +78,8 @@ func TestReadmeStatesTheDateWindowThatTheConstantsGive(t *testing.T) { fmt.Sprintf("two runs are about %s apart", quietFor), fmt.Sprintf("more than %s ahead of GitHub's list", marginAndOverlap), fmt.Sprintf("more than %d hour behind the clock that deletes", int(slack/time.Hour)), + fmt.Sprintf("when it runs the margin of %d hours ahead, up to about %d hours", int(margin/time.Hour), int((margin+slack)/time.Hour)), + fmt.Sprintf("the last run that read the list was about %s before it", quietFor), } { require.Contains(t, text, want) } diff --git a/faucet/redeliver_gaps_test.go b/faucet/redeliver_gaps_test.go index 6bd0d74..98d3395 100644 --- a/faucet/redeliver_gaps_test.go +++ b/faucet/redeliver_gaps_test.go @@ -1063,6 +1063,27 @@ func TestAQuietHookLogsNoGapInTenDays(t *testing.T) { } } +func TestAGapReachesPastGitHubsRetentionByTheSlackASecondAndAsFarAsTheDateRunsAhead(t *testing.T) { + t.Parallel() + + for _, lead := range []time.Duration{0, faucet.RedeliveryDateMargin} { + t.Run(lead.String(), func(t *testing.T) { + t.Parallel() + env := newRedeliveryEnv(t) + env.clock.WithLogger(quartz.NoOpLogger) + env.gh.record("A", env.gh.now().Add(-time.Minute), http200) + env.run() + env.advance(80 * time.Hour) + env.gh.date = dateOff(lead) + before := len(env.loggedGaps()) + require.Equal(t, 1, env.run().Gaps) + + retentionEnd := env.gh.now().Add(-faucet.GitHubDeliveryRetention) + require.Equal(t, faucet.RedeliveryDateSlack+time.Second+lead, env.loggedGaps()[before].to.Sub(retentionEnd)) + }) + } +} + func TestAQuietHookLogsAGapOnlyAfterAnOutageThatTheDateMarginsDoNotCover(t *testing.T) { t.Parallel() From 819ceb969be8f341ebdef60d55fdaefc3a245cca Mon Sep 17 00:00:00 2001 From: Thomas Kosiewski Date: Mon, 16 Nov 2026 13:53:52 +0000 Subject: [PATCH 16/18] docs(faucet): link the follow-up issues of the redelivery limits MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The burst limit moves to #37, and the stretch that a replaced hook leaves unread is #38. --- _Generated with [`xum`](https://github.com/coder/xum) • Model: `anthropic:claude-opus-5-5` • Thinking: `high`_ --- README.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 3056ffa..0973f3c 100644 --- a/README.md +++ b/README.md @@ -156,11 +156,12 @@ The reply is JSON: `busy`, `pages`, `attempts`, `tracked`, `asked`, `recovered`, **Time.** The cron route stops a run after 25 seconds. A run that has such a deadline spends it so that the asks and the purge always get their turn: it reads the list for half of the time, drops a page that is not back by then and leaves the rest as an open range (as at the page limit), settles, asks until 80 percent of the time has passed, and purges. A slow GitHub costs pages and never the asks. A first page that is not back in half of the time ends the run with an error, and it asks for nothing. -**Limits.** GitHub gives no upper bound on the clock that deletes its attempts, other than the `Date` header. So redelivery trusts a `Date` inside a window: up to 6 hours ahead of GitHub's list (the margin), and up to 1 hour behind the clock that deletes (the slack). Outside that window two risks remain, and each can end a failure with no final state. A third limit ends failures inside counted gaps. Tracking issue: https://github.com/coder/faucet/issues/26. +**Limits.** GitHub gives no upper bound on the clock that deletes its attempts, other than the `Date` header. So redelivery trusts a `Date` inside a window: up to 6 hours ahead of GitHub's list (the margin), and up to 1 hour behind the clock that deletes (the slack). Outside that window two risks remain, and each can end a failure with no final state. A third limit ends failures inside counted gaps, and a fourth comes from replacing the hook. Tracking issue: https://github.com/coder/faucet/issues/26. - When a `Date` runs more than 6 hours and 5 minutes ahead of GitHub's list (the margin and the overlap), and the cron then stops for 72 hours or more, the gap after the outage starts later than the last run before the outage. The stretch between the two is as long as the `Date` was ahead, less 6 hours and 5 minutes. The failures that GitHub listed in that stretch and then deleted are neither read nor counted. A position that a build before migration `000008_redelivery_clear` took from such a `Date` carries the same risk through the migration. - When a `Date` is more than 1 hour behind the clock that deletes GitHub's attempts, and the hook is so quiet that no listed attempt shows the lag, a gap after a read to the end of the list ends too early. The failures that GitHub deleted in the slice above the gap are not counted. On a hook with recent attempts the run sees the lag and fails instead. -- A dense burst followed by silence leaves ranges unread. When a burst holds more attempts in the 5 minute overlap than a run has pages, every run reads the same newest pages again and opens one more range, and the older ranges get no page. Their failures are not recovered. A range that no run read before its old end left GitHub's retention is counted and logged as a gap, so each failure in it ends inside a gap. This limit does not come from a clock, and it is tracked separately from the two above. Tracking issue: TODO-ITEM-3-ISSUE. +- A dense burst followed by silence leaves ranges unread. When a burst holds more attempts in the 5 minute overlap than a run has pages, every run reads the same newest pages again and opens one more range, and the older ranges get no page. Their failures are not recovered. A range that no run read before its old end left GitHub's retention is counted and logged as a gap, so each failure in it ends inside a gap. This limit does not come from a clock, and it is tracked separately from the two above. Tracking issue: https://github.com/coder/faucet/issues/37. +- When the operator replaces the hook, a failure that the old hook listed after the last run is neither read nor counted. The old hook's open ranges become gaps, but the stretch above the old position was never a range, and the new hook cannot read the old hook's list. Tracking issue: https://github.com/coder/faucet/issues/38. To embed faucet, set `Deps.Redelivery` to a `RedeliveryDeps`: a `RedeliveryStore` (`pgstore.Store` is one, and `storetest.RunRedelivery` checks yours), and a `HookAdmin`. `NewGitHubHookAdmin` is the one for GitHub's REST API. A `HookAdmin` of your own must put GitHub's `Date` header in `DeliveryPage.At`, because a run whose first page has no `Date` fails. Call `Service.Redeliver(ctx)` on your own schedule. It has no auth. The host decides who may call it. From f3c380ba50fc9d11178c80ffbf40fb4cc15ebb07 Mon Sep 17 00:00:00 2001 From: Thomas Kosiewski Date: Mon, 16 Nov 2026 13:55:28 +0000 Subject: [PATCH 17/18] style(faucet): drop narration comments from the round-4 redelivery fixes Remove the comments that the round-4 fixes added on the migration 000008 fence, the pre-migration statements and build in the migration test, and the pgstore gap reader. The keep-list doc comments on UnreadGap, RedeliveryScan.ClosedGaps and storetest.GapRecords stay. --- faucet/pgstore/migrations/000008_redelivery_clear.up.sql | 7 ++----- faucet/pgstore/pgstore_test.go | 1 - faucet/postgres_migration_test.go | 5 ----- 3 files changed, 2 insertions(+), 11 deletions(-) diff --git a/faucet/pgstore/migrations/000008_redelivery_clear.up.sql b/faucet/pgstore/migrations/000008_redelivery_clear.up.sql index 78381e3..009de7b 100644 --- a/faucet/pgstore/migrations/000008_redelivery_clear.up.sql +++ b/faucet/pgstore/migrations/000008_redelivery_clear.up.sql @@ -1,13 +1,10 @@ -- clear_at is the Date header of GitHub's answer to the last head read: when, -- on GitHub's clock, a run last saw the top of the list. A run takes its gap -- starts from it and never a read bound, because a Date can run ahead of --- GitHub's list. top_attempt_at keeps attempt times only. +-- GitHub's list. ALTER TABLE faucet_redelivery_scan ADD COLUMN clear_at timestamptz; -- Until now top_at could hold a Date that ran ahead of GitHub's list, and no --- row says whether it does (see #26). The position gets a new name, so a run --- of an older build fails when it locks this row and never writes a Date into --- the position again. The epoch ends the lease of a run that started before --- the migration. +-- row says whether it does (see #26). UPDATE faucet_redelivery_scan SET clear_at = top_at, top_at = NULL, epoch = epoch + 1, lease_until = NULL; ALTER TABLE faucet_redelivery_scan RENAME COLUMN top_at TO top_attempt_at; diff --git a/faucet/pgstore/pgstore_test.go b/faucet/pgstore/pgstore_test.go index 0511da2..d5250b4 100644 --- a/faucet/pgstore/pgstore_test.go +++ b/faucet/pgstore/pgstore_test.go @@ -42,7 +42,6 @@ func TestRedeliveryStore(t *testing.T) { }) } -// gapRecordingStore lets storetest read the gaps that pgstore keeps. type gapRecordingStore struct { *pgstore.Store db *sql.DB diff --git a/faucet/postgres_migration_test.go b/faucet/postgres_migration_test.go index 009dd22..03c37a2 100644 --- a/faucet/postgres_migration_test.go +++ b/faucet/postgres_migration_test.go @@ -16,8 +16,6 @@ import ( "github.com/coder/quartz" ) -// The statements on the scan row of the build before migration -// 000008_redelivery_clear, as sqlc generated them at e8382ff. const ( mainEnsureRedeliveryScan = `INSERT INTO faucet_redelivery_scan (id, hook, epoch) VALUES (true, $1, 0) @@ -33,9 +31,6 @@ RETURNING epoch, top_at` mainReleaseRedeliveryLease = `UPDATE faucet_redelivery_scan SET lease_until = NULL WHERE epoch = $1` ) -// mainBuild takes the lease and applies a step that carries only a top, in -// the order and the transactions of the build before the migration. On an -// empty list that build's top is the Date. type mainBuild struct { db *sql.DB hook string From c873800f6a22fadfee64ec3a916a78c794b2d277 Mon Sep 17 00:00:00 2001 From: Thomas Kosiewski Date: Mon, 16 Nov 2026 14:12:11 +0000 Subject: [PATCH 18/18] test(faucet): name the statements of the build before migration 000008 for that build MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- _Generated with [`xum`](https://github.com/coder/xum) • Model: `anthropic:claude-opus-5-5` • Thinking: `high`_ --- faucet/postgres_migration_test.go | 38 +++++++++++++++---------------- 1 file changed, 19 insertions(+), 19 deletions(-) diff --git a/faucet/postgres_migration_test.go b/faucet/postgres_migration_test.go index 03c37a2..5293fce 100644 --- a/faucet/postgres_migration_test.go +++ b/faucet/postgres_migration_test.go @@ -17,45 +17,45 @@ import ( ) const ( - mainEnsureRedeliveryScan = `INSERT INTO faucet_redelivery_scan (id, hook, epoch) + before000008EnsureRedeliveryScan = `INSERT INTO faucet_redelivery_scan (id, hook, epoch) VALUES (true, $1, 0) ON CONFLICT (id) DO NOTHING` - mainLockRedeliveryScan = `SELECT hook, epoch, lease_until, top_at FROM faucet_redelivery_scan FOR UPDATE` - mainTakeRedeliveryLease = `UPDATE faucet_redelivery_scan + before000008LockRedeliveryScan = `SELECT hook, epoch, lease_until, top_at FROM faucet_redelivery_scan FOR UPDATE` + before000008TakeRedeliveryLease = `UPDATE faucet_redelivery_scan SET hook = $1, epoch = epoch + 1, lease_until = $2::timestamptz, top_at = CASE WHEN $3::boolean THEN NULL ELSE top_at END RETURNING epoch, top_at` - mainRaiseRedeliveryTop = `UPDATE faucet_redelivery_scan SET top_at = GREATEST(top_at, $1::timestamptz)` - mainReleaseRedeliveryLease = `UPDATE faucet_redelivery_scan SET lease_until = NULL WHERE epoch = $1` + before000008RaiseRedeliveryTop = `UPDATE faucet_redelivery_scan SET top_at = GREATEST(top_at, $1::timestamptz)` + before000008ReleaseRedeliveryLease = `UPDATE faucet_redelivery_scan SET lease_until = NULL WHERE epoch = $1` ) -type mainBuild struct { +type buildBefore000008 struct { db *sql.DB hook string } -func (b mainBuild) acquire(ctx context.Context, now time.Time) (epoch int64, err error) { +func (b buildBefore000008) acquire(ctx context.Context, now time.Time) (epoch int64, err error) { tx, err := b.db.BeginTx(ctx, &sql.TxOptions{Isolation: sql.LevelReadCommitted}) if err != nil { return 0, err } defer func() { _ = tx.Rollback() }() - if _, err := tx.ExecContext(ctx, mainEnsureRedeliveryScan, b.hook); err != nil { + if _, err := tx.ExecContext(ctx, before000008EnsureRedeliveryScan, b.hook); err != nil { return 0, err } if err := b.lock(ctx, tx, nil); err != nil { return 0, err } var top sql.NullTime - if err := tx.QueryRowContext(ctx, mainTakeRedeliveryLease, b.hook, now.Add(2*time.Minute), false).Scan(&epoch, &top); err != nil { + if err := tx.QueryRowContext(ctx, before000008TakeRedeliveryLease, b.hook, now.Add(2*time.Minute), false).Scan(&epoch, &top); err != nil { return 0, err } return epoch, tx.Commit() } -func (b mainBuild) raiseTop(ctx context.Context, epoch int64, top time.Time) error { +func (b buildBefore000008) raiseTop(ctx context.Context, epoch int64, top time.Time) error { tx, err := b.db.BeginTx(ctx, &sql.TxOptions{Isolation: sql.LevelReadCommitted}) if err != nil { return err @@ -68,19 +68,19 @@ func (b mainBuild) raiseTop(ctx context.Context, epoch int64, top time.Time) err if locked != epoch { return faucet.ErrRedeliveryLeaseLost } - if _, err := tx.ExecContext(ctx, mainRaiseRedeliveryTop, top); err != nil { + if _, err := tx.ExecContext(ctx, before000008RaiseRedeliveryTop, top); err != nil { return err } return tx.Commit() } -func (mainBuild) lock(ctx context.Context, tx *sql.Tx, epoch *int64) error { +func (buildBefore000008) lock(ctx context.Context, tx *sql.Tx, epoch *int64) error { var ( hook string e int64 lease, top sql.NullTime ) - if err := tx.QueryRowContext(ctx, mainLockRedeliveryScan).Scan(&hook, &e, &lease, &top); err != nil { + if err := tx.QueryRowContext(ctx, before000008LockRedeliveryScan).Scan(&hook, &e, &lease, &top); err != nil { return err } if epoch != nil { @@ -89,8 +89,8 @@ func (mainBuild) lock(ctx context.Context, tx *sql.Tx, epoch *int64) error { return nil } -func (b mainBuild) release(ctx context.Context, epoch int64) error { - _, err := b.db.ExecContext(ctx, mainReleaseRedeliveryLease, epoch) +func (b buildBefore000008) release(ctx context.Context, epoch int64) error { + _, err := b.db.ExecContext(ctx, before000008ReleaseRedeliveryLease, epoch) return err } @@ -127,7 +127,7 @@ func TestARunOfTheBuildBeforeTheClearMigrationFailsAfterItAndWritesNoPosition(t env.clock.WithLogger(quartz.NoOpLogger) db := postgresDB(t) env.use(pgstore.New(db), env.gh) - old := mainBuild{db: db, hook: env.gh.Hook()} + old := buildBefore000008{db: db, hook: env.gh.Hook()} migrate000008(t, db, "down") if tc.hadTop { @@ -159,9 +159,9 @@ func TestARunOfTheBuildBeforeTheClearMigrationFailsAfterItAndWritesNoPosition(t sql string args []any }{ - {mainLockRedeliveryScan, nil}, - {mainTakeRedeliveryLease, []any{old.hook, env.now(), false}}, - {mainRaiseRedeliveryTop, []any{env.gh.now()}}, + {before000008LockRedeliveryScan, nil}, + {before000008TakeRedeliveryLease, []any{old.hook, env.now(), false}}, + {before000008RaiseRedeliveryTop, []any{env.gh.now()}}, } { _, err := db.ExecContext(ctx, statement.sql, statement.args...) require.ErrorContains(t, err, `column "top_at" does not exist`, "no statement of the old build reads or writes the position: %s", statement.sql)