From 3e4366c3426b44e77c2a5f3cd0a367f2620985c7 Mon Sep 17 00:00:00 2001 From: Thomas Kosiewski Date: Mon, 16 Nov 2026 15:05:24 +0000 Subject: [PATCH 1/3] fix(githttp): wait for the sweep restore of a missing registered pack MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The orphan sweep can delete the bytes of a pack that a writer registered a moment before, and it restores them within the same pass. A hydration that started inside that window listed the packs 8 times with no wait and failed with ErrPackMissing, so a cold read answered 500 until the restore landed. The hydration now waits between two listings that find the same pack missing (50 ms, doubling to 1 s). A pack that a fold replaced is not listed again, so a fold race still retries at once. The doc comment of Sweep no longer says that the retry absorbs the window when it did not. Signed-off-by: Thomas Kosiewski --- _Generated with [`xum`](https://github.com/coder/xum) • Model: `anthropic:claude-sonnet-5-5` • Thinking: `xhigh`_ Change-Id: I85a778564e1f0af05b2b857d80b007798270a1d2 --- internal/githttp/hydrate.go | 27 ++- internal/githttp/hydrate_test.go | 56 +++++ internal/jobs/sweep.go | 7 +- internal/jobs/sweep_window_test.go | 317 +++++++++++++++++++++++++++++ 4 files changed, 404 insertions(+), 3 deletions(-) create mode 100644 internal/jobs/sweep_window_test.go diff --git a/internal/githttp/hydrate.go b/internal/githttp/hydrate.go index 923269120..10bbde88f 100644 --- a/internal/githttp/hydrate.go +++ b/internal/githttp/hydrate.go @@ -886,6 +886,18 @@ func ensureHook(path, script string) (os.FileInfo, error) { // genuinely gone, not racing. const maxHydrateRaces = 8 +// hydrateMissingWait is the first wait between two listings that both find the +// same pack missing (it doubles each time, to hydrateMissingWaitMax). A fold +// that replaced the pack lists the replacement, so a pack that is listed and +// missing twice in a row is no fold race. The orphan sweep can delete the bytes +// of a pack that a writer registered just before, and restores them within one +// pass: a reader that starts inside that window waits for the restore instead +// of failing within milliseconds. Tests shorten it. +var ( + hydrateMissingWait = 50 * time.Millisecond + hydrateMissingWaitMax = time.Second +) + // ErrPackMissing is returned (wrapped, match with errors.Is) when a // registered pack's blob stays missing after maxHydrateRaces listing passes: // the ref store references bytes that do not exist, which is corruption @@ -925,12 +937,15 @@ func hydratePacksBy(ctx context.Context, bl blob.BlobStore, repoPath, repo strin return err } var lastMissing error - for range maxHydrateRaces { + var prevMissingID string + wait := hydrateMissingWait + for attempt := range maxHydrateRaces { packs, err := list(ctx) if err != nil { return err } lastMissing = nil + missingID := "" for _, p := range packs { base := filepath.Join(packDir, "pack-"+p.ID) if _, err := os.Stat(base + ".idx"); err == nil { @@ -941,12 +956,22 @@ func hydratePacksBy(ctx context.Context, bl blob.BlobStore, repoPath, repo strin return err } lastMissing = fmt.Errorf("pack %s: %w", p.ID, err) + missingID = p.ID break } } if lastMissing == nil { return nil } + if missingID == prevMissingID && attempt < maxHydrateRaces-1 { + select { + case <-ctx.Done(): + return ctx.Err() + case <-time.After(wait): + } + wait = min(wait*2, hydrateMissingWaitMax) + } + prevMissingID = missingID } return fmt.Errorf("hydrate %s: %w after %d attempts: %v", repo, ErrPackMissing, maxHydrateRaces, lastMissing) } diff --git a/internal/githttp/hydrate_test.go b/internal/githttp/hydrate_test.go index c46e0f659..2ecde2166 100644 --- a/internal/githttp/hydrate_test.go +++ b/internal/githttp/hydrate_test.go @@ -679,6 +679,7 @@ func TestHydratePacks_MissingPackExhaustsBound(t *testing.T) { defer bl.Close() gone := ref.Pack{ID: "gone", Key: blob.PackKey(repo, "gone")} st := &scriptedPacks{lists: [][]ref.Pack{{gone}}} + shortenHydrateWait(t) repoPath, err := EnsureCacheRepo(t.TempDir(), repo) if err != nil { @@ -696,6 +697,61 @@ func TestHydratePacks_MissingPackExhaustsBound(t *testing.T) { } } +// shortenHydrateWait makes the wait between two listings that find the same +// pack missing 1 ms for the test. The test must not run in parallel. +func shortenHydrateWait(t *testing.T) { + t.Helper() + oldWait, oldMax := hydrateMissingWait, hydrateMissingWaitMax + hydrateMissingWait, hydrateMissingWaitMax = time.Millisecond, time.Millisecond + t.Cleanup(func() { hydrateMissingWait, hydrateMissingWaitMax = oldWait, oldMax }) +} + +// A pack that is listed and missing on consecutive listings is no fold race: +// the sweeper can have deleted the bytes of a pack that a writer registered, and +// restores them within one pass. The hydration waits between the listings for +// the restore. A pack that a fold replaced needs no wait. +func TestHydratePacks_WaitsForAPackThatStaysMissing(t *testing.T) { + ctx := context.Background() + const repo = "t-hydrate-wait" + bl, err := blob.OpenFile("file://" + t.TempDir()) + if err != nil { + t.Fatalf("OpenFile: %v", err) + } + defer bl.Close() + back := ref.Pack{ID: "back", Key: blob.PackKey(repo, "back")} + oldWait, oldMax := hydrateMissingWait, hydrateMissingWaitMax + hydrateMissingWait, hydrateMissingWaitMax = 100*time.Millisecond, 100*time.Millisecond + defer func() { hydrateMissingWait, hydrateMissingWaitMax = oldWait, oldMax }() + // The blob comes back 250 ms after the first listing. + packFile := buildTestPack(t) + go func() { + time.Sleep(250 * time.Millisecond) + _ = bl.UploadFile(ctx, back.Key, packFile) + }() + st := &scriptedPacks{lists: [][]ref.Pack{{back}}} + repoPath, err := EnsureCacheRepo(t.TempDir(), repo) + if err != nil { + t.Fatalf("EnsureCacheRepo: %v", err) + } + if err := HydratePacks(ctx, st, bl, repoPath, repo); err != nil { + t.Fatalf("hydrate of a pack that comes back within the wait: %v", err) + } + if st.calls < 3 { + t.Fatalf("ListPacks called %d times, want at least 3 (the wait covers the first two misses)", st.calls) + } + + // The wait ends with the context. + gone := ref.Pack{ID: "gone", Key: blob.PackKey(repo, "gone")} + hydrateMissingWait, hydrateMissingWaitMax = time.Hour, time.Hour + cctx, cancel := context.WithTimeout(ctx, 200*time.Millisecond) + defer cancel() + start := time.Now() + err = HydratePacks(cctx, &scriptedPacks{lists: [][]ref.Pack{{gone}}}, bl, repoPath, repo) + if !errors.Is(err, context.DeadlineExceeded) || time.Since(start) > 5*time.Second { + t.Fatalf("hydrate with a context that ends = %v after %v, want the context error soon", err, time.Since(start)) + } +} + // Transport errors must keep failing fast: no re-list, no retry, and no // misclassification as a missing pack. func TestHydratePacks_TransportErrorFailsFast(t *testing.T) { diff --git a/internal/jobs/sweep.go b/internal/jobs/sweep.go index c396acaef..95c125f95 100644 --- a/internal/jobs/sweep.go +++ b/internal/jobs/sweep.go @@ -322,8 +322,11 @@ func NewSweeper(blobs blob.BlobStore, packs PackRegistry, cfg SweepConfig, logge // assumptions (see EnsurePublished's ordering proof). // // Any residual exposure (e.g. a reader racing the restore itself) is -// transient, not data loss — hydration's bounded missing-pack retry -// absorbs it — and the next sweep's heal pass remains the crash backstop. +// transient, not data loss: a registered pack has no bytes until the restore +// lands (one registry read plus one server-side copy), and a hydration that +// finds the same pack missing on consecutive listings waits between them +// (githttp.hydrateMissingWait) for that restore. The next sweep's heal pass +// remains the backstop when the restore fails or the sweeper dies. func (s *Sweeper) Sweep(ctx context.Context) (SweepReport, error) { report, err := s.sweep(ctx) // ADR-48: one outcome per execution, on the choke point both queue diff --git a/internal/jobs/sweep_window_test.go b/internal/jobs/sweep_window_test.go new file mode 100644 index 000000000..3ee72b08b --- /dev/null +++ b/internal/jobs/sweep_window_test.go @@ -0,0 +1,317 @@ +// Sweeper-vs-writer window tests (claim B): a writer re-registers an orphan +// pack key between the sweeper's classify and its delete. The sweeper and the +// writer use real stores (sqlite registry, file blob store, real git packs). +// Decorators over blob.BlobStore and PackRegistry park the sweeper at chosen +// calls, so every interleaving here is deterministic. +package jobs + +import ( + "context" + "errors" + "fmt" + "log/slog" + "os" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/coder/openstorage/internal/blob" + "github.com/coder/openstorage/internal/githttp" + "github.com/coder/openstorage/internal/ref" +) + +// windowPack is one pack the writer publishes: its key, local file and the +// commit its ref move points at. +type windowPack struct { + id, key, path, commit string + size int64 +} + +// mintOrphanPack builds a real pack for a new commit, uploads it, ages it +// past the sweep threshold and leaves it unregistered: an orphan the sweeper +// will classify for deletion. The local pack file stays for the writer. +func mintOrphanPack(t *testing.T, f *repackFixture) windowPack { + t.Helper() + ctx := context.Background() + f.seq++ + blobSHA := strings.TrimSpace(testGit(t, f.scratch, fmt.Sprintf("window %d\n", f.seq), "hash-object", "-w", "--stdin")) + treeSHA := strings.TrimSpace(testGit(t, f.scratch, fmt.Sprintf("100644 blob %s\tdata.txt\n", blobSHA), "mktree")) + commitSHA := strings.TrimSpace(testGit(t, f.scratch, "", "commit-tree", treeSHA, "-m", fmt.Sprintf("w%d", f.seq))) + packDir := t.TempDir() + objs := blobSHA + "\n" + treeSHA + "\n" + commitSHA + "\n" + id := strings.TrimSpace(testGit(t, f.scratch, objs, "pack-objects", "--threads=1", "-q", filepath.Join(packDir, "w"))) + p := windowPack{id: id, key: blob.PackKey(f.repo, id), path: filepath.Join(packDir, "w-"+id+".pack"), commit: commitSHA} + fi, err := os.Stat(p.path) + if err != nil { + t.Fatalf("stat pack: %v", err) + } + p.size = fi.Size() + if err := f.blobs.UploadFile(ctx, p.key, p.path); err != nil { + t.Fatalf("upload orphan: %v", err) + } + past := time.Now().Add(-25 * time.Hour) + if err := os.Chtimes(filepath.Join(f.blobDir, filepath.FromSlash(p.key)), past, past); err != nil { + t.Fatalf("backdate orphan: %v", err) + } + return p +} + +// The writer below runs the sweeper-relevant calls of api.publishPack with +// quota off, one step per helper so a test can interleave them: the Stat at +// publish.go:58, RegisterPacks at :175, blob.EnsurePublished at :200, then +// the caller's ref move. + +// writerStat is publish.go:58. A present key is "published": no upload. +func writerStat(t *testing.T, f *repackFixture, p windowPack) (published bool) { + t.Helper() + _, err := f.blobs.Stat(context.Background(), p.key) + if err != nil && !errors.Is(err, blob.ErrNotFound) { + t.Fatalf("writer stat: %v", err) + } + return err == nil +} + +// writerRegister is publish.go:175 for a published (adopted) pack. +func writerRegister(t *testing.T, f *repackFixture, p windowPack) { + t.Helper() + pack := ref.Pack{ID: p.id, Key: p.key, Size: p.size, Adopted: true} + if err := f.store.RegisterPacks(context.Background(), f.repo, []ref.Pack{pack}); err != nil { + t.Fatalf("writer register: %v", err) + } +} + +// writerVerify is publish.go:200: re-upload when the key is missing. +func writerVerify(t *testing.T, f *repackFixture, p windowPack) (restored bool) { + t.Helper() + restored, err := blob.EnsurePublished(context.Background(), f.blobs, p.key, p.path, nil, nil) + if err != nil { + t.Fatalf("writer verify: %v", err) + } + return restored +} + +// writerMoveRef is the ref move the caller makes once publishPack returns. +func writerMoveRef(t *testing.T, f *repackFixture, p windowPack) { + t.Helper() + won, err := f.store.CASRef(context.Background(), f.repo, "refs/heads/main", f.tip, p.commit) + if err != nil || !won { + t.Fatalf("writer ref move: won=%v err=%v", won, err) + } + f.tip = p.commit +} + +// windowBlobs is the sweeper's blob store. It calls stop before the +// sweeper deletes key ("delete") and before it copies a backup back onto key +// ("restore"), and records when each inner call returned. +type windowBlobs struct { + blob.BlobStore + key string + stop func(step string) error + deletedAt, restoredAt time.Time +} + +func (b *windowBlobs) Delete(ctx context.Context, key string) error { + if key != b.key { + return b.BlobStore.Delete(ctx, key) + } + if err := b.stop("delete"); err != nil { + return err + } + err := b.BlobStore.Delete(ctx, key) + b.deletedAt = time.Now() + return err +} + +func (b *windowBlobs) Copy(ctx context.Context, dst, src string) error { + if dst != b.key { // the quarantine copy writes the backup key + return b.BlobStore.Copy(ctx, dst, src) + } + if err := b.stop("restore"); err != nil { + return err + } + err := b.BlobStore.Copy(ctx, dst, src) + b.restoredAt = time.Now() + return err +} + +// windowPacks is the sweeper's registry. Its third HasPack of key is +// safeDelete's post-delete recheck (1 = classify, 2 = revalidateOrphan), and +// it calls stop before that read. +type windowPacks struct { + PackRegistry + key string + calls int + stop func(step string) error +} + +func (p *windowPacks) HasPack(ctx context.Context, repo, key string) (bool, error) { + if key == p.key { + p.calls++ + if p.calls == 3 { + if err := p.stop("recheck"); err != nil { + return false, err + } + } + } + return p.PackRegistry.HasPack(ctx, repo, key) +} + +// parker parks the sweeper goroutine at a step until the test resumes it. +type parker struct { + at chan string + resume chan error +} + +func newParker() *parker { return &parker{at: make(chan string), resume: make(chan error)} } + +func (p *parker) stop(step string) error { + p.at <- step + return <-p.resume +} + +func (p *parker) expect(t *testing.T, step string) { + t.Helper() + select { + case got := <-p.at: + if got != step { + t.Fatalf("sweeper parked at %q, want %q", got, step) + } + case <-time.After(30 * time.Second): + t.Fatalf("sweeper never reached %q", step) + } +} + +// windowState is what a reader sees: the registry row, the blob, and a +// cold HydratePacks of the repo into a fresh cache. +type windowState struct { + registered, present bool + hydrateErr error + hydrateTook time.Duration +} + +func observeWindow(t *testing.T, f *repackFixture, key string) windowState { + t.Helper() + ctx := context.Background() + var st windowState + var err error + if st.registered, err = f.store.HasPack(ctx, f.repo, key); err != nil { + t.Fatalf("HasPack: %v", err) + } + st.present = f.blobExists(t, key) + repoPath, err := githttp.EnsureCacheRepo(t.TempDir(), f.repo) + if err != nil { + t.Fatalf("EnsureCacheRepo: %v", err) + } + start := time.Now() + st.hydrateErr = githttp.HydratePacks(ctx, f.store, f.blobs, repoPath, f.repo) + st.hydrateTook = time.Since(start) + return st +} + +// wantWedged fails unless the registry lists key, its blob is gone, and a +// cold hydrate fails with ErrPackMissing. +func wantWedged(t *testing.T, where string, st windowState) { + t.Helper() + t.Logf("%s: registered=%v present=%v hydrate=%v (in %v)", where, st.registered, st.present, st.hydrateErr, st.hydrateTook) + if !st.registered || st.present || !errors.Is(st.hydrateErr, githttp.ErrPackMissing) { + t.Fatalf("%s: want registered row, no blob, ErrPackMissing hydrate; got %+v", where, st) + } +} + +func wantHealthy(t *testing.T, where string, st windowState) { + t.Helper() + t.Logf("%s: registered=%v present=%v hydrate=%v", where, st.registered, st.present, st.hydrateErr) + if !st.registered || !st.present || st.hydrateErr != nil { + t.Fatalf("%s: want registered row, blob present, hydrate ok; got %+v", where, st) + } +} + +func windowSweeper(f *repackFixture, key string, stop func(string) error) (*Sweeper, *windowBlobs) { + bl := &windowBlobs{BlobStore: f.blobs, key: key, stop: stop} + pk := &windowPacks{PackRegistry: f.store, key: key, stop: stop} + return NewSweeper(bl, pk, SweepConfig{Threshold: 24 * time.Hour, Delete: true}, slog.New(slog.DiscardHandler)), bl +} + +// Claim B as stated: the writer's whole publishPack (Stat sees the key, +// RegisterPacks, EnsurePublished sees the key, ref move) lands after classify +// and before the delete. The delete then removes the bytes of a committed +// ref's pack. The post-delete recheck sees the row and restores the bytes in +// the same pass. Between the delete and the end of the restore copy, a cold +// hydrate of the repo fails with ErrPackMissing. +func TestSweepWindow_RefCommittedBeforeDeleteIsRestoredInPass(t *testing.T) { + f := newRepackFixture(t) + f.push(t) // a base commit and pack, so main exists before the writer + p := mintOrphanPack(t, f) + park := newParker() + s, _ := windowSweeper(f, p.key, park.stop) + + var report SweepReport + var sweepErr error + done := make(chan struct{}) + go func() { defer close(done); report, sweepErr = s.Sweep(context.Background()) }() + + park.expect(t, "delete") // classify and revalidateOrphan saw K unregistered and aged + if !writerStat(t, f, p) { + t.Fatal("writer Stat must see the key before the delete") + } + writerRegister(t, f, p) + if writerVerify(t, f, p) { + t.Fatal("EnsurePublished must pass without a re-upload before the delete") + } + writerMoveRef(t, f, p) + park.resume <- nil + + park.expect(t, "recheck") // the delete ran + wantWedged(t, "after delete, before recheck", observeWindow(t, f, p.key)) + park.resume <- nil + + park.expect(t, "restore") // the recheck saw the row + wantWedged(t, "after recheck, before restore copy", observeWindow(t, f, p.key)) + park.resume <- nil + <-done + + if sweepErr != nil || report.Deleted != 0 { + t.Fatalf("sweep = %+v, err %v; want no reap", report, sweepErr) + } + wantHealthy(t, "after the sweep returns", observeWindow(t, f, p.key)) + f.checkColdHydrate(t) +} + +// Sweeper.Sweep's doc comment says that a reader racing the restore is +// transient. The hydration retry must wait for the restore: a cold hydrate that +// starts inside the window, with the restore 200 ms away, used to fail with +// ErrPackMissing within milliseconds, because the retry listed again 8 times +// with no wait. +func TestSweepWindow_ColdHydrateInWindowWaitsForRestore(t *testing.T) { + f := newRepackFixture(t) + f.push(t) + p := mintOrphanPack(t, f) + park := newParker() + s, _ := windowSweeper(f, p.key, park.stop) + done := make(chan struct{}) + go func() { defer close(done); _, _ = s.Sweep(context.Background()) }() + + park.expect(t, "delete") + writerStat(t, f, p) + writerRegister(t, f, p) + writerVerify(t, f, p) + writerMoveRef(t, f, p) + park.resume <- nil + park.expect(t, "recheck") + park.resume <- nil + park.expect(t, "restore") + + repoPath, err := githttp.EnsureCacheRepo(t.TempDir(), f.repo) + if err != nil { + t.Fatalf("EnsureCacheRepo: %v", err) + } + hydrated := make(chan error, 1) + go func() { hydrated <- githttp.HydratePacks(context.Background(), f.store, f.blobs, repoPath, f.repo) }() + time.Sleep(200 * time.Millisecond) + park.resume <- nil // the restore copy runs + <-done + if err := <-hydrated; err != nil { + t.Fatalf("cold hydrate that started 200ms before the sweeper's restore: %v", err) + } +} From c49a78b39d4525833475ee0f1cfc0d69a9a67b04 Mon Sep 17 00:00:00 2001 From: Thomas Kosiewski Date: Mon, 16 Nov 2026 15:21:24 +0000 Subject: [PATCH 2/3] chore(verify): record lane-schedule fingerprints for the hydration wait MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Thomas Kosiewski --- _Generated with [`xum`](https://github.com/coder/xum) • Model: `anthropic:claude-sonnet-5-5` • Thinking: `xhigh`_ Change-Id: Icbc8515a2fc19d1ff22e458790fd38a0dc635abc --- .agents/skills/verify-lane-schedule/SKILL.md | 2 +- .agents/skills/verify-lane-schedule/model/fingerprints.txt | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/.agents/skills/verify-lane-schedule/SKILL.md b/.agents/skills/verify-lane-schedule/SKILL.md index ce7cc8227..6ee01b40e 100644 --- a/.agents/skills/verify-lane-schedule/SKILL.md +++ b/.agents/skills/verify-lane-schedule/SKILL.md @@ -226,7 +226,7 @@ With every fact TRUE in the code, setting each fact FALSE in the model alone fai ## Coverage record -Sync commit: `a792629c551f5f5e8065e33e288804e77e38bbd9` (the last commit at which someone confirmed that the model describes this code; only `check.sh --record` moves it). +Sync commit: `3e4366c3426b44e77c2a5f3cd0a367f2620985c7` (the last commit at which someone confirmed that the model describes this code; only `check.sh --record` moves it). Every declaration in these files is covered: diff --git a/.agents/skills/verify-lane-schedule/model/fingerprints.txt b/.agents/skills/verify-lane-schedule/model/fingerprints.txt index c2418529c..c691da9d5 100644 --- a/.agents/skills/verify-lane-schedule/model/fingerprints.txt +++ b/.agents/skills/verify-lane-schedule/model/fingerprints.txt @@ -316,7 +316,7 @@ internal/githttp/hydrate.go:holdFileIdentity 4451039fff307c74d3a416e209140826edc internal/githttp/hydrate.go:holdLineCovers 68e0cad1251be7349a4179f64a25ab38c92ac2e473d6e986c078acaf556520fe internal/githttp/hydrate.go:holdSidecarOwner d344d481c0a6961091973246ecd86c36a1adf26024a7f84daade1a3beb8b72ec internal/githttp/hydrate.go:hydrateOnePack 65e2d6f356099992e8b9e49e8f22855fad64e053f8d4936e87b6ec14b4f382fa -internal/githttp/hydrate.go:hydratePacksBy c8410824050b8b8c2048395646cb00e2a04625bbfead2a0d30b6e0f819e837ca +internal/githttp/hydrate.go:hydratePacksBy 596858a2e0363c40e36ec6db3ef581324457041c17c5f9c42c863d33d88cc624 internal/githttp/hydrate.go:hydrateRefs 10c0aee5ef4b4352de3f3e1d03e899a86951f88b76538a038b6be16f3af840d3 internal/githttp/hydrate.go:import_"bufio" c8716a546ce61a4869563c666058adbec5f4783581ea4f74788f87ec1f0731f1 internal/githttp/hydrate.go:import_"bytes" 151fc787b0679fc8767e367d1971b9f907660a3d72d8fa3888877ef9f8ebd9c3 @@ -419,6 +419,8 @@ internal/githttp/hydrate.go:var_errConfigValueMismatch 6072c7d12edf5f49796836c2e internal/githttp/hydrate.go:var_errPushScanBudget 21c09212ded082bfd4aaab0be40dfe2db6fb09bb6483778ea83d5d7112ff94be internal/githttp/hydrate.go:var_gitSpawnObserver dad15a1d668f4997e06a2c42ecf9814ef4d288463c954ed8503701b311c4d761 internal/githttp/hydrate.go:var_hookVerifyBarrier dec9fba51ee077b370309f5ed4434c9a35961db2b11d69d72854955ca34c4be7 +internal/githttp/hydrate.go:var_hydrateMissingWait 5e73ece8bff435c5d7eaa025de4a7845dd4c89afe3f7f4e23174fcb9569bae36 +internal/githttp/hydrate.go:var_hydrateMissingWaitMax f0661515a42c0fbfa62d8057e46687d5b20c3606976a313e31a9734781e3cd5e internal/githttp/hydrate.go:var_linkFile 47e0a13d524db684a6284b85a674578d5522836e5d22acb6b89909a1d44653d9 internal/githttp/hydrate.go:var_maxPushPeerScanBytes 66203545f4422b2f2c96c6b2ef37e9f1851a855c391a8405cff2b2288909701b internal/githttp/hydrate.go:var_pushHoldTTL d3b2bcf904b9d88d9983388e6f79346bf8b8f4c6466047f7db4f4607ac6b91cb From 491e92a4c1c3731675e8f123155fb8d94dd7737e Mon Sep 17 00:00:00 2001 From: Thomas Kosiewski Date: Mon, 16 Nov 2026 16:07:05 +0000 Subject: [PATCH 3/3] test(githttp): make the hydration wait tests wait for the miss, not for the clock MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The sweeper test waits until the hydration has read the missing pack before it releases the restore. The wait test starts its timer at the first listing. A new test checks that a fold race retries at once, and the sweeper test file keeps only the red test, which saves 5 seconds. Signed-off-by: Thomas Kosiewski --- _Generated with [`xum`](https://github.com/coder/xum) • Model: `anthropic:claude-sonnet-5-5` • Thinking: `xhigh`_ Change-Id: Id8298a2ea63ab49164ba2bc51c22aaa0452b91cc --- internal/githttp/hydrate_test.go | 55 +++++++++++-- internal/jobs/sweep_window_test.go | 120 +++++++---------------------- 2 files changed, 76 insertions(+), 99 deletions(-) diff --git a/internal/githttp/hydrate_test.go b/internal/githttp/hydrate_test.go index 2ecde2166..365f10eac 100644 --- a/internal/githttp/hydrate_test.go +++ b/internal/githttp/hydrate_test.go @@ -706,6 +706,23 @@ func shortenHydrateWait(t *testing.T) { t.Cleanup(func() { hydrateMissingWait, hydrateMissingWaitMax = oldWait, oldMax }) } +// timedRestoreStore lists one pack, and starts a timer at the first listing that +// brings the pack's blob back: a sweeper that restores the pack's bytes. +type timedRestoreStore struct { + ref.RefStore + pack ref.Pack + after time.Duration + back func() + calls int +} + +func (s *timedRestoreStore) ListPacks(context.Context, string) ([]ref.Pack, error) { + if s.calls++; s.calls == 1 { + time.AfterFunc(s.after, s.back) + } + return []ref.Pack{s.pack}, nil +} + // A pack that is listed and missing on consecutive listings is no fold race: // the sweeper can have deleted the bytes of a pack that a writer registered, and // restores them within one pass. The hydration waits between the listings for @@ -722,17 +739,14 @@ func TestHydratePacks_WaitsForAPackThatStaysMissing(t *testing.T) { oldWait, oldMax := hydrateMissingWait, hydrateMissingWaitMax hydrateMissingWait, hydrateMissingWaitMax = 100*time.Millisecond, 100*time.Millisecond defer func() { hydrateMissingWait, hydrateMissingWaitMax = oldWait, oldMax }() - // The blob comes back 250 ms after the first listing. packFile := buildTestPack(t) - go func() { - time.Sleep(250 * time.Millisecond) - _ = bl.UploadFile(ctx, back.Key, packFile) - }() - st := &scriptedPacks{lists: [][]ref.Pack{{back}}} repoPath, err := EnsureCacheRepo(t.TempDir(), repo) if err != nil { t.Fatalf("EnsureCacheRepo: %v", err) } + // The blob comes back 250 ms after the first listing: more than the 8 + // listings take without a wait, less than the waits cover. + st := &timedRestoreStore{pack: back, after: 250 * time.Millisecond, back: func() { _ = bl.UploadFile(ctx, back.Key, packFile) }} if err := HydratePacks(ctx, st, bl, repoPath, repo); err != nil { t.Fatalf("hydrate of a pack that comes back within the wait: %v", err) } @@ -752,6 +766,35 @@ func TestHydratePacks_WaitsForAPackThatStaysMissing(t *testing.T) { } } +// A fold race retries at once: the second listing carries the replacement, so +// the missing pack differs and the hydration does not wait. +func TestHydratePacks_FoldRaceRetriesWithoutWaiting(t *testing.T) { + const repo = "t-hydrate-fold-fast" + bl, err := blob.OpenFile("file://" + t.TempDir()) + if err != nil { + t.Fatalf("OpenFile: %v", err) + } + defer bl.Close() + replacement := ref.Pack{ID: "new", Key: blob.PackKey(repo, "new")} + if err := bl.UploadFile(context.Background(), replacement.Key, buildTestPack(t)); err != nil { + t.Fatalf("UploadFile: %v", err) + } + folded := ref.Pack{ID: "old", Key: blob.PackKey(repo, "old")} + oldWait, oldMax := hydrateMissingWait, hydrateMissingWaitMax + hydrateMissingWait, hydrateMissingWaitMax = time.Hour, time.Hour + defer func() { hydrateMissingWait, hydrateMissingWaitMax = oldWait, oldMax }() + repoPath, err := EnsureCacheRepo(t.TempDir(), repo) + if err != nil { + t.Fatalf("EnsureCacheRepo: %v", err) + } + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + st := &scriptedPacks{lists: [][]ref.Pack{{folded}, {replacement}}} + if err := HydratePacks(ctx, st, bl, repoPath, repo); err != nil || st.calls != 2 { + t.Fatalf("hydrate across a fold = %v after %d listings, want success after 2 with no wait", err, st.calls) + } +} + // Transport errors must keep failing fast: no re-list, no retry, and no // misclassification as a missing pack. func TestHydratePacks_TransportErrorFailsFast(t *testing.T) { diff --git a/internal/jobs/sweep_window_test.go b/internal/jobs/sweep_window_test.go index 3ee72b08b..981e684af 100644 --- a/internal/jobs/sweep_window_test.go +++ b/internal/jobs/sweep_window_test.go @@ -13,6 +13,7 @@ import ( "os" "path/filepath" "strings" + "sync" "testing" "time" @@ -182,102 +183,12 @@ func (p *parker) expect(t *testing.T, step string) { } } -// windowState is what a reader sees: the registry row, the blob, and a -// cold HydratePacks of the repo into a fresh cache. -type windowState struct { - registered, present bool - hydrateErr error - hydrateTook time.Duration -} - -func observeWindow(t *testing.T, f *repackFixture, key string) windowState { - t.Helper() - ctx := context.Background() - var st windowState - var err error - if st.registered, err = f.store.HasPack(ctx, f.repo, key); err != nil { - t.Fatalf("HasPack: %v", err) - } - st.present = f.blobExists(t, key) - repoPath, err := githttp.EnsureCacheRepo(t.TempDir(), f.repo) - if err != nil { - t.Fatalf("EnsureCacheRepo: %v", err) - } - start := time.Now() - st.hydrateErr = githttp.HydratePacks(ctx, f.store, f.blobs, repoPath, f.repo) - st.hydrateTook = time.Since(start) - return st -} - -// wantWedged fails unless the registry lists key, its blob is gone, and a -// cold hydrate fails with ErrPackMissing. -func wantWedged(t *testing.T, where string, st windowState) { - t.Helper() - t.Logf("%s: registered=%v present=%v hydrate=%v (in %v)", where, st.registered, st.present, st.hydrateErr, st.hydrateTook) - if !st.registered || st.present || !errors.Is(st.hydrateErr, githttp.ErrPackMissing) { - t.Fatalf("%s: want registered row, no blob, ErrPackMissing hydrate; got %+v", where, st) - } -} - -func wantHealthy(t *testing.T, where string, st windowState) { - t.Helper() - t.Logf("%s: registered=%v present=%v hydrate=%v", where, st.registered, st.present, st.hydrateErr) - if !st.registered || !st.present || st.hydrateErr != nil { - t.Fatalf("%s: want registered row, blob present, hydrate ok; got %+v", where, st) - } -} - func windowSweeper(f *repackFixture, key string, stop func(string) error) (*Sweeper, *windowBlobs) { bl := &windowBlobs{BlobStore: f.blobs, key: key, stop: stop} pk := &windowPacks{PackRegistry: f.store, key: key, stop: stop} return NewSweeper(bl, pk, SweepConfig{Threshold: 24 * time.Hour, Delete: true}, slog.New(slog.DiscardHandler)), bl } -// Claim B as stated: the writer's whole publishPack (Stat sees the key, -// RegisterPacks, EnsurePublished sees the key, ref move) lands after classify -// and before the delete. The delete then removes the bytes of a committed -// ref's pack. The post-delete recheck sees the row and restores the bytes in -// the same pass. Between the delete and the end of the restore copy, a cold -// hydrate of the repo fails with ErrPackMissing. -func TestSweepWindow_RefCommittedBeforeDeleteIsRestoredInPass(t *testing.T) { - f := newRepackFixture(t) - f.push(t) // a base commit and pack, so main exists before the writer - p := mintOrphanPack(t, f) - park := newParker() - s, _ := windowSweeper(f, p.key, park.stop) - - var report SweepReport - var sweepErr error - done := make(chan struct{}) - go func() { defer close(done); report, sweepErr = s.Sweep(context.Background()) }() - - park.expect(t, "delete") // classify and revalidateOrphan saw K unregistered and aged - if !writerStat(t, f, p) { - t.Fatal("writer Stat must see the key before the delete") - } - writerRegister(t, f, p) - if writerVerify(t, f, p) { - t.Fatal("EnsurePublished must pass without a re-upload before the delete") - } - writerMoveRef(t, f, p) - park.resume <- nil - - park.expect(t, "recheck") // the delete ran - wantWedged(t, "after delete, before recheck", observeWindow(t, f, p.key)) - park.resume <- nil - - park.expect(t, "restore") // the recheck saw the row - wantWedged(t, "after recheck, before restore copy", observeWindow(t, f, p.key)) - park.resume <- nil - <-done - - if sweepErr != nil || report.Deleted != 0 { - t.Fatalf("sweep = %+v, err %v; want no reap", report, sweepErr) - } - wantHealthy(t, "after the sweep returns", observeWindow(t, f, p.key)) - f.checkColdHydrate(t) -} - // Sweeper.Sweep's doc comment says that a reader racing the restore is // transient. The hydration retry must wait for the restore: a cold hydrate that // starts inside the window, with the restore 200 ms away, used to fail with @@ -306,12 +217,35 @@ func TestSweepWindow_ColdHydrateInWindowWaitsForRestore(t *testing.T) { if err != nil { t.Fatalf("EnsureCacheRepo: %v", err) } + // The restore waits until the hydration has read the missing pack, so that + // the test always enters the window, whatever the scheduler does. + seen := ¬FoundSignal{BlobStore: f.blobs, seen: make(chan struct{})} hydrated := make(chan error, 1) - go func() { hydrated <- githttp.HydratePacks(context.Background(), f.store, f.blobs, repoPath, f.repo) }() - time.Sleep(200 * time.Millisecond) - park.resume <- nil // the restore copy runs + go func() { hydrated <- githttp.HydratePacks(context.Background(), f.store, seen, repoPath, f.repo) }() + select { + case <-seen.seen: + case <-time.After(30 * time.Second): + t.Fatal("the hydration never read the missing pack") + } + time.Sleep(200 * time.Millisecond) // the restore is 200 ms away from the first miss + park.resume <- nil // the restore copy runs <-done if err := <-hydrated; err != nil { t.Fatalf("cold hydrate that started 200ms before the sweeper's restore: %v", err) } } + +// notFoundSignal closes seen at the first download that finds no blob. +type notFoundSignal struct { + blob.BlobStore + once sync.Once + seen chan struct{} +} + +func (n *notFoundSignal) DownloadFile(ctx context.Context, key, path string) error { + err := n.BlobStore.DownloadFile(ctx, key, path) + if errors.Is(err, blob.ErrNotFound) { + n.once.Do(func() { close(n.seen) }) + } + return err +}